ai 7.0.100 → 7.0.102
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +36 -0
- package/dist/index.d.ts +82 -20
- package/dist/index.js +2017 -330
- package/dist/index.js.map +1 -1
- package/dist/internal/index.js +12 -5
- package/dist/internal/index.js.map +1 -1
- package/docs/03-agents/07-workflow-agent.mdx +1 -1
- package/docs/03-ai-sdk-core/36-realtime.mdx +41 -18
- package/docs/04-ai-sdk-ui/21-transport.mdx +1 -1
- package/docs/07-reference/02-ai-sdk-ui/05-use-realtime.mdx +364 -48
- package/docs/07-reference/04-ai-sdk-workflow/02-workflow-chat-transport.mdx +4 -4
- package/docs/07-reference/05-ai-sdk-errors/index.mdx +131 -36
- package/package.json +12 -12
- package/src/generate-text/execute-tools-from-stream.ts +7 -0
- package/src/generate-text/stream-text.ts +3 -1
- package/src/prompt/create-tool-model-output.ts +16 -5
- package/src/realtime/__fixtures__/fake-live-websocket.ts +71 -0
- package/src/realtime/__fixtures__/fake-realtime.ts +36 -0
- package/src/realtime/__fixtures__/fake-webrtc.ts +133 -0
- package/src/realtime/browser-realtime-audio.ts +107 -10
- package/src/realtime/browser-realtime-live-websocket.ts +247 -0
- package/src/realtime/browser-realtime-transport.ts +235 -69
- package/src/realtime/browser-realtime-webrtc.ts +582 -0
- package/src/realtime/encode-realtime-frame.ts +33 -0
- package/src/realtime/index.ts +1 -0
- package/src/realtime/realtime-attempt.ts +45 -0
- package/src/realtime/realtime-command-tracker.ts +81 -0
- package/src/realtime/realtime-event-channel.ts +170 -0
- package/src/realtime/realtime-event-reducer.ts +3 -0
- package/src/realtime/realtime-session-state.ts +65 -0
- package/src/realtime/realtime-session.ts +768 -218
- package/src/realtime/realtime-types.ts +1 -1
- package/src/realtime/validate-realtime-setup.ts +37 -0
package/dist/index.js
CHANGED
|
@@ -14,7 +14,7 @@ import {
|
|
|
14
14
|
createIdGenerator as createIdGenerator9,
|
|
15
15
|
dynamicTool,
|
|
16
16
|
experimental_toolCaller,
|
|
17
|
-
generateId as
|
|
17
|
+
generateId as generateId4,
|
|
18
18
|
jsonSchema,
|
|
19
19
|
parseJsonEventStream as parseJsonEventStream3,
|
|
20
20
|
tool,
|
|
@@ -1178,7 +1178,7 @@ import {
|
|
|
1178
1178
|
} from "@ai-sdk/provider-utils";
|
|
1179
1179
|
|
|
1180
1180
|
// src/version.ts
|
|
1181
|
-
var VERSION = true ? "7.0.
|
|
1181
|
+
var VERSION = true ? "7.0.102" : "0.0.0-test";
|
|
1182
1182
|
|
|
1183
1183
|
// src/util/download/download.ts
|
|
1184
1184
|
var download = async ({
|
|
@@ -2007,6 +2007,9 @@ function getMediaTypeFromUrl(url, fallbackMediaType = "application/octet-stream"
|
|
|
2007
2007
|
|
|
2008
2008
|
// src/prompt/create-tool-model-output.ts
|
|
2009
2009
|
import { getErrorMessage as getErrorMessage3 } from "@ai-sdk/provider";
|
|
2010
|
+
import {
|
|
2011
|
+
parseJSON
|
|
2012
|
+
} from "@ai-sdk/provider-utils";
|
|
2010
2013
|
async function createToolModelOutput({
|
|
2011
2014
|
toolCallId,
|
|
2012
2015
|
input,
|
|
@@ -2017,15 +2020,19 @@ async function createToolModelOutput({
|
|
|
2017
2020
|
if (errorMode === "text") {
|
|
2018
2021
|
return { type: "error-text", value: getErrorMessage3(output) };
|
|
2019
2022
|
} else if (errorMode === "json") {
|
|
2020
|
-
return { type: "error-json", value: toJSONValue(output) };
|
|
2023
|
+
return { type: "error-json", value: await toJSONValue(output) };
|
|
2021
2024
|
}
|
|
2022
2025
|
if (tool2 == null ? void 0 : tool2.toModelOutput) {
|
|
2023
2026
|
return await tool2.toModelOutput({ toolCallId, input, output });
|
|
2024
2027
|
}
|
|
2025
|
-
return typeof output === "string" ? { type: "text", value: output } : { type: "json", value: toJSONValue(output) };
|
|
2028
|
+
return typeof output === "string" ? { type: "text", value: output } : { type: "json", value: await toJSONValue(output) };
|
|
2026
2029
|
}
|
|
2027
|
-
function toJSONValue(value) {
|
|
2028
|
-
|
|
2030
|
+
async function toJSONValue(value) {
|
|
2031
|
+
if (value === void 0) {
|
|
2032
|
+
return null;
|
|
2033
|
+
}
|
|
2034
|
+
const serialized = JSON.stringify(value);
|
|
2035
|
+
return serialized === void 0 ? null : await parseJSON({ text: serialized });
|
|
2029
2036
|
}
|
|
2030
2037
|
|
|
2031
2038
|
// src/prompt/prepare-language-model-call-options.ts
|
|
@@ -5549,7 +5556,7 @@ async function generateText({
|
|
|
5549
5556
|
experimental_include,
|
|
5550
5557
|
include = experimental_include,
|
|
5551
5558
|
_internal: {
|
|
5552
|
-
generateId:
|
|
5559
|
+
generateId: generateId5 = originalGenerateId,
|
|
5553
5560
|
generateCallId = originalGenerateCallId,
|
|
5554
5561
|
now: now2 = now
|
|
5555
5562
|
} = {},
|
|
@@ -5949,7 +5956,7 @@ async function generateText({
|
|
|
5949
5956
|
}
|
|
5950
5957
|
);
|
|
5951
5958
|
const responseData = {
|
|
5952
|
-
id: (_b27 = (_a28 = result.response) == null ? void 0 : _a28.id) != null ? _b27 :
|
|
5959
|
+
id: (_b27 = (_a28 = result.response) == null ? void 0 : _a28.id) != null ? _b27 : generateId5(),
|
|
5953
5960
|
timestamp: (_d3 = (_c3 = result.response) == null ? void 0 : _c3.timestamp) != null ? _d3 : /* @__PURE__ */ new Date(),
|
|
5954
5961
|
modelId: (_f2 = (_e2 = result.response) == null ? void 0 : _e2.modelId) != null ? _f2 : stepModel.modelId,
|
|
5955
5962
|
headers: (_g2 = result.response) == null ? void 0 : _g2.headers,
|
|
@@ -6069,7 +6076,7 @@ async function generateText({
|
|
|
6069
6076
|
if (toolApprovalStatus.type === "not-applicable") {
|
|
6070
6077
|
continue;
|
|
6071
6078
|
}
|
|
6072
|
-
const approvalId =
|
|
6079
|
+
const approvalId = generateId5();
|
|
6073
6080
|
const signature = await maybeSignApproval({
|
|
6074
6081
|
secret: experimental_toolApprovalSecret,
|
|
6075
6082
|
approvalId,
|
|
@@ -8393,7 +8400,7 @@ function executeToolsFromStream({
|
|
|
8393
8400
|
toolApproval,
|
|
8394
8401
|
runtimeContext,
|
|
8395
8402
|
toolApprovalSecret,
|
|
8396
|
-
generateId:
|
|
8403
|
+
generateId: generateId5,
|
|
8397
8404
|
onToolExecutionStart,
|
|
8398
8405
|
onToolExecutionEnd,
|
|
8399
8406
|
executeToolInTelemetryContext,
|
|
@@ -8432,7 +8439,7 @@ function executeToolsFromStream({
|
|
|
8432
8439
|
}
|
|
8433
8440
|
return;
|
|
8434
8441
|
}
|
|
8435
|
-
const approvalId =
|
|
8442
|
+
const approvalId = generateId5();
|
|
8436
8443
|
const signature = await maybeSignApproval({
|
|
8437
8444
|
secret: toolApprovalSecret,
|
|
8438
8445
|
approvalId,
|
|
@@ -8467,6 +8474,11 @@ function executeToolsFromStream({
|
|
|
8467
8474
|
reason: toolApprovalStatus.reason,
|
|
8468
8475
|
providerExecuted: chunk.providerExecuted
|
|
8469
8476
|
});
|
|
8477
|
+
controller.enqueue({
|
|
8478
|
+
type: "tool-output-denied",
|
|
8479
|
+
toolCallId: chunk.toolCallId,
|
|
8480
|
+
toolName: chunk.toolName
|
|
8481
|
+
});
|
|
8470
8482
|
return;
|
|
8471
8483
|
}
|
|
8472
8484
|
case "approved": {
|
|
@@ -8715,7 +8727,7 @@ async function streamLanguageModelCall({
|
|
|
8715
8727
|
toolsContext,
|
|
8716
8728
|
experimental_sandbox: sandbox,
|
|
8717
8729
|
_internal: {
|
|
8718
|
-
generateId:
|
|
8730
|
+
generateId: generateId5 = originalGenerateId2,
|
|
8719
8731
|
generateCallId = originalGenerateCallId2,
|
|
8720
8732
|
now: now2 = now
|
|
8721
8733
|
} = {},
|
|
@@ -8801,7 +8813,7 @@ async function streamLanguageModelCall({
|
|
|
8801
8813
|
provider: resolvedModel.provider,
|
|
8802
8814
|
modelId: resolvedModel.modelId,
|
|
8803
8815
|
toolChoice: stepToolChoice,
|
|
8804
|
-
generateId:
|
|
8816
|
+
generateId: generateId5,
|
|
8805
8817
|
now: now2,
|
|
8806
8818
|
callStartTimestampMs,
|
|
8807
8819
|
onLanguageModelCallEnd
|
|
@@ -8823,7 +8835,7 @@ function createLanguageModelV4StreamPartToLanguageModelStreamPartTransform({
|
|
|
8823
8835
|
provider,
|
|
8824
8836
|
modelId,
|
|
8825
8837
|
toolChoice,
|
|
8826
|
-
generateId:
|
|
8838
|
+
generateId: generateId5,
|
|
8827
8839
|
now: now2,
|
|
8828
8840
|
callStartTimestampMs,
|
|
8829
8841
|
onLanguageModelCallEnd
|
|
@@ -8835,7 +8847,7 @@ function createLanguageModelV4StreamPartToLanguageModelStreamPartTransform({
|
|
|
8835
8847
|
const reasoningPartIndexes = /* @__PURE__ */ new Map();
|
|
8836
8848
|
const rawTextPartIndexes = /* @__PURE__ */ new Map();
|
|
8837
8849
|
const rawReasoningPartIndexes = /* @__PURE__ */ new Map();
|
|
8838
|
-
let responseId =
|
|
8850
|
+
let responseId = generateId5();
|
|
8839
8851
|
let responseModelId = modelId;
|
|
8840
8852
|
let timeToFirstOutputMs;
|
|
8841
8853
|
let previousOutputChunkTimestampMs;
|
|
@@ -9242,6 +9254,7 @@ var isOutputChunkType = {
|
|
|
9242
9254
|
"tool-call": true,
|
|
9243
9255
|
"tool-result": false,
|
|
9244
9256
|
"tool-error": false,
|
|
9257
|
+
"tool-output-denied": false,
|
|
9245
9258
|
"tool-execution-end": false,
|
|
9246
9259
|
"model-call-start": false,
|
|
9247
9260
|
"model-call-response-metadata": false,
|
|
@@ -9324,7 +9337,7 @@ function streamText({
|
|
|
9324
9337
|
include = experimental_include,
|
|
9325
9338
|
_internal: {
|
|
9326
9339
|
now: now2 = now,
|
|
9327
|
-
generateId:
|
|
9340
|
+
generateId: generateId5 = originalGenerateId3,
|
|
9328
9341
|
generateCallId = originalGenerateCallId3
|
|
9329
9342
|
} = {},
|
|
9330
9343
|
...settings
|
|
@@ -9403,7 +9416,7 @@ function streamText({
|
|
|
9403
9416
|
onToolExecutionStart: resolvedOnToolExecutionStart,
|
|
9404
9417
|
onToolExecutionEnd: resolvedOnToolExecutionEnd,
|
|
9405
9418
|
now: now2,
|
|
9406
|
-
generateId:
|
|
9419
|
+
generateId: generateId5,
|
|
9407
9420
|
generateCallId,
|
|
9408
9421
|
download: download2,
|
|
9409
9422
|
// assign default values to include:
|
|
@@ -9631,7 +9644,7 @@ var DefaultStreamTextResult = class {
|
|
|
9631
9644
|
providerOptions,
|
|
9632
9645
|
prepareStep,
|
|
9633
9646
|
now: now2,
|
|
9634
|
-
generateId:
|
|
9647
|
+
generateId: generateId5,
|
|
9635
9648
|
generateCallId,
|
|
9636
9649
|
timeout,
|
|
9637
9650
|
onChunk,
|
|
@@ -9686,7 +9699,7 @@ var DefaultStreamTextResult = class {
|
|
|
9686
9699
|
usedIds.add(id);
|
|
9687
9700
|
return id;
|
|
9688
9701
|
}
|
|
9689
|
-
const generatedId =
|
|
9702
|
+
const generatedId = generateId5();
|
|
9690
9703
|
let uniqueId = generatedId;
|
|
9691
9704
|
let suffix = 0;
|
|
9692
9705
|
while (usedIds.has(uniqueId)) {
|
|
@@ -10621,7 +10634,7 @@ var DefaultStreamTextResult = class {
|
|
|
10621
10634
|
toolApproval,
|
|
10622
10635
|
runtimeContext,
|
|
10623
10636
|
toolApprovalSecret: experimental_toolApprovalSecret,
|
|
10624
|
-
generateId:
|
|
10637
|
+
generateId: generateId5,
|
|
10625
10638
|
// the callbacks need to be passed down and handled by executeToolCall
|
|
10626
10639
|
// to guarantee that the onToolExecutionStart callback is invoked before the tool execute function
|
|
10627
10640
|
onToolExecutionStart: filterNullable2(
|
|
@@ -10664,7 +10677,7 @@ var DefaultStreamTextResult = class {
|
|
|
10664
10677
|
let modelCallPerformance = createModelCallPerformance();
|
|
10665
10678
|
const toolExecutionMs = {};
|
|
10666
10679
|
const createStepResponse = () => ({
|
|
10667
|
-
id:
|
|
10680
|
+
id: generateId5(),
|
|
10668
10681
|
timestamp: /* @__PURE__ */ new Date(),
|
|
10669
10682
|
modelId: model.modelId
|
|
10670
10683
|
});
|
|
@@ -10730,7 +10743,8 @@ var DefaultStreamTextResult = class {
|
|
|
10730
10743
|
case "tool-input-start":
|
|
10731
10744
|
case "tool-input-end":
|
|
10732
10745
|
case "tool-input-delta":
|
|
10733
|
-
case "tool-approval-request":
|
|
10746
|
+
case "tool-approval-request":
|
|
10747
|
+
case "tool-output-denied": {
|
|
10734
10748
|
enqueueStepPart(controller, chunk);
|
|
10735
10749
|
break;
|
|
10736
10750
|
}
|
|
@@ -11494,7 +11508,7 @@ function createUIMessageStream({
|
|
|
11494
11508
|
onStepFinish,
|
|
11495
11509
|
onEnd,
|
|
11496
11510
|
onFinish,
|
|
11497
|
-
generateId:
|
|
11511
|
+
generateId: generateId5 = generateIdFunc
|
|
11498
11512
|
}) {
|
|
11499
11513
|
let controller;
|
|
11500
11514
|
const ongoingStreamPromises = [];
|
|
@@ -11586,7 +11600,7 @@ function createUIMessageStream({
|
|
|
11586
11600
|
});
|
|
11587
11601
|
return handleUIMessageStreamFinish({
|
|
11588
11602
|
stream,
|
|
11589
|
-
messageId:
|
|
11603
|
+
messageId: generateId5(),
|
|
11590
11604
|
originalMessages,
|
|
11591
11605
|
onStepEnd: onStepEnd != null ? onStepEnd : onStepFinish,
|
|
11592
11606
|
onEnd: onEnd != null ? onEnd : onFinish,
|
|
@@ -14481,7 +14495,7 @@ async function generateObject(options) {
|
|
|
14481
14495
|
onStepFinish,
|
|
14482
14496
|
onFinish,
|
|
14483
14497
|
_internal: {
|
|
14484
|
-
generateId:
|
|
14498
|
+
generateId: generateId5 = originalGenerateId4,
|
|
14485
14499
|
currentDate = () => /* @__PURE__ */ new Date()
|
|
14486
14500
|
} = {},
|
|
14487
14501
|
...settings
|
|
@@ -14521,7 +14535,7 @@ async function generateObject(options) {
|
|
|
14521
14535
|
const resolvedOnStepStart = onStepStart != null ? onStepStart : experimental_onStepStart;
|
|
14522
14536
|
const resolvedOnStepEnd = onStepEnd != null ? onStepEnd : onStepFinish;
|
|
14523
14537
|
const jsonSchema2 = await outputStrategy.jsonSchema();
|
|
14524
|
-
const callId =
|
|
14538
|
+
const callId = generateId5();
|
|
14525
14539
|
await notify({
|
|
14526
14540
|
event: {
|
|
14527
14541
|
callId,
|
|
@@ -14590,7 +14604,7 @@ async function generateObject(options) {
|
|
|
14590
14604
|
})
|
|
14591
14605
|
);
|
|
14592
14606
|
const responseData = {
|
|
14593
|
-
id: (_b25 = (_a25 = generateResult.response) == null ? void 0 : _a25.id) != null ? _b25 :
|
|
14607
|
+
id: (_b25 = (_a25 = generateResult.response) == null ? void 0 : _a25.id) != null ? _b25 : generateId5(),
|
|
14594
14608
|
timestamp: (_d = (_c = generateResult.response) == null ? void 0 : _c.timestamp) != null ? _d : currentDate(),
|
|
14595
14609
|
modelId: (_f = (_e = generateResult.response) == null ? void 0 : _e.modelId) != null ? _f : model.modelId,
|
|
14596
14610
|
headers: (_g = generateResult.response) == null ? void 0 : _g.headers,
|
|
@@ -14850,7 +14864,7 @@ function streamObject(options) {
|
|
|
14850
14864
|
},
|
|
14851
14865
|
onFinish,
|
|
14852
14866
|
_internal: {
|
|
14853
|
-
generateId:
|
|
14867
|
+
generateId: generateId5 = originalGenerateId5,
|
|
14854
14868
|
currentDate = () => /* @__PURE__ */ new Date(),
|
|
14855
14869
|
now: now2 = now
|
|
14856
14870
|
} = {},
|
|
@@ -14897,7 +14911,7 @@ function streamObject(options) {
|
|
|
14897
14911
|
onError,
|
|
14898
14912
|
onFinish,
|
|
14899
14913
|
download: download2,
|
|
14900
|
-
generateId:
|
|
14914
|
+
generateId: generateId5,
|
|
14901
14915
|
currentDate,
|
|
14902
14916
|
now: now2
|
|
14903
14917
|
});
|
|
@@ -14926,7 +14940,7 @@ var DefaultStreamObjectResult = class {
|
|
|
14926
14940
|
onError,
|
|
14927
14941
|
onFinish,
|
|
14928
14942
|
download: download2,
|
|
14929
|
-
generateId:
|
|
14943
|
+
generateId: generateId5,
|
|
14930
14944
|
currentDate,
|
|
14931
14945
|
now: now2
|
|
14932
14946
|
}) {
|
|
@@ -14960,7 +14974,7 @@ var DefaultStreamObjectResult = class {
|
|
|
14960
14974
|
}
|
|
14961
14975
|
});
|
|
14962
14976
|
this.baseStream = stitchableStream.stream.pipeThrough(eventProcessor);
|
|
14963
|
-
const callId =
|
|
14977
|
+
const callId = generateId5();
|
|
14964
14978
|
(async () => {
|
|
14965
14979
|
const jsonSchema2 = await outputStrategy.jsonSchema();
|
|
14966
14980
|
await notify({
|
|
@@ -15058,7 +15072,7 @@ var DefaultStreamObjectResult = class {
|
|
|
15058
15072
|
let accumulatedText = "";
|
|
15059
15073
|
let textDelta = "";
|
|
15060
15074
|
let fullResponse = {
|
|
15061
|
-
id:
|
|
15075
|
+
id: generateId5(),
|
|
15062
15076
|
timestamp: currentDate(),
|
|
15063
15077
|
modelId: model.modelId
|
|
15064
15078
|
};
|
|
@@ -17022,6 +17036,9 @@ function resolveRealtimeToolDescription({
|
|
|
17022
17036
|
});
|
|
17023
17037
|
}
|
|
17024
17038
|
|
|
17039
|
+
// src/realtime/realtime-session.ts
|
|
17040
|
+
import { UnsupportedFunctionalityError as UnsupportedFunctionalityError6 } from "@ai-sdk/provider";
|
|
17041
|
+
|
|
17025
17042
|
// src/realtime/browser-realtime-audio.ts
|
|
17026
17043
|
var BrowserRealtimeAudio = class {
|
|
17027
17044
|
constructor(options) {
|
|
@@ -17029,16 +17046,23 @@ var BrowserRealtimeAudio = class {
|
|
|
17029
17046
|
this.captureProcessor = null;
|
|
17030
17047
|
this.captureSource = null;
|
|
17031
17048
|
this.captureStream = null;
|
|
17049
|
+
this.ownsCaptureStream = true;
|
|
17050
|
+
this.captureCleanup = [];
|
|
17051
|
+
this.captureGeneration = 0;
|
|
17032
17052
|
this.playbackContext = null;
|
|
17033
17053
|
this.playbackQueue = [];
|
|
17034
17054
|
this.playbackTime = 0;
|
|
17035
17055
|
this.playbackStartTime = 0;
|
|
17036
17056
|
this.activeSources = /* @__PURE__ */ new Set();
|
|
17037
17057
|
this.isPlaying = false;
|
|
17058
|
+
this.playbackPaused = false;
|
|
17059
|
+
var _a25;
|
|
17038
17060
|
this.captureSampleRate = options.captureSampleRate;
|
|
17039
17061
|
this.playbackSampleRate = options.playbackSampleRate;
|
|
17040
17062
|
this.onAudio = options.onAudio;
|
|
17041
17063
|
this.onPlayingChange = options.onPlayingChange;
|
|
17064
|
+
this.maxPlaybackSeconds = (_a25 = options.maxPlaybackSeconds) != null ? _a25 : Infinity;
|
|
17065
|
+
this.onError = options.onError;
|
|
17042
17066
|
this.onCapturingChange = options.onCapturingChange;
|
|
17043
17067
|
}
|
|
17044
17068
|
ensurePlaybackContext() {
|
|
@@ -17046,35 +17070,97 @@ var BrowserRealtimeAudio = class {
|
|
|
17046
17070
|
this.playbackContext = new AudioContext({
|
|
17047
17071
|
sampleRate: this.playbackSampleRate
|
|
17048
17072
|
});
|
|
17073
|
+
const context = this.playbackContext;
|
|
17074
|
+
context.onstatechange = () => {
|
|
17075
|
+
if (this.playbackContext === context)
|
|
17076
|
+
this.setPlaying(
|
|
17077
|
+
context.state === "running" && this.activeSources.size > 0
|
|
17078
|
+
);
|
|
17079
|
+
};
|
|
17049
17080
|
}
|
|
17050
17081
|
}
|
|
17051
|
-
|
|
17082
|
+
async resumePlayback() {
|
|
17083
|
+
var _a25;
|
|
17084
|
+
this.ensurePlaybackContext();
|
|
17085
|
+
const context = this.playbackContext;
|
|
17086
|
+
if (this.playbackPaused) this.stopPlayback();
|
|
17087
|
+
if (this.playbackContext !== context) return;
|
|
17088
|
+
await (context == null ? void 0 : context.resume());
|
|
17089
|
+
if (this.playbackContext !== context) return;
|
|
17090
|
+
this.playbackPaused = false;
|
|
17091
|
+
this.setPlaying(
|
|
17092
|
+
((_a25 = this.playbackContext) == null ? void 0 : _a25.state) === "running" && this.activeSources.size > 0
|
|
17093
|
+
);
|
|
17094
|
+
}
|
|
17095
|
+
startCapture(stream, options) {
|
|
17096
|
+
var _a25, _b25;
|
|
17097
|
+
const generation = this.captureGeneration + 1;
|
|
17098
|
+
this.stopCapture();
|
|
17099
|
+
if (generation !== this.captureGeneration) {
|
|
17100
|
+
if ((_a25 = options == null ? void 0 : options.ownsStream) != null ? _a25 : true)
|
|
17101
|
+
stream.getTracks().forEach((track) => track.stop());
|
|
17102
|
+
return;
|
|
17103
|
+
}
|
|
17104
|
+
this.captureStream = stream;
|
|
17105
|
+
this.ownsCaptureStream = (_b25 = options == null ? void 0 : options.ownsStream) != null ? _b25 : true;
|
|
17052
17106
|
const ctx = new AudioContext({ sampleRate: this.captureSampleRate });
|
|
17053
17107
|
this.captureContext = ctx;
|
|
17054
|
-
|
|
17108
|
+
void ctx.resume().catch((error) => {
|
|
17109
|
+
var _a26;
|
|
17110
|
+
if (this.captureContext === ctx) (_a26 = this.onError) == null ? void 0 : _a26.call(this, error);
|
|
17111
|
+
});
|
|
17055
17112
|
const source = ctx.createMediaStreamSource(stream);
|
|
17056
17113
|
this.captureSource = source;
|
|
17057
|
-
const processor = ctx.createScriptProcessor(
|
|
17114
|
+
const processor = ctx.createScriptProcessor(1024, 1, 1);
|
|
17058
17115
|
this.captureProcessor = processor;
|
|
17059
17116
|
processor.onaudioprocess = (event) => {
|
|
17117
|
+
var _a26;
|
|
17118
|
+
if (this.captureContext !== ctx) return;
|
|
17060
17119
|
const inputData = event.inputBuffer.getChannelData(0);
|
|
17061
17120
|
const samples = resampleAudio(
|
|
17062
17121
|
new Float32Array(inputData),
|
|
17063
17122
|
ctx.sampleRate,
|
|
17064
17123
|
this.captureSampleRate
|
|
17065
17124
|
);
|
|
17066
|
-
|
|
17125
|
+
try {
|
|
17126
|
+
this.onAudio(encodeRealtimeAudio(samples));
|
|
17127
|
+
} catch (error) {
|
|
17128
|
+
(_a26 = this.onError) == null ? void 0 : _a26.call(
|
|
17129
|
+
this,
|
|
17130
|
+
error instanceof Error ? error : new Error(String(error))
|
|
17131
|
+
);
|
|
17132
|
+
}
|
|
17067
17133
|
};
|
|
17068
17134
|
source.connect(processor);
|
|
17069
17135
|
processor.connect(ctx.destination);
|
|
17070
|
-
this.onCapturingChange(
|
|
17136
|
+
const updateCapturing = () => this.onCapturingChange(
|
|
17137
|
+
stream.getAudioTracks().some(
|
|
17138
|
+
(track) => track.enabled && !track.muted && track.readyState === "live"
|
|
17139
|
+
)
|
|
17140
|
+
);
|
|
17141
|
+
for (const track of stream.getAudioTracks()) {
|
|
17142
|
+
for (const event of ["ended", "mute", "unmute"]) {
|
|
17143
|
+
track.addEventListener(event, updateCapturing);
|
|
17144
|
+
this.captureCleanup.push(
|
|
17145
|
+
() => track.removeEventListener(event, updateCapturing)
|
|
17146
|
+
);
|
|
17147
|
+
}
|
|
17148
|
+
}
|
|
17149
|
+
updateCapturing();
|
|
17071
17150
|
}
|
|
17072
17151
|
stopCapture() {
|
|
17073
17152
|
var _a25, _b25, _c, _d;
|
|
17153
|
+
this.captureGeneration++;
|
|
17154
|
+
if (this.captureProcessor != null)
|
|
17155
|
+
this.captureProcessor.onaudioprocess = null;
|
|
17156
|
+
for (const cleanup of this.captureCleanup) cleanup();
|
|
17157
|
+
this.captureCleanup = [];
|
|
17074
17158
|
(_a25 = this.captureProcessor) == null ? void 0 : _a25.disconnect();
|
|
17075
17159
|
(_b25 = this.captureSource) == null ? void 0 : _b25.disconnect();
|
|
17076
|
-
void ((_c = this.captureContext) == null ? void 0 : _c.close())
|
|
17077
|
-
|
|
17160
|
+
void ((_c = this.captureContext) == null ? void 0 : _c.close().catch(() => {
|
|
17161
|
+
}));
|
|
17162
|
+
if (this.ownsCaptureStream)
|
|
17163
|
+
(_d = this.captureStream) == null ? void 0 : _d.getTracks().forEach((track) => track.stop());
|
|
17078
17164
|
this.captureProcessor = null;
|
|
17079
17165
|
this.captureSource = null;
|
|
17080
17166
|
this.captureContext = null;
|
|
@@ -17082,15 +17168,34 @@ var BrowserRealtimeAudio = class {
|
|
|
17082
17168
|
this.onCapturingChange(false);
|
|
17083
17169
|
}
|
|
17084
17170
|
playAudio(base64Audio) {
|
|
17171
|
+
var _a25, _b25, _c;
|
|
17172
|
+
if (this.playbackPaused) return;
|
|
17085
17173
|
this.ensurePlaybackContext();
|
|
17086
|
-
|
|
17174
|
+
const samples = decodeRealtimeAudio(base64Audio);
|
|
17175
|
+
if (Math.max(
|
|
17176
|
+
0,
|
|
17177
|
+
this.playbackTime - ((_b25 = (_a25 = this.playbackContext) == null ? void 0 : _a25.currentTime) != null ? _b25 : 0)
|
|
17178
|
+
) + samples.length / this.playbackSampleRate > this.maxPlaybackSeconds) {
|
|
17179
|
+
this.playbackPaused = true;
|
|
17180
|
+
this.stopPlayback();
|
|
17181
|
+
(_c = this.onError) == null ? void 0 : _c.call(
|
|
17182
|
+
this,
|
|
17183
|
+
new Error(
|
|
17184
|
+
"Realtime audio playback buffer is full; playback paused, call resumePlayback() to resume at the live edge"
|
|
17185
|
+
)
|
|
17186
|
+
);
|
|
17187
|
+
return;
|
|
17188
|
+
}
|
|
17189
|
+
this.playbackQueue.push(samples);
|
|
17087
17190
|
this.schedulePlayback();
|
|
17088
17191
|
}
|
|
17089
17192
|
stopPlayback() {
|
|
17090
17193
|
this.playbackQueue = [];
|
|
17091
17194
|
for (const source of this.activeSources) {
|
|
17092
17195
|
try {
|
|
17196
|
+
source.onended = null;
|
|
17093
17197
|
source.stop();
|
|
17198
|
+
source.disconnect();
|
|
17094
17199
|
} catch (e) {
|
|
17095
17200
|
}
|
|
17096
17201
|
}
|
|
@@ -17109,8 +17214,12 @@ var BrowserRealtimeAudio = class {
|
|
|
17109
17214
|
var _a25;
|
|
17110
17215
|
this.stopCapture();
|
|
17111
17216
|
this.stopPlayback();
|
|
17112
|
-
|
|
17217
|
+
if (this.playbackContext != null) this.playbackContext.onstatechange = null;
|
|
17218
|
+
void ((_a25 = this.playbackContext) == null ? void 0 : _a25.close().catch(() => {
|
|
17219
|
+
}));
|
|
17113
17220
|
this.playbackContext = null;
|
|
17221
|
+
this.playbackTime = 0;
|
|
17222
|
+
this.playbackPaused = false;
|
|
17114
17223
|
}
|
|
17115
17224
|
setPlaying(isPlaying) {
|
|
17116
17225
|
var _a25, _b25;
|
|
@@ -17140,8 +17249,10 @@ var BrowserRealtimeAudio = class {
|
|
|
17140
17249
|
source.start(startTime);
|
|
17141
17250
|
this.playbackTime = startTime + buffer.duration;
|
|
17142
17251
|
this.activeSources.add(source);
|
|
17143
|
-
this.setPlaying(
|
|
17252
|
+
this.setPlaying(ctx.state === "running");
|
|
17253
|
+
if (this.playbackContext !== ctx) return;
|
|
17144
17254
|
source.onended = () => {
|
|
17255
|
+
source.disconnect();
|
|
17145
17256
|
this.activeSources.delete(source);
|
|
17146
17257
|
if (this.playbackQueue.length === 0 && this.activeSources.size === 0) {
|
|
17147
17258
|
this.setPlaying(false);
|
|
@@ -17151,168 +17262,1323 @@ var BrowserRealtimeAudio = class {
|
|
|
17151
17262
|
}
|
|
17152
17263
|
};
|
|
17153
17264
|
|
|
17154
|
-
// src/realtime/
|
|
17265
|
+
// src/realtime/realtime-event-channel.ts
|
|
17155
17266
|
import { safeParseJSON as safeParseJSON5 } from "@ai-sdk/provider-utils";
|
|
17267
|
+
var RealtimeEventChannel = class {
|
|
17268
|
+
constructor(options) {
|
|
17269
|
+
this.options = options;
|
|
17270
|
+
this.active = true;
|
|
17271
|
+
this.finishing = false;
|
|
17272
|
+
this.incoming = Promise.resolve();
|
|
17273
|
+
this.outgoing = Promise.resolve();
|
|
17274
|
+
this.incomingCount = 0;
|
|
17275
|
+
this.incomingBytes = 0;
|
|
17276
|
+
this.outgoingCount = 0;
|
|
17277
|
+
var _a25, _b25, _c;
|
|
17278
|
+
this.parse = (_c = (_b25 = (_a25 = options.model).createServerEventParser) == null ? void 0 : _b25.call(_a25)) != null ? _c : options.model.parseServerEvent.bind(options.model);
|
|
17279
|
+
}
|
|
17280
|
+
dispose() {
|
|
17281
|
+
this.active = false;
|
|
17282
|
+
}
|
|
17283
|
+
/** Stop accepting messages, but deliver already received terminal events. */
|
|
17284
|
+
async finish() {
|
|
17285
|
+
this.stopWriting();
|
|
17286
|
+
await this.incoming;
|
|
17287
|
+
this.dispose();
|
|
17288
|
+
}
|
|
17289
|
+
stopWriting() {
|
|
17290
|
+
this.finishing = true;
|
|
17291
|
+
}
|
|
17292
|
+
send(event, shouldSend) {
|
|
17293
|
+
var _a25;
|
|
17294
|
+
if (!this.active || this.finishing)
|
|
17295
|
+
throw new Error("Realtime connection is closed");
|
|
17296
|
+
if (this.outgoingCount >= ((_a25 = this.options.maxPending) != null ? _a25 : 512)) {
|
|
17297
|
+
throw new Error("Realtime outgoing queue is full");
|
|
17298
|
+
}
|
|
17299
|
+
this.outgoingCount++;
|
|
17300
|
+
const operation = this.outgoing.then(async () => {
|
|
17301
|
+
if (!this.active || this.finishing)
|
|
17302
|
+
throw new Error("Realtime connection is closed");
|
|
17303
|
+
if ((shouldSend == null ? void 0 : shouldSend()) === false) return;
|
|
17304
|
+
if (!this.active || this.finishing)
|
|
17305
|
+
throw new Error("Realtime connection is closed");
|
|
17306
|
+
const data = await this.options.model.serializeClientEvent(event);
|
|
17307
|
+
if (!this.active || this.finishing)
|
|
17308
|
+
throw new Error("Realtime connection is closed");
|
|
17309
|
+
if ((shouldSend == null ? void 0 : shouldSend()) === false) return;
|
|
17310
|
+
if (!this.active || this.finishing)
|
|
17311
|
+
throw new Error("Realtime connection is closed");
|
|
17312
|
+
if (data != null) this.options.send(data);
|
|
17313
|
+
});
|
|
17314
|
+
this.outgoing = operation.catch((error) => {
|
|
17315
|
+
if (!this.finishing) {
|
|
17316
|
+
if (event.type !== "input-audio-append" && event.type !== "session-close")
|
|
17317
|
+
this.report(error);
|
|
17318
|
+
}
|
|
17319
|
+
}).finally(() => {
|
|
17320
|
+
this.outgoingCount--;
|
|
17321
|
+
});
|
|
17322
|
+
return operation;
|
|
17323
|
+
}
|
|
17324
|
+
receive(data) {
|
|
17325
|
+
var _a25, _b25, _c;
|
|
17326
|
+
if (!this.active || this.finishing) return;
|
|
17327
|
+
const bytes = typeof data === "string" ? data.length * 2 : data instanceof Blob ? data.size : (_a25 = data.byteLength) != null ? _a25 : 0;
|
|
17328
|
+
if (this.incomingCount >= ((_b25 = this.options.maxPending) != null ? _b25 : 512) || this.incomingBytes + bytes > ((_c = this.options.maxPendingBytes) != null ? _c : 8 * 1024 * 1024)) {
|
|
17329
|
+
this.finishing = true;
|
|
17330
|
+
this.fatal(
|
|
17331
|
+
new Error(
|
|
17332
|
+
"Realtime incoming queue is full; input protocol continuity was lost"
|
|
17333
|
+
)
|
|
17334
|
+
);
|
|
17335
|
+
return;
|
|
17336
|
+
}
|
|
17337
|
+
this.incomingCount++;
|
|
17338
|
+
this.incomingBytes += bytes;
|
|
17339
|
+
this.incoming = this.incoming.then(async () => {
|
|
17340
|
+
var _a26, _b26;
|
|
17341
|
+
if (!this.active) return;
|
|
17342
|
+
const text2 = typeof data === "string" ? data : data instanceof Blob ? await data.text() : new TextDecoder().decode(data);
|
|
17343
|
+
const parsed = await safeParseJSON5({ text: text2 });
|
|
17344
|
+
if (!this.active) return;
|
|
17345
|
+
if (!parsed.success)
|
|
17346
|
+
throw new Error("Invalid JSON in realtime server message", {
|
|
17347
|
+
cause: parsed.error
|
|
17348
|
+
});
|
|
17349
|
+
const health = (_b26 = (_a26 = this.options.model).getHealthCheckResponse) == null ? void 0 : _b26.call(
|
|
17350
|
+
_a26,
|
|
17351
|
+
parsed.value
|
|
17352
|
+
);
|
|
17353
|
+
if (!this.active) return;
|
|
17354
|
+
if (health != null && !this.finishing) this.options.send(health);
|
|
17355
|
+
if (!this.active) return;
|
|
17356
|
+
const result = this.parse(parsed.value);
|
|
17357
|
+
for (const event of Array.isArray(result) ? result : [result]) {
|
|
17358
|
+
if (!this.active) return;
|
|
17359
|
+
try {
|
|
17360
|
+
await this.options.onEvent(event);
|
|
17361
|
+
} catch (error) {
|
|
17362
|
+
this.report(error);
|
|
17363
|
+
}
|
|
17364
|
+
}
|
|
17365
|
+
}).catch((error) => {
|
|
17366
|
+
this.finishing = true;
|
|
17367
|
+
this.fatal(error instanceof Error ? error : new Error(String(error)));
|
|
17368
|
+
}).finally(() => {
|
|
17369
|
+
this.incomingCount--;
|
|
17370
|
+
this.incomingBytes -= bytes;
|
|
17371
|
+
});
|
|
17372
|
+
}
|
|
17373
|
+
fatal(error) {
|
|
17374
|
+
if (!this.active) return;
|
|
17375
|
+
if (this.options.onFatalError != null) this.options.onFatalError(error);
|
|
17376
|
+
else this.report(error);
|
|
17377
|
+
}
|
|
17378
|
+
report(error) {
|
|
17379
|
+
if (!this.active) return;
|
|
17380
|
+
try {
|
|
17381
|
+
this.options.onError(
|
|
17382
|
+
error instanceof Error ? error : new Error(String(error))
|
|
17383
|
+
);
|
|
17384
|
+
} catch (e) {
|
|
17385
|
+
}
|
|
17386
|
+
}
|
|
17387
|
+
};
|
|
17388
|
+
|
|
17389
|
+
// src/realtime/encode-realtime-frame.ts
|
|
17390
|
+
var REALTIME_MAX_FRAME_BYTES = 128 * 1024;
|
|
17391
|
+
var REALTIME_MAX_BUFFERED_BYTES = 128 * 1024;
|
|
17392
|
+
function encodeRealtimeFrame(value) {
|
|
17393
|
+
if (value instanceof ArrayBuffer || ArrayBuffer.isView(value))
|
|
17394
|
+
return { data: value, byteLength: value.byteLength };
|
|
17395
|
+
if (value instanceof Blob) return { data: value, byteLength: value.size };
|
|
17396
|
+
const data = typeof value === "string" ? value : JSON.stringify(value);
|
|
17397
|
+
if (data === void 0)
|
|
17398
|
+
throw new Error("Realtime event could not be encoded as JSON");
|
|
17399
|
+
return { data, byteLength: new TextEncoder().encode(data).byteLength };
|
|
17400
|
+
}
|
|
17401
|
+
function assertRealtimeFrameBudget(frame, bufferedAmount) {
|
|
17402
|
+
if (frame.byteLength > REALTIME_MAX_FRAME_BYTES)
|
|
17403
|
+
throw new Error(
|
|
17404
|
+
`Realtime frame exceeds the ${REALTIME_MAX_FRAME_BYTES}-byte limit; reduce the event payload`
|
|
17405
|
+
);
|
|
17406
|
+
if (bufferedAmount + frame.byteLength > REALTIME_MAX_BUFFERED_BYTES)
|
|
17407
|
+
throw new Error(
|
|
17408
|
+
`Realtime send buffer is full (${REALTIME_MAX_BUFFERED_BYTES}-byte limit); wait before retrying`
|
|
17409
|
+
);
|
|
17410
|
+
}
|
|
17411
|
+
|
|
17412
|
+
// src/realtime/browser-realtime-transport.ts
|
|
17413
|
+
function getCloseError(event) {
|
|
17414
|
+
if (event.code === 1e3 && event.wasClean) return void 0;
|
|
17415
|
+
return new Error(
|
|
17416
|
+
`Realtime WebSocket closed unexpectedly (code ${event.code}${event.reason === "" ? "" : `: ${event.reason}`})`
|
|
17417
|
+
);
|
|
17418
|
+
}
|
|
17156
17419
|
var BrowserRealtimeTransport = class {
|
|
17157
17420
|
constructor(options) {
|
|
17421
|
+
this.options = options;
|
|
17158
17422
|
this.ws = null;
|
|
17159
|
-
this.
|
|
17423
|
+
this.epoch = 0;
|
|
17424
|
+
this.failing = false;
|
|
17425
|
+
this.closing = false;
|
|
17426
|
+
var _a25;
|
|
17160
17427
|
this.model = options.model;
|
|
17428
|
+
this.continuous = ((_a25 = options.model.capabilities) == null ? void 0 : _a25.conversation) === "continuous";
|
|
17161
17429
|
this.onServerEvent = options.onServerEvent;
|
|
17162
17430
|
this.onError = options.onError;
|
|
17163
17431
|
this.onClose = options.onClose;
|
|
17164
17432
|
}
|
|
17433
|
+
get isOpen() {
|
|
17434
|
+
var _a25;
|
|
17435
|
+
return !this.closing && !this.failing && ((_a25 = this.ws) == null ? void 0 : _a25.readyState) === WebSocket.OPEN;
|
|
17436
|
+
}
|
|
17165
17437
|
connect({
|
|
17438
|
+
mode,
|
|
17166
17439
|
token,
|
|
17167
17440
|
url,
|
|
17168
|
-
onOpen
|
|
17441
|
+
onOpen,
|
|
17442
|
+
protocols
|
|
17169
17443
|
}) {
|
|
17170
|
-
var _a25;
|
|
17171
|
-
|
|
17172
|
-
|
|
17173
|
-
|
|
17174
|
-
|
|
17444
|
+
var _a25, _b25;
|
|
17445
|
+
this.disconnect();
|
|
17446
|
+
const epoch = this.epoch;
|
|
17447
|
+
if (mode !== "relay" && (mode !== "client-secret" || typeof token !== "string" || token.trim() === ""))
|
|
17448
|
+
throw new Error(
|
|
17449
|
+
"Realtime client-secret connection requires a nonempty token"
|
|
17450
|
+
);
|
|
17451
|
+
const wsConfig = mode === "relay" ? { url, protocols } : (_b25 = (_a25 = this.model).getWebSocketConfig) == null ? void 0 : _b25.call(_a25, { token, url });
|
|
17452
|
+
if (this.epoch !== epoch) return;
|
|
17453
|
+
if (wsConfig == null)
|
|
17454
|
+
throw new Error("Model does not support client-secret WebSockets");
|
|
17455
|
+
const finalUrl = wsConfig.url;
|
|
17456
|
+
const finalProtocols = wsConfig.protocols;
|
|
17457
|
+
if (this.epoch !== epoch) return;
|
|
17458
|
+
try {
|
|
17459
|
+
if (typeof finalUrl !== "string" || !/^wss?:\/\//i.test(finalUrl))
|
|
17460
|
+
throw new Error("Invalid realtime WebSocket URL");
|
|
17461
|
+
const parsed = new URL(finalUrl);
|
|
17462
|
+
if (!["ws:", "wss:"].includes(parsed.protocol) || parsed.hostname === "" || finalUrl.includes("#"))
|
|
17463
|
+
throw new Error("Invalid realtime WebSocket URL");
|
|
17464
|
+
} catch (e) {
|
|
17465
|
+
throw new Error(
|
|
17466
|
+
"Invalid realtime WebSocket URL; expected an absolute ws:// or wss:// URL with a host and no fragment"
|
|
17467
|
+
);
|
|
17468
|
+
}
|
|
17469
|
+
if (this.epoch !== epoch) return;
|
|
17470
|
+
const ws = new WebSocket(finalUrl, finalProtocols);
|
|
17175
17471
|
this.ws = ws;
|
|
17472
|
+
let starting = false;
|
|
17473
|
+
let connectionError;
|
|
17474
|
+
const codec = new RealtimeEventChannel({
|
|
17475
|
+
model: this.model,
|
|
17476
|
+
send: (data) => {
|
|
17477
|
+
if (this.epoch !== epoch || this.ws !== ws || !this.isOpen)
|
|
17478
|
+
throw new Error("Realtime WebSocket is not open");
|
|
17479
|
+
this.sendRaw(data);
|
|
17480
|
+
},
|
|
17481
|
+
onEvent: this.onServerEvent,
|
|
17482
|
+
onError: (error) => {
|
|
17483
|
+
if (!starting) this.onError(error);
|
|
17484
|
+
},
|
|
17485
|
+
onFatalError: (error) => this.fail(error)
|
|
17486
|
+
});
|
|
17487
|
+
if (this.epoch !== epoch) {
|
|
17488
|
+
codec.dispose();
|
|
17489
|
+
return;
|
|
17490
|
+
}
|
|
17491
|
+
this.codec = codec;
|
|
17176
17492
|
ws.onopen = () => {
|
|
17177
17493
|
if (this.ws !== ws) return;
|
|
17178
|
-
|
|
17494
|
+
starting = true;
|
|
17495
|
+
try {
|
|
17496
|
+
void Promise.resolve(onOpen()).catch((error) => {
|
|
17497
|
+
if (this.ws === ws)
|
|
17498
|
+
this.fail(
|
|
17499
|
+
error instanceof Error ? error : new Error(String(error))
|
|
17500
|
+
);
|
|
17501
|
+
}).finally(() => {
|
|
17502
|
+
starting = false;
|
|
17503
|
+
});
|
|
17504
|
+
} catch (error) {
|
|
17505
|
+
if (this.epoch === epoch)
|
|
17506
|
+
this.fail(error instanceof Error ? error : new Error(String(error)));
|
|
17507
|
+
}
|
|
17179
17508
|
};
|
|
17180
17509
|
ws.onmessage = (messageEvent) => {
|
|
17181
|
-
|
|
17510
|
+
var _a26;
|
|
17511
|
+
if (this.ws === ws) (_a26 = this.codec) == null ? void 0 : _a26.receive(messageEvent.data);
|
|
17182
17512
|
};
|
|
17183
17513
|
ws.onerror = () => {
|
|
17184
|
-
this.
|
|
17514
|
+
if (this.ws === ws) {
|
|
17515
|
+
connectionError = new Error("WebSocket connection error");
|
|
17516
|
+
this.failing = true;
|
|
17517
|
+
}
|
|
17185
17518
|
};
|
|
17186
|
-
ws.onclose = () => {
|
|
17519
|
+
ws.onclose = (event) => {
|
|
17520
|
+
var _a26;
|
|
17187
17521
|
if (this.ws === ws) {
|
|
17188
17522
|
this.ws = null;
|
|
17189
|
-
|
|
17523
|
+
const closeError = (_a26 = getCloseError(event)) != null ? _a26 : connectionError;
|
|
17524
|
+
codec.stopWriting();
|
|
17525
|
+
this.notifyClosing();
|
|
17526
|
+
if (this.epoch !== epoch) return;
|
|
17527
|
+
const complete = () => {
|
|
17528
|
+
if (this.epoch !== epoch) return;
|
|
17529
|
+
clearTimeout(this.drainTimer);
|
|
17530
|
+
this.epoch++;
|
|
17531
|
+
codec.dispose();
|
|
17532
|
+
try {
|
|
17533
|
+
this.onClose(closeError);
|
|
17534
|
+
} catch (error) {
|
|
17535
|
+
this.reportCallbackError(error);
|
|
17536
|
+
}
|
|
17537
|
+
};
|
|
17538
|
+
this.drainTimer = setTimeout(complete, 1e3);
|
|
17539
|
+
void this.awaitDrain(codec.finish(), complete);
|
|
17190
17540
|
}
|
|
17191
17541
|
};
|
|
17192
17542
|
}
|
|
17193
17543
|
disconnect() {
|
|
17194
17544
|
var _a25;
|
|
17195
|
-
|
|
17545
|
+
this.epoch++;
|
|
17546
|
+
clearTimeout(this.drainTimer);
|
|
17547
|
+
const ws = this.ws;
|
|
17196
17548
|
this.ws = null;
|
|
17549
|
+
(_a25 = this.codec) == null ? void 0 : _a25.dispose();
|
|
17550
|
+
this.codec = void 0;
|
|
17551
|
+
this.failing = false;
|
|
17552
|
+
this.closing = false;
|
|
17553
|
+
ws == null ? void 0 : ws.close();
|
|
17554
|
+
}
|
|
17555
|
+
sendEvent(event, shouldSend, automaticAudio = false) {
|
|
17556
|
+
if (this.failing || this.closing)
|
|
17557
|
+
throw new Error("Realtime connection is closed");
|
|
17558
|
+
if (this.codec == null) return Promise.resolve();
|
|
17559
|
+
const epoch = this.epoch;
|
|
17560
|
+
const failed = (error) => {
|
|
17561
|
+
if (automaticAudio && this.epoch === epoch && this.isOpen)
|
|
17562
|
+
this.fail(error instanceof Error ? error : new Error(String(error)));
|
|
17563
|
+
};
|
|
17564
|
+
try {
|
|
17565
|
+
const sent = this.codec.send(event, shouldSend);
|
|
17566
|
+
void sent.catch(failed);
|
|
17567
|
+
return sent;
|
|
17568
|
+
} catch (error) {
|
|
17569
|
+
failed(error);
|
|
17570
|
+
throw error;
|
|
17571
|
+
}
|
|
17197
17572
|
}
|
|
17198
|
-
|
|
17199
|
-
|
|
17200
|
-
|
|
17201
|
-
if (serialized != null) {
|
|
17202
|
-
this.sendRaw(serialized);
|
|
17203
|
-
}
|
|
17204
|
-
}).catch((error) => {
|
|
17205
|
-
this.onError(
|
|
17206
|
-
error instanceof Error ? error : new Error(`Failed to send realtime event: ${String(error)}`)
|
|
17207
|
-
);
|
|
17208
|
-
});
|
|
17573
|
+
finish() {
|
|
17574
|
+
var _a25, _b25;
|
|
17575
|
+
return (_b25 = (_a25 = this.codec) == null ? void 0 : _a25.finish()) != null ? _b25 : Promise.resolve();
|
|
17209
17576
|
}
|
|
17210
|
-
|
|
17577
|
+
/** Drain accepted events when media or protocol continuity is lost. */
|
|
17578
|
+
fail(error) {
|
|
17211
17579
|
var _a25;
|
|
17212
|
-
if (
|
|
17213
|
-
|
|
17214
|
-
|
|
17215
|
-
|
|
17216
|
-
|
|
17217
|
-
|
|
17580
|
+
if (this.failing) return;
|
|
17581
|
+
this.failing = true;
|
|
17582
|
+
const epoch = this.epoch;
|
|
17583
|
+
const ws = this.ws;
|
|
17584
|
+
this.ws = null;
|
|
17585
|
+
(_a25 = this.codec) == null ? void 0 : _a25.stopWriting();
|
|
17586
|
+
ws == null ? void 0 : ws.close();
|
|
17587
|
+
this.notifyClosing();
|
|
17588
|
+
if (this.epoch !== epoch) return;
|
|
17589
|
+
const drain = this.finish();
|
|
17590
|
+
if (this.options.onFatalError != null)
|
|
17591
|
+
this.options.onFatalError(error, drain);
|
|
17592
|
+
else {
|
|
17593
|
+
const complete = () => {
|
|
17594
|
+
if (epoch !== this.epoch) return;
|
|
17595
|
+
this.disconnect();
|
|
17596
|
+
try {
|
|
17597
|
+
this.onClose();
|
|
17598
|
+
} catch (cause) {
|
|
17599
|
+
this.reportCallbackError(cause);
|
|
17600
|
+
}
|
|
17601
|
+
};
|
|
17602
|
+
this.drainTimer = setTimeout(complete, 1e3);
|
|
17603
|
+
void this.awaitDrain(drain, complete);
|
|
17604
|
+
this.reportCallbackError(error);
|
|
17218
17605
|
}
|
|
17219
17606
|
}
|
|
17220
|
-
|
|
17221
|
-
|
|
17607
|
+
notifyClosing() {
|
|
17608
|
+
var _a25, _b25;
|
|
17609
|
+
if (this.closing) return;
|
|
17610
|
+
this.closing = true;
|
|
17611
|
+
try {
|
|
17612
|
+
(_b25 = (_a25 = this.options).onClosing) == null ? void 0 : _b25.call(_a25);
|
|
17613
|
+
} catch (error) {
|
|
17614
|
+
this.reportCallbackError(error);
|
|
17615
|
+
}
|
|
17222
17616
|
}
|
|
17223
|
-
async
|
|
17224
|
-
|
|
17225
|
-
|
|
17226
|
-
|
|
17227
|
-
} else if (messageEvent.data instanceof Blob) {
|
|
17228
|
-
text2 = await messageEvent.data.text();
|
|
17229
|
-
} else {
|
|
17230
|
-
text2 = new TextDecoder().decode(messageEvent.data);
|
|
17617
|
+
async awaitDrain(drain, complete) {
|
|
17618
|
+
try {
|
|
17619
|
+
await drain;
|
|
17620
|
+
} catch (e) {
|
|
17231
17621
|
}
|
|
17232
|
-
|
|
17233
|
-
|
|
17234
|
-
|
|
17235
|
-
|
|
17236
|
-
const autoResponse = this.model.getHealthCheckResponse(rawEvent);
|
|
17237
|
-
if (autoResponse != null) {
|
|
17238
|
-
this.sendRaw(autoResponse);
|
|
17239
|
-
}
|
|
17622
|
+
try {
|
|
17623
|
+
complete();
|
|
17624
|
+
} catch (error) {
|
|
17625
|
+
this.reportCallbackError(error);
|
|
17240
17626
|
}
|
|
17241
|
-
|
|
17242
|
-
|
|
17243
|
-
|
|
17244
|
-
|
|
17627
|
+
}
|
|
17628
|
+
reportCallbackError(error) {
|
|
17629
|
+
try {
|
|
17630
|
+
this.onError(error instanceof Error ? error : new Error(String(error)));
|
|
17631
|
+
} catch (e) {
|
|
17245
17632
|
}
|
|
17246
17633
|
}
|
|
17634
|
+
sendRaw(data) {
|
|
17635
|
+
const ws = this.ws;
|
|
17636
|
+
const epoch = this.epoch;
|
|
17637
|
+
if (ws == null || !this.isOpen) return;
|
|
17638
|
+
const frame = encodeRealtimeFrame(data);
|
|
17639
|
+
if (this.epoch !== epoch || this.ws !== ws || !this.isOpen)
|
|
17640
|
+
throw new Error("Realtime connection is closed");
|
|
17641
|
+
if (this.continuous) assertRealtimeFrameBudget(frame, ws.bufferedAmount);
|
|
17642
|
+
ws.send(frame.data);
|
|
17643
|
+
}
|
|
17644
|
+
dispose() {
|
|
17645
|
+
this.disconnect();
|
|
17646
|
+
}
|
|
17247
17647
|
};
|
|
17248
17648
|
|
|
17249
|
-
// src/realtime/realtime-
|
|
17250
|
-
import {
|
|
17251
|
-
|
|
17252
|
-
|
|
17253
|
-
|
|
17254
|
-
|
|
17255
|
-
|
|
17256
|
-
|
|
17257
|
-
|
|
17258
|
-
|
|
17259
|
-
|
|
17260
|
-
var
|
|
17261
|
-
|
|
17262
|
-
|
|
17263
|
-
|
|
17264
|
-
|
|
17265
|
-
|
|
17266
|
-
|
|
17267
|
-
|
|
17268
|
-
this.
|
|
17269
|
-
|
|
17270
|
-
|
|
17271
|
-
|
|
17272
|
-
|
|
17649
|
+
// src/realtime/browser-realtime-live-websocket.ts
|
|
17650
|
+
import { UnsupportedFunctionalityError as UnsupportedFunctionalityError5 } from "@ai-sdk/provider";
|
|
17651
|
+
var BrowserRealtimeLiveWebSocket = class {
|
|
17652
|
+
constructor(options) {
|
|
17653
|
+
this.options = options;
|
|
17654
|
+
this.generation = 0;
|
|
17655
|
+
this.ready = false;
|
|
17656
|
+
this.capturingStarted = false;
|
|
17657
|
+
this.pendingAudio = 0;
|
|
17658
|
+
this.captureGeneration = 0;
|
|
17659
|
+
this.captureEnabled = true;
|
|
17660
|
+
var _a25, _b25, _c, _d, _e, _f, _g, _h, _i, _j, _k, _l;
|
|
17661
|
+
const input = (_a25 = options.sessionConfig) == null ? void 0 : _a25.inputAudioFormat;
|
|
17662
|
+
const output = (_b25 = options.sessionConfig) == null ? void 0 : _b25.outputAudioFormat;
|
|
17663
|
+
const sampleRate = (_e = (_d = (_c = input == null ? void 0 : input.rate) != null ? _c : output == null ? void 0 : output.rate) != null ? _d : options.sampleRate) != null ? _e : 24e3;
|
|
17664
|
+
const format = {
|
|
17665
|
+
type: (_g = (_f = input == null ? void 0 : input.type) != null ? _f : output == null ? void 0 : output.type) != null ? _g : "audio/pcm",
|
|
17666
|
+
rate: sampleRate
|
|
17667
|
+
};
|
|
17668
|
+
this.config = {
|
|
17669
|
+
...options.sessionConfig,
|
|
17670
|
+
inputAudioFormat: {
|
|
17671
|
+
...format,
|
|
17672
|
+
...input
|
|
17673
|
+
},
|
|
17674
|
+
outputAudioFormat: {
|
|
17675
|
+
...format,
|
|
17676
|
+
...output
|
|
17677
|
+
}
|
|
17678
|
+
};
|
|
17679
|
+
this.transport = new BrowserRealtimeTransport({
|
|
17680
|
+
model: options.model,
|
|
17681
|
+
onServerEvent: options.onEvent,
|
|
17682
|
+
onError: options.onError,
|
|
17683
|
+
onFatalError: options.onFatalError,
|
|
17684
|
+
onClosing: () => {
|
|
17685
|
+
var _a26;
|
|
17686
|
+
const generation = this.generation;
|
|
17687
|
+
(_a26 = options.onClosing) == null ? void 0 : _a26.call(options);
|
|
17688
|
+
if (generation === this.generation) this.stopCapture();
|
|
17689
|
+
},
|
|
17690
|
+
onClose: options.onClose
|
|
17691
|
+
});
|
|
17692
|
+
this.audio = new BrowserRealtimeAudio({
|
|
17693
|
+
captureSampleRate: (_i = (_h = this.config.inputAudioFormat) == null ? void 0 : _h.rate) != null ? _i : sampleRate,
|
|
17694
|
+
playbackSampleRate: (_k = (_j = this.config.outputAudioFormat) == null ? void 0 : _j.rate) != null ? _k : sampleRate,
|
|
17695
|
+
maxPlaybackSeconds: (_l = options.maxPlaybackBufferSeconds) != null ? _l : 2,
|
|
17696
|
+
onAudio: (audio) => this.sendAudio(audio),
|
|
17697
|
+
onError: options.onError,
|
|
17698
|
+
onCapturingChange: options.onCapturing,
|
|
17699
|
+
onPlayingChange: options.onPlaying
|
|
17700
|
+
});
|
|
17273
17701
|
}
|
|
17274
|
-
|
|
17275
|
-
|
|
17702
|
+
connect(options) {
|
|
17703
|
+
const generation = this.generation + 1;
|
|
17704
|
+
this.dispose();
|
|
17705
|
+
if (generation !== this.generation) return;
|
|
17706
|
+
for (const format of [
|
|
17707
|
+
this.config.inputAudioFormat,
|
|
17708
|
+
this.config.outputAudioFormat
|
|
17709
|
+
]) {
|
|
17710
|
+
if ((format == null ? void 0 : format.type) !== "audio/pcm")
|
|
17711
|
+
throw new UnsupportedFunctionalityError5({
|
|
17712
|
+
functionality: "Continuous browser WebSocket audio supports PCM16 only"
|
|
17713
|
+
});
|
|
17714
|
+
if (format.rate == null || !Number.isFinite(format.rate) || format.rate < 8e3 || format.rate > 96e3)
|
|
17715
|
+
throw new Error("Invalid realtime PCM sample rate");
|
|
17716
|
+
}
|
|
17717
|
+
this.stream = options.stream;
|
|
17718
|
+
this.captureEnabled = options.capture !== false;
|
|
17719
|
+
this.audio.ensurePlaybackContext();
|
|
17720
|
+
void this.audio.resumePlayback().catch((error) => {
|
|
17721
|
+
if (generation === this.generation) this.options.onError(error);
|
|
17722
|
+
});
|
|
17723
|
+
this.transport.connect({
|
|
17724
|
+
mode: "relay",
|
|
17725
|
+
url: options.url,
|
|
17726
|
+
protocols: options.protocols,
|
|
17727
|
+
onOpen: () => {
|
|
17728
|
+
var _a25, _b25;
|
|
17729
|
+
return this.transport.sendEvent({
|
|
17730
|
+
type: (_b25 = (_a25 = this.options.model.capabilities) == null ? void 0 : _a25.startup) != null ? _b25 : "session-update",
|
|
17731
|
+
config: this.config
|
|
17732
|
+
});
|
|
17733
|
+
}
|
|
17734
|
+
});
|
|
17276
17735
|
}
|
|
17277
|
-
|
|
17278
|
-
|
|
17736
|
+
startCapture() {
|
|
17737
|
+
this.ready = true;
|
|
17738
|
+
if (!this.captureEnabled || this.capturingStarted) return;
|
|
17739
|
+
void this.resumeCapture().catch((error) => this.options.onError(error));
|
|
17279
17740
|
}
|
|
17280
|
-
|
|
17281
|
-
|
|
17282
|
-
|
|
17283
|
-
|
|
17284
|
-
|
|
17285
|
-
|
|
17286
|
-
|
|
17287
|
-
|
|
17288
|
-
|
|
17289
|
-
|
|
17290
|
-
|
|
17291
|
-
|
|
17741
|
+
async resumeCapture(suppliedStream) {
|
|
17742
|
+
const generation = this.captureGeneration + 1;
|
|
17743
|
+
this.stopCapture();
|
|
17744
|
+
if (generation !== this.captureGeneration) return;
|
|
17745
|
+
this.ready = true;
|
|
17746
|
+
this.capturingStarted = true;
|
|
17747
|
+
this.captureEnabled = true;
|
|
17748
|
+
if (suppliedStream != null) this.stream = suppliedStream;
|
|
17749
|
+
const supplied = this.stream;
|
|
17750
|
+
const stream = supplied != null ? supplied : await navigator.mediaDevices.getUserMedia({ audio: true });
|
|
17751
|
+
if (generation !== this.captureGeneration || !this.ready) {
|
|
17752
|
+
if (supplied == null) stream.getTracks().forEach((track) => track.stop());
|
|
17753
|
+
return;
|
|
17754
|
+
}
|
|
17755
|
+
if (!stream.getAudioTracks().some((track) => track.readyState === "live")) {
|
|
17756
|
+
if (supplied == null) stream.getTracks().forEach((track) => track.stop());
|
|
17757
|
+
throw new Error("Realtime requires a live audio track");
|
|
17758
|
+
}
|
|
17759
|
+
this.audio.startCapture(stream, { ownsStream: supplied == null });
|
|
17292
17760
|
}
|
|
17293
|
-
|
|
17294
|
-
return
|
|
17295
|
-
|
|
17296
|
-
|
|
17297
|
-
|
|
17298
|
-
|
|
17299
|
-
|
|
17761
|
+
sendAudio(audio) {
|
|
17762
|
+
if (!this.ready || !this.transport.isOpen) return;
|
|
17763
|
+
if (this.pendingAudio >= 8) {
|
|
17764
|
+
this.transport.fail(new Error("Realtime audio send queue is full"));
|
|
17765
|
+
return;
|
|
17766
|
+
}
|
|
17767
|
+
const generation = this.generation;
|
|
17768
|
+
let sent;
|
|
17769
|
+
try {
|
|
17770
|
+
sent = this.transport.sendEvent(
|
|
17771
|
+
{ type: "input-audio-append", audio },
|
|
17772
|
+
void 0,
|
|
17773
|
+
true
|
|
17774
|
+
);
|
|
17775
|
+
} catch (error) {
|
|
17776
|
+
if (!this.ready || !this.transport.isOpen) return;
|
|
17777
|
+
throw error;
|
|
17778
|
+
}
|
|
17779
|
+
this.pendingAudio++;
|
|
17780
|
+
void sent.then(
|
|
17781
|
+
() => {
|
|
17782
|
+
if (generation === this.generation) this.pendingAudio--;
|
|
17783
|
+
},
|
|
17784
|
+
(error) => {
|
|
17785
|
+
if (generation === this.generation && this.ready && this.transport.isOpen)
|
|
17786
|
+
this.options.onFatalError(error);
|
|
17300
17787
|
}
|
|
17301
|
-
|
|
17788
|
+
);
|
|
17302
17789
|
}
|
|
17303
|
-
|
|
17304
|
-
var _a25;
|
|
17305
|
-
|
|
17306
|
-
|
|
17307
|
-
|
|
17308
|
-
|
|
17309
|
-
|
|
17310
|
-
|
|
17311
|
-
|
|
17790
|
+
sendEvent(event, shouldSend) {
|
|
17791
|
+
var _a25, _b25;
|
|
17792
|
+
if (event.type === "session-update") {
|
|
17793
|
+
for (const key of ["inputAudioFormat", "outputAudioFormat"]) {
|
|
17794
|
+
const update = event.config[key];
|
|
17795
|
+
if (update != null && (update.type !== ((_a25 = this.config[key]) == null ? void 0 : _a25.type) || update.rate != null && update.rate !== ((_b25 = this.config[key]) == null ? void 0 : _b25.rate))) {
|
|
17796
|
+
throw new Error(
|
|
17797
|
+
"Changing Live WebSocket audio format requires reconnecting"
|
|
17798
|
+
);
|
|
17312
17799
|
}
|
|
17313
|
-
break;
|
|
17314
17800
|
}
|
|
17315
|
-
|
|
17801
|
+
}
|
|
17802
|
+
return this.transport.sendEvent(event, shouldSend);
|
|
17803
|
+
}
|
|
17804
|
+
playAudio(audio) {
|
|
17805
|
+
try {
|
|
17806
|
+
this.audio.playAudio(audio);
|
|
17807
|
+
} catch (error) {
|
|
17808
|
+
this.options.onError(
|
|
17809
|
+
error instanceof Error ? error : new Error(String(error))
|
|
17810
|
+
);
|
|
17811
|
+
}
|
|
17812
|
+
}
|
|
17813
|
+
stopPlayback() {
|
|
17814
|
+
this.audio.stopPlayback();
|
|
17815
|
+
}
|
|
17816
|
+
resumePlayback() {
|
|
17817
|
+
return this.audio.resumePlayback();
|
|
17818
|
+
}
|
|
17819
|
+
stopCapture() {
|
|
17820
|
+
this.captureGeneration++;
|
|
17821
|
+
this.ready = false;
|
|
17822
|
+
this.capturingStarted = false;
|
|
17823
|
+
this.captureEnabled = false;
|
|
17824
|
+
this.audio.stopCapture();
|
|
17825
|
+
}
|
|
17826
|
+
finish() {
|
|
17827
|
+
return this.transport.finish();
|
|
17828
|
+
}
|
|
17829
|
+
dispose() {
|
|
17830
|
+
this.generation++;
|
|
17831
|
+
this.stopCapture();
|
|
17832
|
+
this.ready = false;
|
|
17833
|
+
this.capturingStarted = false;
|
|
17834
|
+
this.pendingAudio = 0;
|
|
17835
|
+
this.stream = void 0;
|
|
17836
|
+
this.transport.disconnect();
|
|
17837
|
+
this.audio.dispose();
|
|
17838
|
+
}
|
|
17839
|
+
};
|
|
17840
|
+
|
|
17841
|
+
// src/realtime/browser-realtime-webrtc.ts
|
|
17842
|
+
import { safeParseJSON as safeParseJSON6 } from "@ai-sdk/provider-utils";
|
|
17843
|
+
var MAX_SESSION_ANSWER_BYTES = 1024 * 1024;
|
|
17844
|
+
async function readSessionAnswer(response) {
|
|
17845
|
+
var _a25;
|
|
17846
|
+
const reader = (_a25 = response.body) == null ? void 0 : _a25.getReader();
|
|
17847
|
+
if (reader == null) throw new Error("Invalid realtime session answer");
|
|
17848
|
+
const decoder = new TextDecoder();
|
|
17849
|
+
let bytes = 0;
|
|
17850
|
+
let text2 = "";
|
|
17851
|
+
try {
|
|
17852
|
+
if (Number(response.headers.get("content-length")) > MAX_SESSION_ANSWER_BYTES)
|
|
17853
|
+
throw new Error("Realtime session answer exceeds the 1 MiB limit");
|
|
17854
|
+
while (true) {
|
|
17855
|
+
const { done, value } = await reader.read();
|
|
17856
|
+
if (done) return text2 + decoder.decode();
|
|
17857
|
+
bytes += value.byteLength;
|
|
17858
|
+
if (bytes > MAX_SESSION_ANSWER_BYTES)
|
|
17859
|
+
throw new Error("Realtime session answer exceeds the 1 MiB limit");
|
|
17860
|
+
text2 += decoder.decode(value, { stream: true });
|
|
17861
|
+
}
|
|
17862
|
+
} catch (error) {
|
|
17863
|
+
void reader.cancel(error).catch(() => {
|
|
17864
|
+
});
|
|
17865
|
+
throw error;
|
|
17866
|
+
} finally {
|
|
17867
|
+
reader.releaseLock();
|
|
17868
|
+
}
|
|
17869
|
+
}
|
|
17870
|
+
function selectAudioTrack(stream) {
|
|
17871
|
+
var _a25;
|
|
17872
|
+
const live = stream.getAudioTracks().filter((track) => track.readyState === "live");
|
|
17873
|
+
return (_a25 = live.find((track) => track.enabled && !track.muted)) != null ? _a25 : live[0];
|
|
17874
|
+
}
|
|
17875
|
+
var BrowserRealtimeWebRTC = class {
|
|
17876
|
+
constructor(options) {
|
|
17877
|
+
this.options = options;
|
|
17878
|
+
this.ownsStream = false;
|
|
17879
|
+
this.generation = 0;
|
|
17880
|
+
this.closingNotified = false;
|
|
17881
|
+
this.captureGeneration = 0;
|
|
17882
|
+
this.senderOperations = Promise.resolve();
|
|
17883
|
+
this.trackCleanups = [];
|
|
17884
|
+
}
|
|
17885
|
+
async connect({
|
|
17886
|
+
api,
|
|
17887
|
+
sessionConfig,
|
|
17888
|
+
stream,
|
|
17889
|
+
timeoutMs,
|
|
17890
|
+
capture = true
|
|
17891
|
+
}) {
|
|
17892
|
+
var _a25, _b25;
|
|
17893
|
+
const generation = this.generation + 1;
|
|
17894
|
+
this.dispose();
|
|
17895
|
+
const current = () => generation === this.generation;
|
|
17896
|
+
if (!current()) return;
|
|
17897
|
+
const config = (_b25 = (_a25 = this.options.model).getWebRTCConfig) == null ? void 0 : _b25.call(_a25);
|
|
17898
|
+
if (!current()) return;
|
|
17899
|
+
if (config == null)
|
|
17900
|
+
throw new Error("This model does not support WebRTC configuration");
|
|
17901
|
+
const abort = new AbortController();
|
|
17902
|
+
this.abort = abort;
|
|
17903
|
+
const timeout = setTimeout(
|
|
17904
|
+
() => abort.abort(new Error("Realtime startup timed out")),
|
|
17905
|
+
timeoutMs
|
|
17906
|
+
);
|
|
17907
|
+
const cancelled = new Promise((_, reject) => {
|
|
17908
|
+
abort.signal.addEventListener(
|
|
17909
|
+
"abort",
|
|
17910
|
+
() => reject(abort.signal.reason),
|
|
17911
|
+
{ once: true }
|
|
17912
|
+
);
|
|
17913
|
+
});
|
|
17914
|
+
const start = async () => {
|
|
17915
|
+
var _a26;
|
|
17916
|
+
const captureGeneration = this.captureGeneration;
|
|
17917
|
+
let media = capture ? stream != null ? stream : await navigator.mediaDevices.getUserMedia({ audio: true }) : void 0;
|
|
17918
|
+
if (!current() || abort.signal.aborted) {
|
|
17919
|
+
if (stream == null) media == null ? void 0 : media.getTracks().forEach((track2) => track2.stop());
|
|
17920
|
+
throw new Error("Realtime connection cancelled");
|
|
17921
|
+
}
|
|
17922
|
+
if (captureGeneration !== this.captureGeneration) {
|
|
17923
|
+
if (stream == null) media == null ? void 0 : media.getTracks().forEach((track2) => track2.stop());
|
|
17924
|
+
media = void 0;
|
|
17925
|
+
}
|
|
17926
|
+
this.stream = media;
|
|
17927
|
+
this.ownsStream = stream == null;
|
|
17928
|
+
const track = media == null ? void 0 : selectAudioTrack(media);
|
|
17929
|
+
if (media != null && track == null) {
|
|
17930
|
+
throw new Error("Realtime requires a live audio track");
|
|
17931
|
+
}
|
|
17932
|
+
const pc = new RTCPeerConnection();
|
|
17933
|
+
this.pc = pc;
|
|
17934
|
+
const audio = document.createElement("audio");
|
|
17935
|
+
audio.autoplay = true;
|
|
17936
|
+
this.audio = audio;
|
|
17937
|
+
audio.onplaying = () => {
|
|
17938
|
+
if (current()) this.options.onPlaying(true);
|
|
17939
|
+
};
|
|
17940
|
+
const stopped = () => {
|
|
17941
|
+
if (current()) this.options.onPlaying(false);
|
|
17942
|
+
};
|
|
17943
|
+
audio.onpause = stopped;
|
|
17944
|
+
audio.onended = stopped;
|
|
17945
|
+
audio.onwaiting = stopped;
|
|
17946
|
+
pc.ontrack = (event) => {
|
|
17947
|
+
var _a27;
|
|
17948
|
+
if (!current()) return;
|
|
17949
|
+
audio.srcObject = (_a27 = event.streams[0]) != null ? _a27 : new MediaStream([event.track]);
|
|
17950
|
+
void audio.play().catch((error) => {
|
|
17951
|
+
if (current()) this.options.onError(error);
|
|
17952
|
+
});
|
|
17953
|
+
};
|
|
17954
|
+
pc.onconnectionstatechange = () => {
|
|
17955
|
+
var _a27;
|
|
17956
|
+
if (!current()) return;
|
|
17957
|
+
if (pc.connectionState === "connected")
|
|
17958
|
+
clearTimeout(this.disconnectTimer);
|
|
17959
|
+
else if (pc.connectionState === "disconnected") {
|
|
17960
|
+
clearTimeout(this.disconnectTimer);
|
|
17961
|
+
this.disconnectTimer = setTimeout(() => {
|
|
17962
|
+
if (current() && pc.connectionState === "disconnected")
|
|
17963
|
+
this.fail(
|
|
17964
|
+
new Error(
|
|
17965
|
+
"Realtime peer disconnected beyond recovery grace period"
|
|
17966
|
+
)
|
|
17967
|
+
);
|
|
17968
|
+
}, (_a27 = this.options.disconnectTimeoutMs) != null ? _a27 : 5e3);
|
|
17969
|
+
} else if (pc.connectionState === "closed") {
|
|
17970
|
+
this.drainClose(new Error("Realtime peer connection closed"));
|
|
17971
|
+
} else if (pc.connectionState === "failed") {
|
|
17972
|
+
this.fail(
|
|
17973
|
+
new Error(`Realtime peer connection ${pc.connectionState}`)
|
|
17974
|
+
);
|
|
17975
|
+
}
|
|
17976
|
+
};
|
|
17977
|
+
pc.oniceconnectionstatechange = () => {
|
|
17978
|
+
if (current() && pc.iceConnectionState === "failed")
|
|
17979
|
+
this.fail(new Error("Realtime ICE connection failed"));
|
|
17980
|
+
};
|
|
17981
|
+
const dc = pc.createDataChannel(config.dataChannelLabel);
|
|
17982
|
+
this.dc = dc;
|
|
17983
|
+
const codec = new RealtimeEventChannel({
|
|
17984
|
+
model: this.options.model,
|
|
17985
|
+
onEvent: this.options.onEvent,
|
|
17986
|
+
onError: this.options.onError,
|
|
17987
|
+
onFatalError: (error) => {
|
|
17988
|
+
if (current()) this.fail(error);
|
|
17989
|
+
},
|
|
17990
|
+
send: (data) => {
|
|
17991
|
+
if (!current() || this.finishing || dc.readyState !== "open")
|
|
17992
|
+
throw new Error("Realtime data channel is not open");
|
|
17993
|
+
if (dc.bufferedAmount > 1024 * 1024)
|
|
17994
|
+
throw new Error("Realtime data channel buffer is full");
|
|
17995
|
+
const encoded = typeof data === "string" ? data : JSON.stringify(data);
|
|
17996
|
+
if (!current() || this.finishing || dc.readyState !== "open")
|
|
17997
|
+
throw new Error("Realtime data channel is not open");
|
|
17998
|
+
dc.send(encoded);
|
|
17999
|
+
}
|
|
18000
|
+
});
|
|
18001
|
+
if (!current()) {
|
|
18002
|
+
codec.dispose();
|
|
18003
|
+
return;
|
|
18004
|
+
}
|
|
18005
|
+
this.codec = codec;
|
|
18006
|
+
dc.onmessage = (event) => {
|
|
18007
|
+
var _a27;
|
|
18008
|
+
if (current()) (_a27 = this.codec) == null ? void 0 : _a27.receive(event.data);
|
|
18009
|
+
};
|
|
18010
|
+
dc.onclose = () => {
|
|
18011
|
+
if (current())
|
|
18012
|
+
this.drainClose(new Error("Realtime data channel closed"));
|
|
18013
|
+
};
|
|
18014
|
+
dc.onerror = () => {
|
|
18015
|
+
if (current()) this.fail(new Error("Realtime data channel error"));
|
|
18016
|
+
};
|
|
18017
|
+
const opened = new Promise((resolve3, reject) => {
|
|
18018
|
+
this.rejectStartup = reject;
|
|
18019
|
+
dc.onopen = () => {
|
|
18020
|
+
this.rejectStartup = void 0;
|
|
18021
|
+
resolve3();
|
|
18022
|
+
};
|
|
18023
|
+
});
|
|
18024
|
+
void opened.catch(() => {
|
|
18025
|
+
});
|
|
18026
|
+
const gathered = new Promise((resolve3) => {
|
|
18027
|
+
pc.onicegatheringstatechange = () => {
|
|
18028
|
+
if (pc.iceGatheringState === "complete") resolve3();
|
|
18029
|
+
};
|
|
18030
|
+
});
|
|
18031
|
+
this.sender = media != null && track != null ? pc.addTrack(track, media) : pc.addTransceiver("audio", { direction: "sendrecv" }).sender;
|
|
18032
|
+
this.observeCapture();
|
|
18033
|
+
if (!current() || abort.signal.aborted) return;
|
|
18034
|
+
const offer = await pc.createOffer();
|
|
18035
|
+
if (!current() || abort.signal.aborted) return;
|
|
18036
|
+
await pc.setLocalDescription(offer);
|
|
18037
|
+
if (pc.iceGatheringState !== "complete")
|
|
18038
|
+
await Promise.race([gathered, cancelled]);
|
|
18039
|
+
if (!current() || abort.signal.aborted) return;
|
|
18040
|
+
const sdp = (_a26 = pc.localDescription) == null ? void 0 : _a26.sdp;
|
|
18041
|
+
if (sdp == null) throw new Error("Realtime offer has no SDP");
|
|
18042
|
+
const response = await fetch(api, {
|
|
18043
|
+
method: "POST",
|
|
18044
|
+
headers: { "Content-Type": "application/json" },
|
|
18045
|
+
body: JSON.stringify({ sdp, sessionConfig }),
|
|
18046
|
+
signal: abort.signal
|
|
18047
|
+
});
|
|
18048
|
+
if (!response.ok)
|
|
18049
|
+
throw new Error(
|
|
18050
|
+
`Failed to create realtime session: ${response.status}`
|
|
18051
|
+
);
|
|
18052
|
+
const parsed = await safeParseJSON6({
|
|
18053
|
+
text: await readSessionAnswer(response)
|
|
18054
|
+
});
|
|
18055
|
+
if (!current() || abort.signal.aborted) return;
|
|
18056
|
+
if (!parsed.success || typeof parsed.value !== "object" || parsed.value == null || !("sdp" in parsed.value) || typeof parsed.value.sdp !== "string" || parsed.value.sdp.trim().length === 0 || !("sessionId" in parsed.value) || typeof parsed.value.sessionId !== "string" || parsed.value.sessionId.trim().length === 0) {
|
|
18057
|
+
throw new Error("Invalid realtime session answer");
|
|
18058
|
+
}
|
|
18059
|
+
await pc.setRemoteDescription({ type: "answer", sdp: parsed.value.sdp });
|
|
18060
|
+
await Promise.race([opened, cancelled]);
|
|
18061
|
+
};
|
|
18062
|
+
try {
|
|
18063
|
+
await Promise.race([start(), cancelled]);
|
|
18064
|
+
} catch (error) {
|
|
18065
|
+
if (current()) this.dispose();
|
|
18066
|
+
throw error;
|
|
18067
|
+
} finally {
|
|
18068
|
+
clearTimeout(timeout);
|
|
18069
|
+
this.rejectStartup = void 0;
|
|
18070
|
+
}
|
|
18071
|
+
}
|
|
18072
|
+
sendEvent(event, shouldSend) {
|
|
18073
|
+
var _a25, _b25, _c;
|
|
18074
|
+
if (this.finishing || ((_a25 = this.dc) == null ? void 0 : _a25.readyState) !== "open")
|
|
18075
|
+
throw new Error("Realtime data channel is not open");
|
|
18076
|
+
return (_c = (_b25 = this.codec) == null ? void 0 : _b25.send(event, shouldSend)) != null ? _c : Promise.resolve();
|
|
18077
|
+
}
|
|
18078
|
+
async resumePlayback() {
|
|
18079
|
+
if (this.audio == null)
|
|
18080
|
+
throw new Error("Realtime playback is not connected");
|
|
18081
|
+
await this.audio.play();
|
|
18082
|
+
}
|
|
18083
|
+
stopPlayback() {
|
|
18084
|
+
var _a25;
|
|
18085
|
+
(_a25 = this.audio) == null ? void 0 : _a25.pause();
|
|
18086
|
+
}
|
|
18087
|
+
finish() {
|
|
18088
|
+
var _a25, _b25;
|
|
18089
|
+
return (_b25 = (_a25 = this.codec) == null ? void 0 : _a25.finish()) != null ? _b25 : Promise.resolve();
|
|
18090
|
+
}
|
|
18091
|
+
fail(error) {
|
|
18092
|
+
var _a25;
|
|
18093
|
+
if (this.finishing === "failure") return;
|
|
18094
|
+
this.finishing = "failure";
|
|
18095
|
+
clearTimeout(this.finishTimer);
|
|
18096
|
+
if (this.rejectStartup != null) {
|
|
18097
|
+
this.rejectStartup(error);
|
|
18098
|
+
(_a25 = this.abort) == null ? void 0 : _a25.abort(error);
|
|
18099
|
+
return;
|
|
18100
|
+
}
|
|
18101
|
+
const generation = this.generation;
|
|
18102
|
+
const drain = this.finish();
|
|
18103
|
+
this.notifyClosing();
|
|
18104
|
+
if (generation !== this.generation) return;
|
|
18105
|
+
void this.stopCaptureForShutdown(generation);
|
|
18106
|
+
if (generation !== this.generation) return;
|
|
18107
|
+
if (this.options.onFatalError != null) {
|
|
18108
|
+
this.options.onFatalError(error, drain);
|
|
18109
|
+
return;
|
|
18110
|
+
}
|
|
18111
|
+
const complete = () => {
|
|
18112
|
+
if (generation !== this.generation) return;
|
|
18113
|
+
this.dispose();
|
|
18114
|
+
try {
|
|
18115
|
+
this.options.onClose();
|
|
18116
|
+
} catch (cause) {
|
|
18117
|
+
this.reportCallbackError(cause);
|
|
18118
|
+
}
|
|
18119
|
+
};
|
|
18120
|
+
this.finishTimer = setTimeout(complete, 1e3);
|
|
18121
|
+
void this.awaitDrain(drain, complete);
|
|
18122
|
+
try {
|
|
18123
|
+
this.options.onError(error);
|
|
18124
|
+
} catch (e) {
|
|
18125
|
+
}
|
|
18126
|
+
}
|
|
18127
|
+
drainClose(error) {
|
|
18128
|
+
if (this.finishing) return;
|
|
18129
|
+
if (this.rejectStartup != null) {
|
|
18130
|
+
this.fail(error);
|
|
18131
|
+
return;
|
|
18132
|
+
}
|
|
18133
|
+
this.finishing = "close";
|
|
18134
|
+
const generation = this.generation;
|
|
18135
|
+
const complete = () => {
|
|
18136
|
+
if (generation !== this.generation || this.finishing !== "close") return;
|
|
18137
|
+
clearTimeout(this.finishTimer);
|
|
18138
|
+
try {
|
|
18139
|
+
this.options.onClose(error);
|
|
18140
|
+
} catch (cause) {
|
|
18141
|
+
this.reportCallbackError(cause);
|
|
18142
|
+
} finally {
|
|
18143
|
+
if (generation === this.generation) this.dispose();
|
|
18144
|
+
}
|
|
18145
|
+
};
|
|
18146
|
+
this.finishTimer = setTimeout(complete, 1e3);
|
|
18147
|
+
void this.awaitDrain(this.finish(), complete);
|
|
18148
|
+
this.notifyClosing();
|
|
18149
|
+
if (generation !== this.generation) return;
|
|
18150
|
+
void this.stopCaptureForShutdown(generation);
|
|
18151
|
+
}
|
|
18152
|
+
notifyClosing() {
|
|
18153
|
+
var _a25, _b25;
|
|
18154
|
+
if (this.closingNotified) return;
|
|
18155
|
+
this.closingNotified = true;
|
|
18156
|
+
try {
|
|
18157
|
+
(_b25 = (_a25 = this.options).onClosing) == null ? void 0 : _b25.call(_a25);
|
|
18158
|
+
} catch (error) {
|
|
18159
|
+
this.reportCallbackError(error);
|
|
18160
|
+
}
|
|
18161
|
+
}
|
|
18162
|
+
async stopCaptureForShutdown(generation) {
|
|
18163
|
+
try {
|
|
18164
|
+
await this.stopCapture();
|
|
18165
|
+
} catch (cause) {
|
|
18166
|
+
if (generation !== this.generation || this.finishing !== "close") return;
|
|
18167
|
+
try {
|
|
18168
|
+
this.fail(cause instanceof Error ? cause : new Error(String(cause)));
|
|
18169
|
+
} catch (error) {
|
|
18170
|
+
this.reportCallbackError(error);
|
|
18171
|
+
}
|
|
18172
|
+
}
|
|
18173
|
+
}
|
|
18174
|
+
async awaitDrain(drain, complete) {
|
|
18175
|
+
try {
|
|
18176
|
+
await drain;
|
|
18177
|
+
} catch (e) {
|
|
18178
|
+
}
|
|
18179
|
+
try {
|
|
18180
|
+
complete();
|
|
18181
|
+
} catch (error) {
|
|
18182
|
+
this.reportCallbackError(error);
|
|
18183
|
+
}
|
|
18184
|
+
}
|
|
18185
|
+
reportCallbackError(error) {
|
|
18186
|
+
try {
|
|
18187
|
+
this.options.onError(
|
|
18188
|
+
error instanceof Error ? error : new Error(String(error))
|
|
18189
|
+
);
|
|
18190
|
+
} catch (e) {
|
|
18191
|
+
}
|
|
18192
|
+
}
|
|
18193
|
+
async stopCapture() {
|
|
18194
|
+
var _a25;
|
|
18195
|
+
const generation = ++this.captureGeneration;
|
|
18196
|
+
for (const cleanup of this.trackCleanups) cleanup();
|
|
18197
|
+
this.trackCleanups = [];
|
|
18198
|
+
if (this.ownsStream)
|
|
18199
|
+
(_a25 = this.stream) == null ? void 0 : _a25.getTracks().forEach((track) => track.stop());
|
|
18200
|
+
this.stream = void 0;
|
|
18201
|
+
const pc = this.pc;
|
|
18202
|
+
const sender = this.sender;
|
|
18203
|
+
await this.queueSenderOperation(async () => {
|
|
18204
|
+
if (pc !== this.pc || sender !== this.sender) return;
|
|
18205
|
+
if (pc != null && sender != null) await this.detachSender(pc, sender);
|
|
18206
|
+
if (generation === this.captureGeneration && pc === this.pc)
|
|
18207
|
+
this.options.onCapturing(false);
|
|
18208
|
+
});
|
|
18209
|
+
}
|
|
18210
|
+
async startCapture(supplied) {
|
|
18211
|
+
const generation = this.captureGeneration + 1;
|
|
18212
|
+
await this.stopCapture();
|
|
18213
|
+
if (generation !== this.captureGeneration || this.pc == null || this.finishing)
|
|
18214
|
+
return;
|
|
18215
|
+
const media = supplied != null ? supplied : await navigator.mediaDevices.getUserMedia({ audio: true });
|
|
18216
|
+
if (generation !== this.captureGeneration || this.pc == null || this.finishing) {
|
|
18217
|
+
if (supplied == null) media.getTracks().forEach((track2) => track2.stop());
|
|
18218
|
+
return;
|
|
18219
|
+
}
|
|
18220
|
+
const track = selectAudioTrack(media);
|
|
18221
|
+
if (track == null) {
|
|
18222
|
+
if (supplied == null) media.getTracks().forEach((track2) => track2.stop());
|
|
18223
|
+
throw new Error("Realtime requires a live audio track");
|
|
18224
|
+
}
|
|
18225
|
+
const pc = this.pc;
|
|
18226
|
+
const sender = this.sender;
|
|
18227
|
+
await this.queueSenderOperation(async () => {
|
|
18228
|
+
const current = () => generation === this.captureGeneration && pc === this.pc && sender === this.sender && pc.connectionState !== "closed" && !this.finishing;
|
|
18229
|
+
let attached = false;
|
|
18230
|
+
try {
|
|
18231
|
+
if (!current() || sender == null) return;
|
|
18232
|
+
await sender.replaceTrack(track);
|
|
18233
|
+
if (!current()) {
|
|
18234
|
+
if (pc === this.pc) await this.detachSender(pc, sender);
|
|
18235
|
+
return;
|
|
18236
|
+
}
|
|
18237
|
+
this.stream = media;
|
|
18238
|
+
this.ownsStream = supplied == null;
|
|
18239
|
+
attached = true;
|
|
18240
|
+
this.observeCapture();
|
|
18241
|
+
} finally {
|
|
18242
|
+
if (!attached && supplied == null)
|
|
18243
|
+
media.getTracks().forEach((track2) => track2.stop());
|
|
18244
|
+
}
|
|
18245
|
+
});
|
|
18246
|
+
}
|
|
18247
|
+
queueSenderOperation(operation) {
|
|
18248
|
+
const pending = this.senderOperations.then(operation);
|
|
18249
|
+
this.senderOperations = pending.catch(() => {
|
|
18250
|
+
});
|
|
18251
|
+
return pending;
|
|
18252
|
+
}
|
|
18253
|
+
async detachSender(pc, sender) {
|
|
18254
|
+
if (pc.connectionState === "closed") return;
|
|
18255
|
+
try {
|
|
18256
|
+
await sender.replaceTrack(null);
|
|
18257
|
+
} catch (error) {
|
|
18258
|
+
if (pc !== this.pc) return;
|
|
18259
|
+
this.closePeer(pc);
|
|
18260
|
+
this.fail(error instanceof Error ? error : new Error(String(error)));
|
|
18261
|
+
}
|
|
18262
|
+
}
|
|
18263
|
+
closePeer(pc) {
|
|
18264
|
+
pc.ontrack = null;
|
|
18265
|
+
pc.onconnectionstatechange = null;
|
|
18266
|
+
pc.oniceconnectionstatechange = null;
|
|
18267
|
+
pc.onicegatheringstatechange = null;
|
|
18268
|
+
if (pc.connectionState !== "closed") pc.close();
|
|
18269
|
+
}
|
|
18270
|
+
observeCapture() {
|
|
18271
|
+
const generation = this.captureGeneration;
|
|
18272
|
+
const sender = this.sender;
|
|
18273
|
+
const track = sender == null ? void 0 : sender.track;
|
|
18274
|
+
const update = () => {
|
|
18275
|
+
if (generation === this.captureGeneration && this.sender === sender)
|
|
18276
|
+
this.options.onCapturing(
|
|
18277
|
+
(sender == null ? void 0 : sender.track) != null && sender.track.readyState === "live" && sender.track.enabled && !sender.track.muted
|
|
18278
|
+
);
|
|
18279
|
+
};
|
|
18280
|
+
if (track != null) {
|
|
18281
|
+
for (const event of ["ended", "mute", "unmute"]) {
|
|
18282
|
+
track.addEventListener(event, update);
|
|
18283
|
+
this.trackCleanups.push(() => track.removeEventListener(event, update));
|
|
18284
|
+
}
|
|
18285
|
+
}
|
|
18286
|
+
update();
|
|
18287
|
+
}
|
|
18288
|
+
dispose() {
|
|
18289
|
+
var _a25, _b25, _c;
|
|
18290
|
+
this.generation++;
|
|
18291
|
+
this.captureGeneration++;
|
|
18292
|
+
clearTimeout(this.finishTimer);
|
|
18293
|
+
clearTimeout(this.disconnectTimer);
|
|
18294
|
+
this.finishing = void 0;
|
|
18295
|
+
this.closingNotified = false;
|
|
18296
|
+
this.senderOperations = Promise.resolve();
|
|
18297
|
+
(_a25 = this.abort) == null ? void 0 : _a25.abort(new Error("Realtime connection cancelled"));
|
|
18298
|
+
this.abort = void 0;
|
|
18299
|
+
(_b25 = this.codec) == null ? void 0 : _b25.dispose();
|
|
18300
|
+
this.codec = void 0;
|
|
18301
|
+
if (this.dc != null) {
|
|
18302
|
+
this.dc.onopen = null;
|
|
18303
|
+
this.dc.onclose = null;
|
|
18304
|
+
this.dc.onerror = null;
|
|
18305
|
+
this.dc.onmessage = null;
|
|
18306
|
+
this.dc.close();
|
|
18307
|
+
this.dc = void 0;
|
|
18308
|
+
}
|
|
18309
|
+
if (this.pc != null) {
|
|
18310
|
+
this.closePeer(this.pc);
|
|
18311
|
+
this.pc = void 0;
|
|
18312
|
+
}
|
|
18313
|
+
for (const cleanup of this.trackCleanups) cleanup();
|
|
18314
|
+
this.trackCleanups = [];
|
|
18315
|
+
if (this.ownsStream)
|
|
18316
|
+
(_c = this.stream) == null ? void 0 : _c.getTracks().forEach((track) => track.stop());
|
|
18317
|
+
this.stream = void 0;
|
|
18318
|
+
this.sender = void 0;
|
|
18319
|
+
if (this.audio != null) {
|
|
18320
|
+
this.audio.onplaying = null;
|
|
18321
|
+
this.audio.onpause = null;
|
|
18322
|
+
this.audio.onended = null;
|
|
18323
|
+
this.audio.onwaiting = null;
|
|
18324
|
+
this.audio.pause();
|
|
18325
|
+
this.audio.srcObject = null;
|
|
18326
|
+
this.audio = void 0;
|
|
18327
|
+
}
|
|
18328
|
+
this.options.onCapturing(false);
|
|
18329
|
+
this.options.onPlaying(false);
|
|
18330
|
+
}
|
|
18331
|
+
};
|
|
18332
|
+
|
|
18333
|
+
// src/realtime/realtime-attempt.ts
|
|
18334
|
+
var RealtimeAttempt = class {
|
|
18335
|
+
constructor() {
|
|
18336
|
+
this.abort = new AbortController();
|
|
18337
|
+
this.timers = /* @__PURE__ */ new Map();
|
|
18338
|
+
this.active = true;
|
|
18339
|
+
this.ready = false;
|
|
18340
|
+
this.closing = false;
|
|
18341
|
+
this.transportClosing = false;
|
|
18342
|
+
}
|
|
18343
|
+
timer(name25, duration, callback) {
|
|
18344
|
+
this.clearTimer(name25);
|
|
18345
|
+
this.timers.set(
|
|
18346
|
+
name25,
|
|
18347
|
+
setTimeout(() => {
|
|
18348
|
+
this.timers.delete(name25);
|
|
18349
|
+
if (this.active) callback();
|
|
18350
|
+
}, duration)
|
|
18351
|
+
);
|
|
18352
|
+
}
|
|
18353
|
+
clearTimer(name25) {
|
|
18354
|
+
clearTimeout(this.timers.get(name25));
|
|
18355
|
+
this.timers.delete(name25);
|
|
18356
|
+
}
|
|
18357
|
+
beginClose() {
|
|
18358
|
+
var _a25;
|
|
18359
|
+
this.closing = true;
|
|
18360
|
+
return (_a25 = this.closePromise) != null ? _a25 : this.closePromise = new Promise((resolve3) => {
|
|
18361
|
+
this.settleClose = resolve3;
|
|
18362
|
+
});
|
|
18363
|
+
}
|
|
18364
|
+
retire() {
|
|
18365
|
+
var _a25;
|
|
18366
|
+
if (!this.active) return;
|
|
18367
|
+
this.active = false;
|
|
18368
|
+
this.abort.abort();
|
|
18369
|
+
for (const timer of this.timers.values()) clearTimeout(timer);
|
|
18370
|
+
this.timers.clear();
|
|
18371
|
+
(_a25 = this.settleClose) == null ? void 0 : _a25.call(this);
|
|
18372
|
+
this.settleClose = void 0;
|
|
18373
|
+
}
|
|
18374
|
+
};
|
|
18375
|
+
|
|
18376
|
+
// src/realtime/realtime-command-tracker.ts
|
|
18377
|
+
import { generateId as generateId3 } from "@ai-sdk/provider-utils";
|
|
18378
|
+
var RealtimeCommandTracker = class {
|
|
18379
|
+
constructor(sendCommand) {
|
|
18380
|
+
this.sendCommand = sendCommand;
|
|
18381
|
+
this.pending = /* @__PURE__ */ new Map();
|
|
18382
|
+
this.recent = /* @__PURE__ */ new Set();
|
|
18383
|
+
}
|
|
18384
|
+
validateId(id) {
|
|
18385
|
+
if (this.pending.has(id) || this.recent.has(id))
|
|
18386
|
+
throw new Error(
|
|
18387
|
+
"Realtime command ID was already used; retry with a fresh eventId"
|
|
18388
|
+
);
|
|
18389
|
+
if (this.pending.size >= 512)
|
|
18390
|
+
throw new Error("Too many pending realtime commands");
|
|
18391
|
+
}
|
|
18392
|
+
complete(id) {
|
|
18393
|
+
const command = this.pending.get(id);
|
|
18394
|
+
if (command == null) return;
|
|
18395
|
+
this.pending.delete(id);
|
|
18396
|
+
this.recent.add(id);
|
|
18397
|
+
if (this.recent.size > 4096) {
|
|
18398
|
+
const oldest = this.recent.values().next().value;
|
|
18399
|
+
if (oldest != null) this.recent.delete(oldest);
|
|
18400
|
+
}
|
|
18401
|
+
return command;
|
|
18402
|
+
}
|
|
18403
|
+
send(event) {
|
|
18404
|
+
var _a25;
|
|
18405
|
+
const muted = event.type === "input-audio-mute" ? true : event.type === "input-audio-unmute" ? false : void 0;
|
|
18406
|
+
if (event.type === "input-audio-mute" || event.type === "input-audio-unmute")
|
|
18407
|
+
event = { ...event, eventId: (_a25 = event.eventId) != null ? _a25 : generateId3() };
|
|
18408
|
+
const id = "eventId" in event ? event.eventId : void 0;
|
|
18409
|
+
if (id != null) {
|
|
18410
|
+
this.validateId(id);
|
|
18411
|
+
this.pending.set(id, { muted });
|
|
18412
|
+
}
|
|
18413
|
+
const awaitsAcknowledgement = event.type === "context-append" || event.type === "session-update" || muted != null;
|
|
18414
|
+
try {
|
|
18415
|
+
const sent = this.sendCommand(event);
|
|
18416
|
+
void sent.then(
|
|
18417
|
+
() => {
|
|
18418
|
+
if (id != null && !awaitsAcknowledgement) this.complete(id);
|
|
18419
|
+
},
|
|
18420
|
+
() => {
|
|
18421
|
+
if (id != null) this.complete(id);
|
|
18422
|
+
}
|
|
18423
|
+
);
|
|
18424
|
+
return sent;
|
|
18425
|
+
} catch (error) {
|
|
18426
|
+
if (id != null) this.complete(id);
|
|
18427
|
+
throw error;
|
|
18428
|
+
}
|
|
18429
|
+
}
|
|
18430
|
+
receive(event) {
|
|
18431
|
+
if (event.type === "command-acknowledged" && event.clientEventId != null)
|
|
18432
|
+
return this.complete(event.clientEventId);
|
|
18433
|
+
if (event.type === "error" && event.clientEventId != null)
|
|
18434
|
+
this.complete(event.clientEventId);
|
|
18435
|
+
}
|
|
18436
|
+
};
|
|
18437
|
+
|
|
18438
|
+
// src/realtime/validate-realtime-setup.ts
|
|
18439
|
+
var setupSchema = z.object({
|
|
18440
|
+
token: z.string().refine((value) => value.trim().length > 0),
|
|
18441
|
+
url: z.string().refine((value) => {
|
|
18442
|
+
try {
|
|
18443
|
+
const url = new URL(value);
|
|
18444
|
+
return (url.protocol === "ws:" || url.protocol === "wss:") && url.hostname !== "";
|
|
18445
|
+
} catch (e) {
|
|
18446
|
+
return false;
|
|
18447
|
+
}
|
|
18448
|
+
}),
|
|
18449
|
+
expiresAt: z.number().positive().max(Number.MAX_SAFE_INTEGER).optional(),
|
|
18450
|
+
tools: z.array(
|
|
18451
|
+
z.object({
|
|
18452
|
+
type: z.literal("function"),
|
|
18453
|
+
name: z.string().min(1),
|
|
18454
|
+
description: z.string().optional(),
|
|
18455
|
+
parameters: z.record(z.string(), z.unknown())
|
|
18456
|
+
})
|
|
18457
|
+
).optional()
|
|
18458
|
+
});
|
|
18459
|
+
function validateRealtimeSetup(payload) {
|
|
18460
|
+
const result = setupSchema.safeParse(payload);
|
|
18461
|
+
if (!result.success)
|
|
18462
|
+
throw new Error(
|
|
18463
|
+
"Invalid realtime setup: expected a nonempty token, a ws(s) URL, and valid optional expiresAt/tools"
|
|
18464
|
+
);
|
|
18465
|
+
return result.data;
|
|
18466
|
+
}
|
|
18467
|
+
|
|
18468
|
+
// src/realtime/realtime-session-state.ts
|
|
18469
|
+
function createSessionState() {
|
|
18470
|
+
return {
|
|
18471
|
+
transcripts: [],
|
|
18472
|
+
delegations: [],
|
|
18473
|
+
isInputMuted: false,
|
|
18474
|
+
finalization: "pending"
|
|
18475
|
+
};
|
|
18476
|
+
}
|
|
18477
|
+
function reduceSessionState(state, event, limit, muted) {
|
|
18478
|
+
var _a25;
|
|
18479
|
+
switch (event.type) {
|
|
18480
|
+
case "session-started":
|
|
18481
|
+
return {
|
|
18482
|
+
...state,
|
|
18483
|
+
sessionId: event.sessionId,
|
|
18484
|
+
delegationMode: event.delegationMode
|
|
18485
|
+
};
|
|
18486
|
+
case "session-usage":
|
|
18487
|
+
return { ...state, usage: event.usage };
|
|
18488
|
+
case "session-closed":
|
|
18489
|
+
return {
|
|
18490
|
+
...state,
|
|
18491
|
+
sessionId: (_a25 = event.sessionId) != null ? _a25 : state.sessionId,
|
|
18492
|
+
usage: event.usage,
|
|
18493
|
+
terminationReason: event.reason,
|
|
18494
|
+
finalization: "confirmed"
|
|
18495
|
+
};
|
|
18496
|
+
case "transcript-fragment":
|
|
18497
|
+
return {
|
|
18498
|
+
...state,
|
|
18499
|
+
transcripts: [...state.transcripts, event].slice(-limit)
|
|
18500
|
+
};
|
|
18501
|
+
case "delegation-created":
|
|
18502
|
+
return state.delegations.some(
|
|
18503
|
+
(item) => item.delegationId === event.delegationId
|
|
18504
|
+
) ? state : {
|
|
18505
|
+
...state,
|
|
18506
|
+
delegations: [...state.delegations, event].slice(-limit)
|
|
18507
|
+
};
|
|
18508
|
+
case "command-acknowledged":
|
|
18509
|
+
return muted == null ? state : { ...state, isInputMuted: muted };
|
|
18510
|
+
default:
|
|
18511
|
+
return state;
|
|
18512
|
+
}
|
|
18513
|
+
}
|
|
18514
|
+
|
|
18515
|
+
// src/realtime/realtime-event-reducer.ts
|
|
18516
|
+
import { safeParseJSON as safeParseJSON7 } from "@ai-sdk/provider-utils";
|
|
18517
|
+
function createInitialRealtimeState() {
|
|
18518
|
+
return {
|
|
18519
|
+
status: "disconnected",
|
|
18520
|
+
messages: [],
|
|
18521
|
+
events: [],
|
|
18522
|
+
isCapturing: false,
|
|
18523
|
+
isPlaying: false
|
|
18524
|
+
};
|
|
18525
|
+
}
|
|
18526
|
+
var RealtimeEventReducer = class {
|
|
18527
|
+
constructor(maxEvents = 500) {
|
|
18528
|
+
this.maxEvents = maxEvents;
|
|
18529
|
+
this.currentAssistantMessageId = null;
|
|
18530
|
+
this.textAccumulators = /* @__PURE__ */ new Map();
|
|
18531
|
+
this.toolArgAccumulators = /* @__PURE__ */ new Map();
|
|
18532
|
+
this.toolCallIdToMessageId = /* @__PURE__ */ new Map();
|
|
18533
|
+
this.toolCallIdToName = /* @__PURE__ */ new Map();
|
|
18534
|
+
this.inputAudioMessageInsertIndex = /* @__PURE__ */ new Map();
|
|
18535
|
+
this.itemIdToPartLocation = /* @__PURE__ */ new Map();
|
|
18536
|
+
}
|
|
18537
|
+
setStatus(state, status) {
|
|
18538
|
+
return { ...state, status };
|
|
18539
|
+
}
|
|
18540
|
+
setCapturing(state, isCapturing) {
|
|
18541
|
+
return { ...state, isCapturing };
|
|
18542
|
+
}
|
|
18543
|
+
setPlaying(state, isPlaying) {
|
|
18544
|
+
return { ...state, isPlaying };
|
|
18545
|
+
}
|
|
18546
|
+
addUserTextMessage(state, text2) {
|
|
18547
|
+
return {
|
|
18548
|
+
...state,
|
|
18549
|
+
messages: [
|
|
18550
|
+
...state.messages,
|
|
18551
|
+
{
|
|
18552
|
+
id: `user-${Date.now()}`,
|
|
18553
|
+
role: "user",
|
|
18554
|
+
parts: [{ type: "text", text: text2, state: "done" }]
|
|
18555
|
+
}
|
|
18556
|
+
]
|
|
18557
|
+
};
|
|
18558
|
+
}
|
|
18559
|
+
addToolOutput(state, callId, result) {
|
|
18560
|
+
return {
|
|
18561
|
+
state: this.updateToolPartState(state, callId, result),
|
|
18562
|
+
output: {
|
|
18563
|
+
callId,
|
|
18564
|
+
name: this.toolCallIdToName.get(callId),
|
|
18565
|
+
output: JSON.stringify(result)
|
|
18566
|
+
}
|
|
18567
|
+
};
|
|
18568
|
+
}
|
|
18569
|
+
async reduceServerEvent(state, event) {
|
|
18570
|
+
var _a25;
|
|
18571
|
+
let nextState = this.pushEvent(state, event);
|
|
18572
|
+
const effects = [];
|
|
18573
|
+
switch (event.type) {
|
|
18574
|
+
case "session-created":
|
|
18575
|
+
case "session-updated": {
|
|
18576
|
+
if (nextState.status === "connecting") {
|
|
18577
|
+
nextState = this.setStatus(nextState, "connected");
|
|
18578
|
+
}
|
|
18579
|
+
break;
|
|
18580
|
+
}
|
|
18581
|
+
case "audio-delta": {
|
|
17316
18582
|
effects.push({
|
|
17317
18583
|
type: "play-audio",
|
|
17318
18584
|
itemId: event.itemId,
|
|
@@ -17376,7 +18642,7 @@ var RealtimeEventReducer = class {
|
|
|
17376
18642
|
case "function-call-arguments-done": {
|
|
17377
18643
|
this.toolArgAccumulators.delete(event.callId);
|
|
17378
18644
|
this.toolCallIdToName.set(event.callId, event.name);
|
|
17379
|
-
const parseResult = await
|
|
18645
|
+
const parseResult = await safeParseJSON7({ text: event.arguments });
|
|
17380
18646
|
const parsedInput = parseResult.success ? parseResult.value : {};
|
|
17381
18647
|
const messageId = this.toolCallIdToMessageId.get(event.callId);
|
|
17382
18648
|
if (messageId != null) {
|
|
@@ -17616,98 +18882,375 @@ var RealtimeEventReducer = class {
|
|
|
17616
18882
|
// src/realtime/realtime-session.ts
|
|
17617
18883
|
var AbstractRealtimeSession = class {
|
|
17618
18884
|
constructor(options) {
|
|
18885
|
+
this.options = options;
|
|
17619
18886
|
this.state = createInitialRealtimeState();
|
|
18887
|
+
this.publication = 0;
|
|
18888
|
+
this.captureGeneration = 0;
|
|
18889
|
+
this.captureRequested = false;
|
|
17620
18890
|
this.currentResponseItemId = null;
|
|
17621
|
-
// Tool calls requested by the current (tool-bearing) response, the outputs
|
|
17622
|
-
// that have been submitted for them, and whether that response has finished
|
|
17623
|
-
// delivering its tool calls. Used to request a single response only once
|
|
17624
|
-
// every tool output for the turn has been submitted.
|
|
17625
18891
|
this.toolCallsInResponse = /* @__PURE__ */ new Set();
|
|
17626
18892
|
this.submittedToolOutputs = /* @__PURE__ */ new Set();
|
|
17627
18893
|
this.responseToolCallsClosed = false;
|
|
17628
|
-
var _a25, _b25, _c, _d, _e
|
|
17629
|
-
|
|
17630
|
-
this.
|
|
17631
|
-
this.
|
|
18894
|
+
var _a25, _b25, _c, _d, _e;
|
|
18895
|
+
const capabilities = options.model.capabilities;
|
|
18896
|
+
this.sessionLifecycle = (capabilities == null ? void 0 : capabilities.startup) === "session-start" || (capabilities == null ? void 0 : capabilities.finalization) === "session-close";
|
|
18897
|
+
this.continuous = (capabilities == null ? void 0 : capabilities.conversation) === "continuous";
|
|
17632
18898
|
this.maxEvents = (_a25 = options.maxEvents) != null ? _a25 : 500;
|
|
18899
|
+
if (!Number.isSafeInteger(this.maxEvents) || this.maxEvents < 1)
|
|
18900
|
+
throw new Error("maxEvents must be a positive integer");
|
|
18901
|
+
for (const timeout of [
|
|
18902
|
+
(_b25 = options.startupTimeoutMs) != null ? _b25 : 3e4,
|
|
18903
|
+
(_c = options.closeTimeoutMs) != null ? _c : 15e3,
|
|
18904
|
+
(_d = options.rtcDisconnectTimeoutMs) != null ? _d : 5e3
|
|
18905
|
+
]) {
|
|
18906
|
+
if (!Number.isFinite(timeout) || timeout <= 0 || timeout > 2147483647)
|
|
18907
|
+
throw new Error(
|
|
18908
|
+
"Realtime timeouts must be positive finite timer durations"
|
|
18909
|
+
);
|
|
18910
|
+
}
|
|
18911
|
+
const budget = (_e = options.maxPlaybackBufferSeconds) != null ? _e : 2;
|
|
18912
|
+
if (!Number.isFinite(budget) || budget <= 0)
|
|
18913
|
+
throw new Error("maxPlaybackBufferSeconds must be positive and finite");
|
|
17633
18914
|
this.reducer = new RealtimeEventReducer(this.maxEvents);
|
|
17634
18915
|
this.onToolCall = options.onToolCall;
|
|
17635
18916
|
this.onEvent = options.onEvent;
|
|
17636
18917
|
this.onError = options.onError;
|
|
17637
|
-
const sampleRate = (_b25 = options.sampleRate) != null ? _b25 : 24e3;
|
|
17638
|
-
const captureSampleRate = (_e = (_d = (_c = options.sessionConfig) == null ? void 0 : _c.inputAudioFormat) == null ? void 0 : _d.rate) != null ? _e : sampleRate;
|
|
17639
|
-
const playbackSampleRate = (_h = (_g = (_f = options.sessionConfig) == null ? void 0 : _f.outputAudioFormat) == null ? void 0 : _g.rate) != null ? _h : sampleRate;
|
|
17640
|
-
this.transport = new BrowserRealtimeTransport({
|
|
17641
|
-
model: this.model,
|
|
17642
|
-
onServerEvent: (event) => this.handleServerEvent(event),
|
|
17643
|
-
onError: (error) => {
|
|
17644
|
-
var _a26;
|
|
17645
|
-
this.applyState(this.reducer.setStatus(this.state, "error"));
|
|
17646
|
-
(_a26 = this.onError) == null ? void 0 : _a26.call(this, error);
|
|
17647
|
-
},
|
|
17648
|
-
onClose: () => {
|
|
17649
|
-
this.applyState(this.reducer.setStatus(this.state, "disconnected"));
|
|
17650
|
-
}
|
|
17651
|
-
});
|
|
17652
|
-
this.audio = new BrowserRealtimeAudio({
|
|
17653
|
-
captureSampleRate,
|
|
17654
|
-
playbackSampleRate,
|
|
17655
|
-
onAudio: (audio) => this.sendAudio(audio),
|
|
17656
|
-
onCapturingChange: (isCapturing) => {
|
|
17657
|
-
this.applyState(this.reducer.setCapturing(this.state, isCapturing));
|
|
17658
|
-
},
|
|
17659
|
-
onPlayingChange: (isPlaying) => {
|
|
17660
|
-
this.applyState(this.reducer.setPlaying(this.state, isPlaying));
|
|
17661
|
-
}
|
|
17662
|
-
});
|
|
17663
18918
|
}
|
|
17664
|
-
|
|
17665
|
-
|
|
17666
|
-
|
|
17667
|
-
|
|
18919
|
+
validateConnection() {
|
|
18920
|
+
var _a25, _b25, _c;
|
|
18921
|
+
const { model, api } = this.options;
|
|
18922
|
+
if ((!this.continuous || api.session != null) && this.options.maxPlaybackBufferSeconds != null)
|
|
18923
|
+
throw new Error(
|
|
18924
|
+
"maxPlaybackBufferSeconds is supported only for continuous PCM sessions"
|
|
18925
|
+
);
|
|
18926
|
+
const connection = api.session != null ? "webrtc" : api.token != null ? "client-secret-websocket" : "server-websocket";
|
|
18927
|
+
if (!((_b25 = (_a25 = model.capabilities) == null ? void 0 : _a25.connections) != null ? _b25 : ["client-secret-websocket"]).includes(connection) || ((_c = model.capabilities) == null ? void 0 : _c.transports) != null && !model.capabilities.transports.includes(
|
|
18928
|
+
api.session != null ? "webrtc" : "websocket"
|
|
18929
|
+
))
|
|
18930
|
+
throw new Error(`Realtime model does not support ${connection}`);
|
|
18931
|
+
if (api.session != null) {
|
|
18932
|
+
if (model.getWebRTCConfig == null)
|
|
18933
|
+
throw new Error("Realtime model does not support WebRTC configuration");
|
|
18934
|
+
return;
|
|
18935
|
+
}
|
|
18936
|
+
if (this.continuous && api.websocket == null)
|
|
18937
|
+
throw new Error(
|
|
18938
|
+
"Continuous PCM sessions require an application WebSocket relay"
|
|
18939
|
+
);
|
|
18940
|
+
if (api.token != null && model.getWebSocketConfig == null)
|
|
18941
|
+
throw new Error(
|
|
18942
|
+
"Realtime model does not support client-secret WebSocket configuration"
|
|
18943
|
+
);
|
|
18944
|
+
}
|
|
18945
|
+
async connect(connectOptions) {
|
|
18946
|
+
var _a25, _b25, _c, _d;
|
|
18947
|
+
if ((_a25 = this.attempt) == null ? void 0 : _a25.active)
|
|
18948
|
+
throw new Error("Realtime session is already active");
|
|
18949
|
+
const attempt = new RealtimeAttempt();
|
|
18950
|
+
this.attempt = attempt;
|
|
18951
|
+
const current = () => this.attempt === attempt && attempt.active;
|
|
17668
18952
|
try {
|
|
17669
|
-
|
|
17670
|
-
|
|
17671
|
-
|
|
17672
|
-
|
|
18953
|
+
this.applyState({ ...this.state, status: "connecting" });
|
|
18954
|
+
if (!current()) return;
|
|
18955
|
+
this.validateConnection();
|
|
18956
|
+
const { model, api, sessionConfig } = this.options;
|
|
18957
|
+
if ((connectOptions == null ? void 0 : connectOptions.capture) === false) this.stopAudioCapture();
|
|
18958
|
+
if (!current()) return;
|
|
18959
|
+
if ((connectOptions == null ? void 0 : connectOptions.stream) != null && connectOptions.stream !== this.suppliedStream) {
|
|
18960
|
+
if (this.captureRequested && !this.sessionLifecycle && !this.continuous && api.session == null)
|
|
18961
|
+
this.startAudioCapture(connectOptions.stream);
|
|
18962
|
+
else this.suppliedStream = connectOptions.stream;
|
|
18963
|
+
}
|
|
18964
|
+
if (!current()) return;
|
|
18965
|
+
this.reducer = new RealtimeEventReducer(this.maxEvents);
|
|
18966
|
+
this.currentResponseItemId = null;
|
|
18967
|
+
this.toolCallsInResponse.clear();
|
|
18968
|
+
this.submittedToolOutputs.clear();
|
|
18969
|
+
this.responseToolCallsClosed = false;
|
|
18970
|
+
if (this.sessionLifecycle)
|
|
18971
|
+
this.applyState({ ...this.state, session: createSessionState() });
|
|
18972
|
+
if (!current()) return;
|
|
18973
|
+
this.commands = new RealtimeCommandTracker((event) => {
|
|
18974
|
+
const writable = () => current() && !attempt.transportClosing && attempt.cause == null && (!attempt.closing || event.type === "session-close");
|
|
18975
|
+
if (!writable()) throw new Error("Realtime connection is closed");
|
|
18976
|
+
return this.sendTransport(event, writable);
|
|
17673
18977
|
});
|
|
17674
|
-
|
|
17675
|
-
|
|
17676
|
-
|
|
17677
|
-
|
|
17678
|
-
const
|
|
17679
|
-
|
|
17680
|
-
|
|
17681
|
-
|
|
18978
|
+
attempt.timer("startup", (_b25 = this.options.startupTimeoutMs) != null ? _b25 : 3e4, () => {
|
|
18979
|
+
if (current())
|
|
18980
|
+
this.fail(new Error("Realtime session startup timed out"));
|
|
18981
|
+
});
|
|
18982
|
+
const callbacks = {
|
|
18983
|
+
model,
|
|
18984
|
+
onEvent: async (event) => {
|
|
18985
|
+
var _a26, _b26, _c2;
|
|
18986
|
+
if (!current()) return;
|
|
18987
|
+
try {
|
|
18988
|
+
await this.handleServerEvent(event, attempt);
|
|
18989
|
+
} catch (error) {
|
|
18990
|
+
if (current())
|
|
18991
|
+
this.fail(
|
|
18992
|
+
error,
|
|
18993
|
+
(_c2 = (_b26 = (_a26 = this.rtc) != null ? _a26 : this.pcm) != null ? _b26 : this.transport) == null ? void 0 : _c2.finish()
|
|
18994
|
+
);
|
|
18995
|
+
}
|
|
18996
|
+
},
|
|
18997
|
+
onError: (error) => {
|
|
18998
|
+
if (current()) void this.reportError(error, attempt);
|
|
18999
|
+
},
|
|
19000
|
+
onFatalError: (error, drain) => {
|
|
19001
|
+
if (current()) this.fail(error, drain);
|
|
19002
|
+
},
|
|
19003
|
+
onClosing: () => {
|
|
19004
|
+
if (!current()) return;
|
|
19005
|
+
attempt.beginClose();
|
|
19006
|
+
attempt.transportClosing = true;
|
|
19007
|
+
attempt.clearTimer("startup");
|
|
19008
|
+
attempt.clearTimer("close");
|
|
19009
|
+
this.stopAudioCapture();
|
|
19010
|
+
if (!current()) return;
|
|
19011
|
+
this.applyState({
|
|
19012
|
+
...this.state,
|
|
19013
|
+
status: attempt.cause == null ? "closing" : "error"
|
|
19014
|
+
});
|
|
19015
|
+
},
|
|
19016
|
+
onClose: (error) => {
|
|
19017
|
+
var _a26, _b26;
|
|
19018
|
+
if (!current()) return;
|
|
19019
|
+
const finalizationConfirmed = ((_a26 = model.capabilities) == null ? void 0 : _a26.finalization) === "session-close" && ((_b26 = this.state.session) == null ? void 0 : _b26.finalization) === "confirmed";
|
|
19020
|
+
if (error != null && !finalizationConfirmed) this.fail(error);
|
|
19021
|
+
else if (!attempt.ready && !finalizationConfirmed)
|
|
19022
|
+
this.fail(
|
|
19023
|
+
new Error("Realtime connection closed before becoming ready")
|
|
19024
|
+
);
|
|
19025
|
+
else this.disconnect();
|
|
19026
|
+
},
|
|
19027
|
+
onCapturing: (isCapturing) => {
|
|
19028
|
+
if (current()) this.applyState({ ...this.state, isCapturing });
|
|
19029
|
+
},
|
|
19030
|
+
onPlaying: (isPlaying) => {
|
|
19031
|
+
if (current()) this.applyState({ ...this.state, isPlaying });
|
|
19032
|
+
}
|
|
17682
19033
|
};
|
|
17683
|
-
|
|
17684
|
-
|
|
17685
|
-
|
|
17686
|
-
|
|
17687
|
-
|
|
17688
|
-
|
|
17689
|
-
|
|
17690
|
-
|
|
19034
|
+
if (api.session != null) {
|
|
19035
|
+
this.rtc = new BrowserRealtimeWebRTC({
|
|
19036
|
+
...callbacks,
|
|
19037
|
+
disconnectTimeoutMs: this.options.rtcDisconnectTimeoutMs
|
|
19038
|
+
});
|
|
19039
|
+
await this.rtc.connect({
|
|
19040
|
+
api: api.session,
|
|
19041
|
+
sessionConfig,
|
|
19042
|
+
stream: connectOptions == null ? void 0 : connectOptions.stream,
|
|
19043
|
+
capture: connectOptions == null ? void 0 : connectOptions.capture,
|
|
19044
|
+
timeoutMs: (_c = this.options.startupTimeoutMs) != null ? _c : 3e4
|
|
19045
|
+
});
|
|
19046
|
+
} else if (api.websocket != null && ((_d = model.capabilities) == null ? void 0 : _d.conversation) === "continuous") {
|
|
19047
|
+
this.pcm = new BrowserRealtimeLiveWebSocket({
|
|
19048
|
+
...callbacks,
|
|
19049
|
+
sessionConfig,
|
|
19050
|
+
sampleRate: this.options.sampleRate,
|
|
19051
|
+
maxPlaybackBufferSeconds: this.options.maxPlaybackBufferSeconds
|
|
19052
|
+
});
|
|
19053
|
+
this.pcm.connect({
|
|
19054
|
+
url: api.websocket,
|
|
19055
|
+
protocols: api.protocols,
|
|
19056
|
+
stream: connectOptions == null ? void 0 : connectOptions.stream,
|
|
19057
|
+
capture: connectOptions == null ? void 0 : connectOptions.capture
|
|
19058
|
+
});
|
|
19059
|
+
} else {
|
|
19060
|
+
this.ensureAudio();
|
|
19061
|
+
let config = sessionConfig != null ? sessionConfig : {};
|
|
19062
|
+
let token;
|
|
19063
|
+
let url = api.websocket;
|
|
19064
|
+
if (api.token != null) {
|
|
19065
|
+
const response = await fetch(api.token, {
|
|
19066
|
+
method: "POST",
|
|
19067
|
+
headers: { "Content-Type": "application/json" },
|
|
19068
|
+
body: JSON.stringify({ sessionConfig }),
|
|
19069
|
+
signal: attempt.abort.signal
|
|
19070
|
+
});
|
|
19071
|
+
if (!current()) return;
|
|
19072
|
+
if (!response.ok)
|
|
19073
|
+
throw new Error(
|
|
19074
|
+
`Failed to fetch realtime setup: ${response.status}`
|
|
19075
|
+
);
|
|
19076
|
+
const payload = await response.json().catch(() => {
|
|
19077
|
+
throw new Error("Invalid realtime setup response");
|
|
17691
19078
|
});
|
|
19079
|
+
if (!current() || attempt.closing) return;
|
|
19080
|
+
const setup = validateRealtimeSetup(payload);
|
|
19081
|
+
token = setup.token;
|
|
19082
|
+
url = setup.url;
|
|
19083
|
+
config = {
|
|
19084
|
+
...sessionConfig,
|
|
19085
|
+
...setup.tools == null ? {} : { tools: setup.tools }
|
|
19086
|
+
};
|
|
17692
19087
|
}
|
|
17693
|
-
|
|
19088
|
+
if (url == null) throw new Error("Realtime WebSocket URL is missing");
|
|
19089
|
+
if (api.token != null && token == null)
|
|
19090
|
+
throw new Error("Realtime client-secret connection requires a token");
|
|
19091
|
+
this.ensureAudio().ensurePlaybackContext();
|
|
19092
|
+
if (!current()) return;
|
|
19093
|
+
this.transport = new BrowserRealtimeTransport({
|
|
19094
|
+
...callbacks,
|
|
19095
|
+
onServerEvent: callbacks.onEvent
|
|
19096
|
+
});
|
|
19097
|
+
this.transport.connect({
|
|
19098
|
+
...token != null ? { mode: "client-secret", token } : { mode: "relay", protocols: api.protocols },
|
|
19099
|
+
url,
|
|
19100
|
+
onOpen: () => {
|
|
19101
|
+
var _a26, _b26;
|
|
19102
|
+
const writable = () => current() && !attempt.closing && attempt.cause == null;
|
|
19103
|
+
if (!writable()) return;
|
|
19104
|
+
return this.sendTransport(
|
|
19105
|
+
{
|
|
19106
|
+
type: (_b26 = (_a26 = model.capabilities) == null ? void 0 : _a26.startup) != null ? _b26 : "session-update",
|
|
19107
|
+
config
|
|
19108
|
+
},
|
|
19109
|
+
writable
|
|
19110
|
+
);
|
|
19111
|
+
}
|
|
19112
|
+
});
|
|
19113
|
+
}
|
|
17694
19114
|
} catch (error) {
|
|
17695
|
-
|
|
17696
|
-
|
|
17697
|
-
|
|
17698
|
-
|
|
17699
|
-
|
|
19115
|
+
if (current()) this.fail(error);
|
|
19116
|
+
}
|
|
19117
|
+
}
|
|
19118
|
+
fail(error, drain) {
|
|
19119
|
+
const attempt = this.attempt;
|
|
19120
|
+
if (attempt == null || !attempt.active || attempt.cause != null) return;
|
|
19121
|
+
attempt.cause = error instanceof Error ? error : new Error(String(error));
|
|
19122
|
+
if (drain != null) attempt.beginClose();
|
|
19123
|
+
this.applyState({ ...this.state, status: "error" });
|
|
19124
|
+
if (this.attempt !== attempt || !attempt.active) return;
|
|
19125
|
+
this.stopAudioCapture();
|
|
19126
|
+
if (this.attempt !== attempt || !attempt.active) return;
|
|
19127
|
+
if (drain == null) this.disconnect();
|
|
19128
|
+
else void this.drainAttempt(attempt, drain);
|
|
19129
|
+
void this.reportError(attempt.cause, attempt);
|
|
19130
|
+
}
|
|
19131
|
+
async drainAttempt(attempt, drain) {
|
|
19132
|
+
if (this.attempt === attempt && attempt.active)
|
|
19133
|
+
attempt.timer("drain", 1e3, () => this.finishAttempt(attempt));
|
|
19134
|
+
try {
|
|
19135
|
+
await drain;
|
|
19136
|
+
} catch (e) {
|
|
19137
|
+
}
|
|
19138
|
+
this.finishAttempt(attempt);
|
|
19139
|
+
}
|
|
19140
|
+
finishAttempt(attempt) {
|
|
19141
|
+
if (this.attempt !== attempt || !attempt.active) return;
|
|
19142
|
+
try {
|
|
19143
|
+
this.disconnect();
|
|
19144
|
+
} catch (error) {
|
|
19145
|
+
void this.reportError(error, attempt);
|
|
17700
19146
|
}
|
|
17701
19147
|
}
|
|
19148
|
+
closeFailed(attempt, error) {
|
|
19149
|
+
var _a25, _b25;
|
|
19150
|
+
if (this.attempt !== attempt || !attempt.active || attempt.transportClosing)
|
|
19151
|
+
return;
|
|
19152
|
+
attempt.clearTimer("close");
|
|
19153
|
+
const transport = (_b25 = (_a25 = this.rtc) != null ? _a25 : this.pcm) != null ? _b25 : this.transport;
|
|
19154
|
+
void this.drainAttempt(attempt, transport == null ? void 0 : transport.finish());
|
|
19155
|
+
void this.reportError(error, attempt);
|
|
19156
|
+
}
|
|
17702
19157
|
disconnect() {
|
|
17703
|
-
this.
|
|
17704
|
-
this.
|
|
19158
|
+
const attempt = this.attempt;
|
|
19159
|
+
this.captureGeneration++;
|
|
19160
|
+
this.captureRequested = false;
|
|
19161
|
+
this.suppliedStream = void 0;
|
|
19162
|
+
const transport = this.transport;
|
|
19163
|
+
const audio = this.audio;
|
|
19164
|
+
const pcm = this.pcm;
|
|
19165
|
+
const rtc = this.rtc;
|
|
19166
|
+
this.transport = void 0;
|
|
19167
|
+
this.audio = void 0;
|
|
19168
|
+
this.pcm = void 0;
|
|
19169
|
+
this.rtc = void 0;
|
|
19170
|
+
this.commands = void 0;
|
|
19171
|
+
attempt == null ? void 0 : attempt.retire();
|
|
19172
|
+
transport == null ? void 0 : transport.dispose();
|
|
19173
|
+
audio == null ? void 0 : audio.dispose();
|
|
19174
|
+
pcm == null ? void 0 : pcm.dispose();
|
|
19175
|
+
rtc == null ? void 0 : rtc.dispose();
|
|
19176
|
+
if (this.attempt !== attempt) return;
|
|
19177
|
+
const session = this.state.session;
|
|
19178
|
+
this.applyState({
|
|
19179
|
+
...this.state,
|
|
19180
|
+
status: (attempt == null ? void 0 : attempt.cause) != null ? "error" : "disconnected",
|
|
19181
|
+
isCapturing: false,
|
|
19182
|
+
isPlaying: false,
|
|
19183
|
+
...session != null ? {
|
|
19184
|
+
session: session.finalization === "confirmed" ? session : { ...session, finalization: "unconfirmed" }
|
|
19185
|
+
} : {}
|
|
19186
|
+
});
|
|
19187
|
+
}
|
|
19188
|
+
/** Wait for final usage when the model supports a session-close acknowledgement. */
|
|
19189
|
+
close(options) {
|
|
19190
|
+
var _a25, _b25, _c, _d;
|
|
19191
|
+
const attempt = this.attempt;
|
|
19192
|
+
if ((attempt == null ? void 0 : attempt.closePromise) != null) return attempt.closePromise;
|
|
19193
|
+
if (this.state.status !== "connected" || ((_a25 = this.options.model.capabilities) == null ? void 0 : _a25.finalization) !== "session-close" || attempt == null) {
|
|
19194
|
+
this.disconnect();
|
|
19195
|
+
return Promise.resolve();
|
|
19196
|
+
}
|
|
19197
|
+
if ((options == null ? void 0 : options.eventId) != null) (_b25 = this.commands) == null ? void 0 : _b25.validateId(options.eventId);
|
|
19198
|
+
const promise = attempt.beginClose();
|
|
19199
|
+
this.applyState({ ...this.state, status: "closing" });
|
|
19200
|
+
if (this.attempt !== attempt || !attempt.active || attempt.transportClosing)
|
|
19201
|
+
return promise;
|
|
19202
|
+
this.stopAudioCapture();
|
|
19203
|
+
if (this.attempt !== attempt || !attempt.active || attempt.transportClosing)
|
|
19204
|
+
return promise;
|
|
19205
|
+
attempt.timer(
|
|
19206
|
+
"close",
|
|
19207
|
+
(_c = this.options.closeTimeoutMs) != null ? _c : 15e3,
|
|
19208
|
+
() => this.finishAttempt(attempt)
|
|
19209
|
+
);
|
|
19210
|
+
const failed = (error) => this.closeFailed(attempt, error);
|
|
19211
|
+
try {
|
|
19212
|
+
void ((_d = this.commands) == null ? void 0 : _d.send({ type: "session-close", eventId: options == null ? void 0 : options.eventId }).catch(failed));
|
|
19213
|
+
} catch (error) {
|
|
19214
|
+
failed(error);
|
|
19215
|
+
}
|
|
19216
|
+
return promise;
|
|
19217
|
+
}
|
|
19218
|
+
sendTransport(event, guard, automaticAudio = false) {
|
|
19219
|
+
var _a25, _b25;
|
|
19220
|
+
const transport = (_b25 = (_a25 = this.rtc) != null ? _a25 : this.pcm) != null ? _b25 : this.transport;
|
|
19221
|
+
if (transport == null) throw new Error("Realtime connection is not open");
|
|
19222
|
+
if (transport === this.transport)
|
|
19223
|
+
return this.transport.sendEvent(event, guard, automaticAudio);
|
|
19224
|
+
return transport.sendEvent(event, guard);
|
|
17705
19225
|
}
|
|
17706
|
-
// ── Sending events ─────────────────────────────────────────────────
|
|
17707
19226
|
sendEvent(event) {
|
|
17708
|
-
|
|
19227
|
+
var _a25, _b25, _c, _d;
|
|
19228
|
+
if (this.state.status === "error" || this.state.status === "closing" || ((_a25 = this.attempt) == null ? void 0 : _a25.closing) || !((_b25 = this.attempt) == null ? void 0 : _b25.active))
|
|
19229
|
+
throw new Error("Realtime session is not accepting submissions");
|
|
19230
|
+
if (this.sessionLifecycle && this.state.status !== "connected")
|
|
19231
|
+
throw new Error("Realtime session is not accepting submissions");
|
|
19232
|
+
if (this.rtc != null && event.type === "input-audio-append" || (this.continuous || this.rtc != null) && (event.type === "input-audio-commit" || event.type === "input-audio-clear"))
|
|
19233
|
+
throw new UnsupportedFunctionalityError6({
|
|
19234
|
+
functionality: "JSON audio commands unsupported for this session transport"
|
|
19235
|
+
});
|
|
19236
|
+
if (event.type === "session-close")
|
|
19237
|
+
return this.close({ eventId: event.eventId });
|
|
19238
|
+
if (event.type === "session-start" && this.sessionLifecycle)
|
|
19239
|
+
throw new Error("Realtime session has already started");
|
|
19240
|
+
if (!this.sessionLifecycle && this.state.session == null) {
|
|
19241
|
+
const attempt = this.attempt;
|
|
19242
|
+
return this.sendTransport(
|
|
19243
|
+
event,
|
|
19244
|
+
() => this.attempt === attempt && attempt.active && !attempt.closing && attempt.cause == null
|
|
19245
|
+
);
|
|
19246
|
+
}
|
|
19247
|
+
return (_d = (_c = this.commands) == null ? void 0 : _c.send(event)) != null ? _d : this.sendTransport(event);
|
|
17709
19248
|
}
|
|
17710
19249
|
sendTextMessage(text2) {
|
|
19250
|
+
if (this.continuous)
|
|
19251
|
+
throw new UnsupportedFunctionalityError6({
|
|
19252
|
+
functionality: "sendTextMessage for continuous sessions; client-delegation text is application-owned, use context-append for context"
|
|
19253
|
+
});
|
|
17711
19254
|
this.sendEvent({
|
|
17712
19255
|
type: "conversation-item-create",
|
|
17713
19256
|
item: { type: "text-message", role: "user", text: text2 }
|
|
@@ -17715,8 +19258,12 @@ var AbstractRealtimeSession = class {
|
|
|
17715
19258
|
this.sendEvent({ type: "response-create" });
|
|
17716
19259
|
this.applyState(this.reducer.addUserTextMessage(this.state, text2));
|
|
17717
19260
|
}
|
|
17718
|
-
sendAudio(
|
|
17719
|
-
|
|
19261
|
+
sendAudio(audio) {
|
|
19262
|
+
const attempt = this.attempt;
|
|
19263
|
+
void this.sendEvent({ type: "input-audio-append", audio }).catch((error) => {
|
|
19264
|
+
if ((attempt == null ? void 0 : attempt.active) && this.attempt === attempt)
|
|
19265
|
+
void this.reportError(error, attempt);
|
|
19266
|
+
});
|
|
17720
19267
|
}
|
|
17721
19268
|
commitAudio() {
|
|
17722
19269
|
this.sendEvent({ type: "input-audio-commit" });
|
|
@@ -17733,161 +19280,301 @@ var AbstractRealtimeSession = class {
|
|
|
17733
19280
|
cancelResponse() {
|
|
17734
19281
|
this.sendEvent({ type: "response-cancel" });
|
|
17735
19282
|
}
|
|
17736
|
-
|
|
19283
|
+
sendAutomaticAudio(audio) {
|
|
19284
|
+
var _a25;
|
|
19285
|
+
if (this.state.status !== "connected" || !((_a25 = this.attempt) == null ? void 0 : _a25.active) || this.attempt.closing)
|
|
19286
|
+
return;
|
|
19287
|
+
const attempt = this.attempt;
|
|
19288
|
+
const failed = (error) => {
|
|
19289
|
+
if ((attempt == null ? void 0 : attempt.active) && this.attempt === attempt && !attempt.closing)
|
|
19290
|
+
this.fail(error);
|
|
19291
|
+
};
|
|
19292
|
+
try {
|
|
19293
|
+
void this.sendTransport(
|
|
19294
|
+
{ type: "input-audio-append", audio },
|
|
19295
|
+
() => this.attempt === attempt && attempt.active && !attempt.closing,
|
|
19296
|
+
true
|
|
19297
|
+
).catch(failed);
|
|
19298
|
+
} catch (error) {
|
|
19299
|
+
failed(error);
|
|
19300
|
+
}
|
|
19301
|
+
}
|
|
17737
19302
|
addToolOutput(callId, result) {
|
|
19303
|
+
if (this.continuous)
|
|
19304
|
+
throw new UnsupportedFunctionalityError6({
|
|
19305
|
+
functionality: "addToolOutput for continuous sessions; client delegation is application-owned"
|
|
19306
|
+
});
|
|
19307
|
+
const attempt = this.attempt;
|
|
19308
|
+
if (!(attempt == null ? void 0 : attempt.active) || attempt.closing || attempt.cause != null)
|
|
19309
|
+
throw new Error("Realtime session is not accepting submissions");
|
|
17738
19310
|
const { state, output } = this.reducer.addToolOutput(
|
|
17739
19311
|
this.state,
|
|
17740
19312
|
callId,
|
|
17741
19313
|
result
|
|
17742
19314
|
);
|
|
17743
19315
|
this.applyState(state);
|
|
19316
|
+
if (this.attempt !== attempt || !(attempt == null ? void 0 : attempt.active)) return;
|
|
17744
19317
|
this.sendEvent({
|
|
17745
19318
|
type: "conversation-item-create",
|
|
17746
|
-
item: {
|
|
17747
|
-
type: "function-call-output",
|
|
17748
|
-
callId: output.callId,
|
|
17749
|
-
name: output.name,
|
|
17750
|
-
output: output.output
|
|
17751
|
-
}
|
|
19319
|
+
item: { type: "function-call-output", ...output }
|
|
17752
19320
|
});
|
|
17753
19321
|
this.submittedToolOutputs.add(callId);
|
|
17754
19322
|
this.maybeRequestToolResponse();
|
|
17755
19323
|
}
|
|
17756
|
-
/**
|
|
17757
|
-
* Requests a single response once the tool-bearing response has finished
|
|
17758
|
-
* delivering its tool calls and every one of them has an output. Requesting a
|
|
17759
|
-
* response after each individual output can cause the model to continue
|
|
17760
|
-
* without the full tool context on multi-tool turns.
|
|
17761
|
-
*/
|
|
17762
19324
|
maybeRequestToolResponse() {
|
|
17763
|
-
|
|
17764
|
-
if (this.toolCallsInResponse.size === 0
|
|
17765
|
-
|
|
17766
|
-
|
|
17767
|
-
|
|
19325
|
+
var _a25;
|
|
19326
|
+
if (this.state.status !== "connected" || ((_a25 = this.attempt) == null ? void 0 : _a25.closing) || !this.responseToolCallsClosed || this.toolCallsInResponse.size === 0 || [...this.toolCallsInResponse].some(
|
|
19327
|
+
(id) => !this.submittedToolOutputs.has(id)
|
|
19328
|
+
))
|
|
19329
|
+
return;
|
|
17768
19330
|
this.sendEvent({ type: "response-create" });
|
|
17769
19331
|
this.toolCallsInResponse.clear();
|
|
17770
19332
|
this.submittedToolOutputs.clear();
|
|
17771
19333
|
this.responseToolCallsClosed = false;
|
|
17772
19334
|
}
|
|
17773
|
-
// ── Audio capture ──────────────────────────────────────────────────
|
|
17774
19335
|
startAudioCapture(stream) {
|
|
17775
|
-
|
|
19336
|
+
var _a25;
|
|
19337
|
+
const legacy = !this.sessionLifecycle && !this.continuous && this.options.api.session == null;
|
|
19338
|
+
if (!legacy && this.state.status !== "connected")
|
|
19339
|
+
throw new Error("Realtime session is not accepting capture");
|
|
19340
|
+
if (((_a25 = this.attempt) == null ? void 0 : _a25.active) && (this.attempt.closing || this.attempt.cause != null))
|
|
19341
|
+
throw new Error("Realtime session is not accepting capture");
|
|
19342
|
+
this.suppliedStream = stream;
|
|
19343
|
+
if (legacy) {
|
|
19344
|
+
this.captureGeneration++;
|
|
19345
|
+
this.captureRequested = true;
|
|
19346
|
+
this.ensureAudio().startCapture(stream);
|
|
19347
|
+
return;
|
|
19348
|
+
}
|
|
19349
|
+
const attempt = this.attempt;
|
|
19350
|
+
void this.resumeAudioCapture().catch((error) => {
|
|
19351
|
+
if (attempt == null ? void 0 : attempt.active) void this.reportError(error, attempt);
|
|
19352
|
+
});
|
|
19353
|
+
}
|
|
19354
|
+
async resumeAudioCapture() {
|
|
19355
|
+
const accepting = () => {
|
|
19356
|
+
var _a25, _b25;
|
|
19357
|
+
return !((_a25 = this.attempt) == null ? void 0 : _a25.closing) && ((_b25 = this.attempt) == null ? void 0 : _b25.cause) == null && (this.state.status === "connected" || !this.sessionLifecycle && this.state.status === "connecting");
|
|
19358
|
+
};
|
|
19359
|
+
if (!accepting())
|
|
19360
|
+
throw new Error("Realtime session is not accepting capture");
|
|
19361
|
+
this.captureRequested = true;
|
|
19362
|
+
if (this.rtc != null) return this.rtc.startCapture(this.suppliedStream);
|
|
19363
|
+
if (this.pcm != null) return this.pcm.resumeCapture(this.suppliedStream);
|
|
19364
|
+
const audio = this.audio;
|
|
19365
|
+
if (audio == null)
|
|
19366
|
+
throw new Error("Realtime capture transport is not ready");
|
|
19367
|
+
const captureGeneration = ++this.captureGeneration;
|
|
19368
|
+
const attempt = this.attempt;
|
|
19369
|
+
const supplied = this.suppliedStream;
|
|
19370
|
+
const stream = supplied != null ? supplied : await navigator.mediaDevices.getUserMedia({ audio: true });
|
|
19371
|
+
if (!(attempt == null ? void 0 : attempt.active) || this.attempt !== attempt || this.audio !== audio || !accepting() || captureGeneration !== this.captureGeneration) {
|
|
19372
|
+
if (supplied == null) stream.getTracks().forEach((track) => track.stop());
|
|
19373
|
+
return;
|
|
19374
|
+
}
|
|
19375
|
+
audio.startCapture(stream, {
|
|
19376
|
+
ownsStream: supplied == null || !this.sessionLifecycle || this.options.api.token != null
|
|
19377
|
+
});
|
|
17776
19378
|
}
|
|
17777
19379
|
stopAudioCapture() {
|
|
17778
|
-
this.
|
|
19380
|
+
this.captureGeneration++;
|
|
19381
|
+
this.captureRequested = false;
|
|
19382
|
+
if (this.audio != null && (!this.sessionLifecycle || this.options.api.token != null))
|
|
19383
|
+
this.suppliedStream = void 0;
|
|
19384
|
+
const pcm = this.pcm;
|
|
19385
|
+
const audio = this.audio;
|
|
19386
|
+
const rtc = this.rtc;
|
|
19387
|
+
const attempt = this.attempt;
|
|
19388
|
+
if (rtc != null)
|
|
19389
|
+
void rtc.stopCapture().catch((error) => {
|
|
19390
|
+
if ((attempt == null ? void 0 : attempt.active) && this.attempt === attempt)
|
|
19391
|
+
void this.reportError(error, attempt);
|
|
19392
|
+
});
|
|
19393
|
+
pcm == null ? void 0 : pcm.stopCapture();
|
|
19394
|
+
audio == null ? void 0 : audio.stopCapture();
|
|
17779
19395
|
}
|
|
17780
|
-
// ── Playback ───────────────────────────────────────────────────────
|
|
17781
19396
|
stopPlayback() {
|
|
17782
|
-
|
|
19397
|
+
var _a25, _b25, _c;
|
|
19398
|
+
(_c = (_b25 = (_a25 = this.rtc) != null ? _a25 : this.pcm) != null ? _b25 : this.audio) == null ? void 0 : _c.stopPlayback();
|
|
19399
|
+
}
|
|
19400
|
+
async resumePlayback() {
|
|
19401
|
+
var _a25, _b25, _c;
|
|
19402
|
+
await ((_c = (_b25 = (_a25 = this.rtc) != null ? _a25 : this.pcm) != null ? _b25 : this.audio) == null ? void 0 : _c.resumePlayback());
|
|
17783
19403
|
}
|
|
17784
|
-
// ── Cleanup ────────────────────────────────────────────────────────
|
|
17785
19404
|
dispose() {
|
|
17786
|
-
this.
|
|
17787
|
-
|
|
17788
|
-
|
|
17789
|
-
|
|
17790
|
-
|
|
17791
|
-
|
|
17792
|
-
|
|
17793
|
-
|
|
17794
|
-
|
|
17795
|
-
)
|
|
17796
|
-
|
|
19405
|
+
this.disconnect();
|
|
19406
|
+
}
|
|
19407
|
+
ensureAudio() {
|
|
19408
|
+
var _a25, _b25, _c, _d, _e, _f;
|
|
19409
|
+
if (this.audio != null) return this.audio;
|
|
19410
|
+
const { sessionConfig, sampleRate } = this.options;
|
|
19411
|
+
const audio = new BrowserRealtimeAudio({
|
|
19412
|
+
captureSampleRate: (_c = (_b25 = (_a25 = sessionConfig == null ? void 0 : sessionConfig.inputAudioFormat) == null ? void 0 : _a25.rate) != null ? _b25 : sampleRate) != null ? _c : 24e3,
|
|
19413
|
+
playbackSampleRate: (_f = (_e = (_d = sessionConfig == null ? void 0 : sessionConfig.outputAudioFormat) == null ? void 0 : _d.rate) != null ? _e : sampleRate) != null ? _f : 24e3,
|
|
19414
|
+
onAudio: (value) => {
|
|
19415
|
+
if (this.audio === audio) this.sendAutomaticAudio(value);
|
|
19416
|
+
},
|
|
19417
|
+
onError: (error) => {
|
|
19418
|
+
if (this.audio !== audio) return;
|
|
19419
|
+
void this.reportError(error, this.attempt);
|
|
19420
|
+
},
|
|
19421
|
+
onCapturingChange: (isCapturing) => {
|
|
19422
|
+
if (this.audio === audio)
|
|
19423
|
+
this.applyState({ ...this.state, isCapturing });
|
|
19424
|
+
},
|
|
19425
|
+
onPlayingChange: (isPlaying) => {
|
|
19426
|
+
if (this.audio === audio) this.applyState({ ...this.state, isPlaying });
|
|
19427
|
+
}
|
|
19428
|
+
});
|
|
19429
|
+
this.audio = audio;
|
|
19430
|
+
return audio;
|
|
17797
19431
|
}
|
|
17798
|
-
// ── Private helpers ────────────────────────────────────────────────
|
|
17799
19432
|
applyState(nextState) {
|
|
17800
|
-
const
|
|
19433
|
+
const publication = ++this.publication;
|
|
19434
|
+
const previous = this.state;
|
|
17801
19435
|
this.state = nextState;
|
|
17802
|
-
|
|
17803
|
-
this.
|
|
17804
|
-
|
|
17805
|
-
|
|
17806
|
-
|
|
17807
|
-
|
|
17808
|
-
|
|
17809
|
-
|
|
17810
|
-
|
|
17811
|
-
|
|
17812
|
-
|
|
17813
|
-
|
|
17814
|
-
if (
|
|
17815
|
-
this.setState("isPlaying", nextState.isPlaying);
|
|
17816
|
-
}
|
|
17817
|
-
}
|
|
17818
|
-
async executeToolCall({
|
|
17819
|
-
name: name25,
|
|
17820
|
-
args,
|
|
17821
|
-
callId
|
|
17822
|
-
}) {
|
|
17823
|
-
var _a25, _b25;
|
|
17824
|
-
if (this.onToolCall == null) {
|
|
17825
|
-
(_a25 = this.onError) == null ? void 0 : _a25.call(this, new Error(`No handler provided for tool "${name25}"`));
|
|
19436
|
+
const update = (key) => {
|
|
19437
|
+
if (publication === this.publication && previous[key] !== nextState[key])
|
|
19438
|
+
this.setState(key, nextState[key]);
|
|
19439
|
+
};
|
|
19440
|
+
update("status");
|
|
19441
|
+
update("messages");
|
|
19442
|
+
update("events");
|
|
19443
|
+
update("isCapturing");
|
|
19444
|
+
update("isPlaying");
|
|
19445
|
+
update("session");
|
|
19446
|
+
}
|
|
19447
|
+
async executeTool(callId, name25, args, attempt) {
|
|
19448
|
+
if (this.continuous || this.attempt !== attempt || !attempt.active || attempt.closing || attempt.cause != null)
|
|
17826
19449
|
return;
|
|
17827
|
-
}
|
|
17828
19450
|
try {
|
|
19451
|
+
if (this.onToolCall == null) {
|
|
19452
|
+
void this.reportError(
|
|
19453
|
+
new Error(`No handler provided for tool "${name25}"`),
|
|
19454
|
+
attempt
|
|
19455
|
+
);
|
|
19456
|
+
return;
|
|
19457
|
+
}
|
|
17829
19458
|
const result = await this.onToolCall({
|
|
17830
19459
|
toolCall: { toolCallId: callId, toolName: name25, args }
|
|
17831
19460
|
});
|
|
17832
|
-
if (result !== void 0)
|
|
19461
|
+
if (result !== void 0 && this.attempt === attempt && (attempt == null ? void 0 : attempt.active) && this.state.status === "connected")
|
|
17833
19462
|
this.addToolOutput(callId, result);
|
|
17834
|
-
}
|
|
17835
19463
|
} catch (error) {
|
|
17836
|
-
(
|
|
17837
|
-
this,
|
|
17838
|
-
error instanceof Error ? error : new Error(`Client tool execution failed: ${String(error)}`)
|
|
17839
|
-
);
|
|
19464
|
+
if (attempt.active) void this.reportError(error, attempt);
|
|
17840
19465
|
}
|
|
17841
19466
|
}
|
|
17842
|
-
async handleServerEvent(event) {
|
|
17843
|
-
var _a25;
|
|
19467
|
+
async handleServerEvent(event, attempt) {
|
|
19468
|
+
var _a25, _b25, _c, _d, _e, _f;
|
|
19469
|
+
const current = () => this.attempt === attempt && attempt.active;
|
|
19470
|
+
if (!current()) return;
|
|
19471
|
+
if (event.type === "session-started" && event.delegationMode === "provider")
|
|
19472
|
+
throw new UnsupportedFunctionalityError6({
|
|
19473
|
+
functionality: "Provider delegation mode; this realtime runtime supports client delegation only"
|
|
19474
|
+
});
|
|
19475
|
+
if (this.continuous && event.type === "audio-delta")
|
|
19476
|
+
throw new UnsupportedFunctionalityError6({
|
|
19477
|
+
functionality: "Turn-based audio-delta in a continuous PCM session; use audio-chunk"
|
|
19478
|
+
});
|
|
19479
|
+
const command = (_a25 = this.commands) == null ? void 0 : _a25.receive(event);
|
|
17844
19480
|
const result = await this.reducer.reduceServerEvent(this.state, event);
|
|
17845
|
-
|
|
17846
|
-
(
|
|
19481
|
+
if (!current()) return;
|
|
19482
|
+
const session = (_b25 = this.state.session) != null ? _b25 : event.type === "session-started" ? createSessionState() : void 0;
|
|
19483
|
+
this.applyState({
|
|
19484
|
+
...result.state,
|
|
19485
|
+
status: this.state.status,
|
|
19486
|
+
...session == null ? {} : {
|
|
19487
|
+
session: reduceSessionState(
|
|
19488
|
+
session,
|
|
19489
|
+
event,
|
|
19490
|
+
this.maxEvents,
|
|
19491
|
+
command == null ? void 0 : command.muted
|
|
19492
|
+
)
|
|
19493
|
+
}
|
|
19494
|
+
});
|
|
19495
|
+
if (!current()) return;
|
|
17847
19496
|
for (const effect of result.effects) {
|
|
17848
|
-
this.handleReducerEffect(effect);
|
|
19497
|
+
this.handleReducerEffect(effect, attempt);
|
|
19498
|
+
if (!current()) return;
|
|
17849
19499
|
}
|
|
19500
|
+
if (event.type === "audio-chunk")
|
|
19501
|
+
(_d = (_c = this.pcm) != null ? _c : this.audio) == null ? void 0 : _d.playAudio(event.delta);
|
|
19502
|
+
if (!current()) return;
|
|
17850
19503
|
if (event.type === "response-done" && this.toolCallsInResponse.size > 0) {
|
|
17851
19504
|
this.responseToolCallsClosed = true;
|
|
17852
19505
|
this.maybeRequestToolResponse();
|
|
17853
19506
|
}
|
|
19507
|
+
if (!current()) return;
|
|
19508
|
+
if (event.type === "session-closed" && session != null) {
|
|
19509
|
+
this.disconnect();
|
|
19510
|
+
this.notifyEvent(event, attempt, true);
|
|
19511
|
+
return;
|
|
19512
|
+
}
|
|
19513
|
+
if ((event.type === "session-started" || event.type === "session-created" || event.type === "session-updated") && attempt.active && !attempt.closing && attempt.cause == null) {
|
|
19514
|
+
const ready = ((_e = this.options.model.capabilities) == null ? void 0 : _e.startup) === "session-start" ? event.type === "session-started" : event.type !== "session-started";
|
|
19515
|
+
if (ready) {
|
|
19516
|
+
attempt.ready = true;
|
|
19517
|
+
attempt.clearTimer("startup");
|
|
19518
|
+
this.applyState({ ...this.state, status: "connected" });
|
|
19519
|
+
if (!current() || attempt.closing) return;
|
|
19520
|
+
(_f = this.pcm) == null ? void 0 : _f.startCapture();
|
|
19521
|
+
}
|
|
19522
|
+
}
|
|
19523
|
+
if (current()) this.notifyEvent(event, attempt);
|
|
17854
19524
|
}
|
|
17855
|
-
handleReducerEffect(effect) {
|
|
17856
|
-
var _a25;
|
|
19525
|
+
handleReducerEffect(effect, attempt) {
|
|
19526
|
+
var _a25, _b25, _c, _d;
|
|
17857
19527
|
switch (effect.type) {
|
|
17858
|
-
case "play-audio":
|
|
19528
|
+
case "play-audio":
|
|
17859
19529
|
this.currentResponseItemId = effect.itemId;
|
|
17860
|
-
this.audio.playAudio(effect.delta);
|
|
19530
|
+
(_a25 = this.audio) == null ? void 0 : _a25.playAudio(effect.delta);
|
|
17861
19531
|
break;
|
|
17862
|
-
|
|
17863
|
-
|
|
17864
|
-
|
|
17865
|
-
const
|
|
17866
|
-
this.audio.stopPlayback();
|
|
17867
|
-
if (this.
|
|
19532
|
+
case "speech-started":
|
|
19533
|
+
if (!this.continuous && this.state.isPlaying) {
|
|
19534
|
+
const playedMs = (_c = (_b25 = this.audio) == null ? void 0 : _b25.getPlaybackOffsetMs()) != null ? _c : 0;
|
|
19535
|
+
const itemId = this.currentResponseItemId;
|
|
19536
|
+
(_d = this.audio) == null ? void 0 : _d.stopPlayback();
|
|
19537
|
+
if (this.attempt !== attempt || !attempt.active) return;
|
|
19538
|
+
if (itemId != null && !attempt.closing && attempt.cause == null)
|
|
17868
19539
|
this.sendEvent({
|
|
17869
19540
|
type: "conversation-item-truncate",
|
|
17870
|
-
itemId
|
|
19541
|
+
itemId,
|
|
17871
19542
|
contentIndex: 0,
|
|
17872
19543
|
audioEndMs: Math.round(playedMs)
|
|
17873
19544
|
});
|
|
17874
|
-
}
|
|
17875
19545
|
}
|
|
17876
19546
|
break;
|
|
17877
|
-
|
|
17878
|
-
|
|
19547
|
+
case "tool-call":
|
|
19548
|
+
if (this.continuous) break;
|
|
17879
19549
|
this.toolCallsInResponse.add(effect.callId);
|
|
17880
|
-
void this.
|
|
17881
|
-
name: effect.name,
|
|
17882
|
-
args: effect.args,
|
|
17883
|
-
callId: effect.callId
|
|
17884
|
-
});
|
|
19550
|
+
void this.executeTool(effect.callId, effect.name, effect.args, attempt);
|
|
17885
19551
|
break;
|
|
17886
|
-
|
|
17887
|
-
|
|
17888
|
-
(_a25 = this.onError) == null ? void 0 : _a25.call(this, effect.error);
|
|
19552
|
+
case "error":
|
|
19553
|
+
void this.reportError(effect.error, attempt);
|
|
17889
19554
|
break;
|
|
17890
|
-
|
|
19555
|
+
}
|
|
19556
|
+
}
|
|
19557
|
+
async reportError(error, attempt) {
|
|
19558
|
+
var _a25;
|
|
19559
|
+
if (this.attempt !== attempt) return;
|
|
19560
|
+
try {
|
|
19561
|
+
await ((_a25 = this.onError) == null ? void 0 : _a25.call(
|
|
19562
|
+
this,
|
|
19563
|
+
error instanceof Error ? error : new Error(String(error))
|
|
19564
|
+
));
|
|
19565
|
+
} catch (e) {
|
|
19566
|
+
}
|
|
19567
|
+
}
|
|
19568
|
+
notifyEvent(event, attempt, terminal = false) {
|
|
19569
|
+
var _a25;
|
|
19570
|
+
if (this.attempt !== attempt || !attempt.active && !terminal) return;
|
|
19571
|
+
const report = (error) => {
|
|
19572
|
+
if (attempt.active || terminal) void this.reportError(error, attempt);
|
|
19573
|
+
};
|
|
19574
|
+
try {
|
|
19575
|
+
void Promise.resolve((_a25 = this.onEvent) == null ? void 0 : _a25.call(this, event)).catch(report);
|
|
19576
|
+
} catch (error) {
|
|
19577
|
+
report(error);
|
|
17891
19578
|
}
|
|
17892
19579
|
}
|
|
17893
19580
|
};
|
|
@@ -18544,7 +20231,7 @@ var DefaultTranscriptionResult = class {
|
|
|
18544
20231
|
|
|
18545
20232
|
// src/transcribe/stream-transcribe.ts
|
|
18546
20233
|
import {
|
|
18547
|
-
UnsupportedFunctionalityError as
|
|
20234
|
+
UnsupportedFunctionalityError as UnsupportedFunctionalityError7
|
|
18548
20235
|
} from "@ai-sdk/provider";
|
|
18549
20236
|
import {
|
|
18550
20237
|
DelayedPromise as DelayedPromise3,
|
|
@@ -18567,7 +20254,7 @@ function streamTranscribe({
|
|
|
18567
20254
|
}
|
|
18568
20255
|
const doStream = (_a25 = resolvedModel.doStream) == null ? void 0 : _a25.bind(resolvedModel);
|
|
18569
20256
|
if (doStream == null) {
|
|
18570
|
-
throw new
|
|
20257
|
+
throw new UnsupportedFunctionalityError7({
|
|
18571
20258
|
functionality: "streaming transcription",
|
|
18572
20259
|
message: `The ${resolvedModel.provider} model "${resolvedModel.modelId}" does not support streaming transcription.` + (typeof model === "string" ? " String model IDs resolve through the global provider (AI Gateway by default). If that provider does not support streaming transcription, pass a provider model instance instead (e.g. openai.transcription('gpt-realtime-whisper')) or upgrade @ai-sdk/gateway to a version with streaming transcription support." : "")
|
|
18573
20260
|
});
|
|
@@ -19143,13 +20830,13 @@ import {
|
|
|
19143
20830
|
} from "@ai-sdk/provider-utils";
|
|
19144
20831
|
|
|
19145
20832
|
// src/ui/convert-file-list-to-file-ui-parts.ts
|
|
19146
|
-
import { UnsupportedFunctionalityError as
|
|
20833
|
+
import { UnsupportedFunctionalityError as UnsupportedFunctionalityError8 } from "@ai-sdk/provider";
|
|
19147
20834
|
async function convertFileListToFileUIParts(files) {
|
|
19148
20835
|
if (files == null) {
|
|
19149
20836
|
return [];
|
|
19150
20837
|
}
|
|
19151
20838
|
if (!globalThis.FileList || !(files instanceof globalThis.FileList)) {
|
|
19152
|
-
throw new
|
|
20839
|
+
throw new UnsupportedFunctionalityError8({
|
|
19153
20840
|
functionality: "FileList",
|
|
19154
20841
|
message: "FileList is not supported in the current environment"
|
|
19155
20842
|
});
|
|
@@ -19333,8 +21020,8 @@ var DefaultChatTransport = class extends HttpChatTransport {
|
|
|
19333
21020
|
// src/ui/chat.ts
|
|
19334
21021
|
var AbstractChat = class {
|
|
19335
21022
|
constructor({
|
|
19336
|
-
generateId:
|
|
19337
|
-
id =
|
|
21023
|
+
generateId: generateId5 = generateIdFunc2,
|
|
21024
|
+
id = generateId5(),
|
|
19338
21025
|
transport = new DefaultChatTransport(),
|
|
19339
21026
|
messageMetadataSchema,
|
|
19340
21027
|
dataPartSchemas,
|
|
@@ -19546,7 +21233,7 @@ var AbstractChat = class {
|
|
|
19546
21233
|
};
|
|
19547
21234
|
this.id = id;
|
|
19548
21235
|
this.transport = transport;
|
|
19549
|
-
this.generateId =
|
|
21236
|
+
this.generateId = generateId5;
|
|
19550
21237
|
this.messageMetadataSchema = messageMetadataSchema;
|
|
19551
21238
|
this.dataPartSchemas = dataPartSchemas;
|
|
19552
21239
|
this.state = state;
|
|
@@ -20132,7 +21819,7 @@ export {
|
|
|
20132
21819
|
extractReasoningMiddleware,
|
|
20133
21820
|
fingerprintTools,
|
|
20134
21821
|
gateway3 as gateway,
|
|
20135
|
-
|
|
21822
|
+
generateId4 as generateId,
|
|
20136
21823
|
generateImage,
|
|
20137
21824
|
generateObject,
|
|
20138
21825
|
generateSpeech,
|