@charivo/stt 0.4.0 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +13 -0
- package/dist/index.js +3 -0
- package/dist/index.mjs +3 -0
- package/dist/openai/index.d.mts +10 -2
- package/dist/openai/index.d.ts +10 -2
- package/dist/openai/index.js +4 -0
- package/dist/openai/index.mjs +2 -0
- package/dist/openai-realtime/index.d.mts +19 -0
- package/dist/openai-realtime/index.d.ts +19 -0
- package/dist/openai-realtime/index.js +581 -0
- package/dist/openai-realtime/index.mjs +560 -0
- package/package.json +9 -3
package/README.md
CHANGED
|
@@ -26,6 +26,14 @@ const text = await sttManager.stop();
|
|
|
26
26
|
|
|
27
27
|
- `createSTTManager(transcriber)`
|
|
28
28
|
- `isWebSTTSupported()` (from `@charivo/stt/web`) — SSR-safe check for Web Speech recognition support
|
|
29
|
+
- `@charivo/stt/openai`: `createOpenAISTTTranscriber(config)` (browser
|
|
30
|
+
transcriber, dev/testing only) and, for server-side use,
|
|
31
|
+
`createOpenAISTTProvider(config)`, `OpenAISTTProvider`, `type OpenAISTTConfig`
|
|
32
|
+
- `@charivo/stt/openai-realtime`: `createOpenAIRealtimeSTTTranscriber({ bootstrap })`
|
|
33
|
+
(live WebRTC streaming transcriber, `gpt-realtime-whisper`) — the app injects
|
|
34
|
+
`bootstrap(request) => Promise<{ answerSdp }>`, owns the credentials, and
|
|
35
|
+
mints the `type: "transcription"` session with `turn_detection: null`; no
|
|
36
|
+
key-bearing helper is shipped
|
|
29
37
|
|
|
30
38
|
## Event Bridge
|
|
31
39
|
|
|
@@ -36,5 +44,10 @@ through the shared event bus.
|
|
|
36
44
|
When connected, the manager emits:
|
|
37
45
|
|
|
38
46
|
- `stt:start`
|
|
47
|
+
- `stt:partial` (streaming transcribers only, e.g. `@charivo/stt/openai-realtime`)
|
|
39
48
|
- `stt:stop`
|
|
40
49
|
- `stt:error`
|
|
50
|
+
|
|
51
|
+
For the streaming transcriber, a mid-session failure does not push an event on
|
|
52
|
+
its own — it surfaces the next time the app calls `stop()`, which rejects and
|
|
53
|
+
emits `stt:error` (never a successful `stt:stop`).
|
package/dist/index.js
CHANGED
|
@@ -31,6 +31,9 @@ var STTManagerImpl = class {
|
|
|
31
31
|
eventEmitter;
|
|
32
32
|
constructor(sttTranscriber) {
|
|
33
33
|
this.sttTranscriber = sttTranscriber;
|
|
34
|
+
this.sttTranscriber.onPartial?.(
|
|
35
|
+
(transcription) => this.eventEmitter?.emit("stt:partial", { transcription })
|
|
36
|
+
);
|
|
34
37
|
}
|
|
35
38
|
/**
|
|
36
39
|
* Set event emitter for STT events
|
package/dist/index.mjs
CHANGED
|
@@ -7,6 +7,9 @@ var STTManagerImpl = class {
|
|
|
7
7
|
eventEmitter;
|
|
8
8
|
constructor(sttTranscriber) {
|
|
9
9
|
this.sttTranscriber = sttTranscriber;
|
|
10
|
+
this.sttTranscriber.onPartial?.(
|
|
11
|
+
(transcription) => this.eventEmitter?.emit("stt:partial", { transcription })
|
|
12
|
+
);
|
|
10
13
|
}
|
|
11
14
|
/**
|
|
12
15
|
* Set event emitter for STT events
|
package/dist/openai/index.d.mts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { STTTranscriber } from '@charivo/core';
|
|
1
|
+
import { STTProvider, STTOptions, STTTranscriber } from '@charivo/core';
|
|
2
2
|
|
|
3
3
|
interface OpenAISTTConfig {
|
|
4
4
|
apiKey: string;
|
|
@@ -6,8 +6,16 @@ interface OpenAISTTConfig {
|
|
|
6
6
|
defaultLanguage?: string;
|
|
7
7
|
dangerouslyAllowBrowser?: boolean;
|
|
8
8
|
}
|
|
9
|
+
declare class OpenAISTTProvider implements STTProvider {
|
|
10
|
+
private openai;
|
|
11
|
+
private defaultModel;
|
|
12
|
+
private defaultLanguage?;
|
|
13
|
+
constructor(config: OpenAISTTConfig);
|
|
14
|
+
transcribe(audio: Blob | ArrayBuffer, options?: STTOptions): Promise<string>;
|
|
15
|
+
}
|
|
16
|
+
declare function createOpenAISTTProvider(config: OpenAISTTConfig): OpenAISTTProvider;
|
|
9
17
|
|
|
10
18
|
type OpenAISTTTranscriberConfig = OpenAISTTConfig;
|
|
11
19
|
declare function createOpenAISTTTranscriber(config: OpenAISTTTranscriberConfig): STTTranscriber;
|
|
12
20
|
|
|
13
|
-
export { type OpenAISTTTranscriberConfig, createOpenAISTTTranscriber };
|
|
21
|
+
export { type OpenAISTTConfig, OpenAISTTProvider, type OpenAISTTTranscriberConfig, createOpenAISTTProvider, createOpenAISTTTranscriber };
|
package/dist/openai/index.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { STTTranscriber } from '@charivo/core';
|
|
1
|
+
import { STTProvider, STTOptions, STTTranscriber } from '@charivo/core';
|
|
2
2
|
|
|
3
3
|
interface OpenAISTTConfig {
|
|
4
4
|
apiKey: string;
|
|
@@ -6,8 +6,16 @@ interface OpenAISTTConfig {
|
|
|
6
6
|
defaultLanguage?: string;
|
|
7
7
|
dangerouslyAllowBrowser?: boolean;
|
|
8
8
|
}
|
|
9
|
+
declare class OpenAISTTProvider implements STTProvider {
|
|
10
|
+
private openai;
|
|
11
|
+
private defaultModel;
|
|
12
|
+
private defaultLanguage?;
|
|
13
|
+
constructor(config: OpenAISTTConfig);
|
|
14
|
+
transcribe(audio: Blob | ArrayBuffer, options?: STTOptions): Promise<string>;
|
|
15
|
+
}
|
|
16
|
+
declare function createOpenAISTTProvider(config: OpenAISTTConfig): OpenAISTTProvider;
|
|
9
17
|
|
|
10
18
|
type OpenAISTTTranscriberConfig = OpenAISTTConfig;
|
|
11
19
|
declare function createOpenAISTTTranscriber(config: OpenAISTTTranscriberConfig): STTTranscriber;
|
|
12
20
|
|
|
13
|
-
export { type OpenAISTTTranscriberConfig, createOpenAISTTTranscriber };
|
|
21
|
+
export { type OpenAISTTConfig, OpenAISTTProvider, type OpenAISTTTranscriberConfig, createOpenAISTTProvider, createOpenAISTTTranscriber };
|
package/dist/openai/index.js
CHANGED
|
@@ -30,6 +30,8 @@ var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: tru
|
|
|
30
30
|
// src/openai/index.ts
|
|
31
31
|
var openai_exports = {};
|
|
32
32
|
__export(openai_exports, {
|
|
33
|
+
OpenAISTTProvider: () => OpenAISTTProvider,
|
|
34
|
+
createOpenAISTTProvider: () => createOpenAISTTProvider,
|
|
33
35
|
createOpenAISTTTranscriber: () => createOpenAISTTTranscriber
|
|
34
36
|
});
|
|
35
37
|
module.exports = __toCommonJS(openai_exports);
|
|
@@ -209,5 +211,7 @@ function createOpenAISTTTranscriber(config) {
|
|
|
209
211
|
}
|
|
210
212
|
// Annotate the CommonJS export names for ESM import in node:
|
|
211
213
|
0 && (module.exports = {
|
|
214
|
+
OpenAISTTProvider,
|
|
215
|
+
createOpenAISTTProvider,
|
|
212
216
|
createOpenAISTTTranscriber
|
|
213
217
|
});
|
package/dist/openai/index.mjs
CHANGED
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
import { STTTranscriber } from '@charivo/core';
|
|
2
|
+
|
|
3
|
+
interface OpenAIRealtimeTranscriptionSessionRequest {
|
|
4
|
+
sdpOffer: string;
|
|
5
|
+
session: {
|
|
6
|
+
model: string;
|
|
7
|
+
language?: string;
|
|
8
|
+
};
|
|
9
|
+
}
|
|
10
|
+
interface OpenAIRealtimeTranscriptionBootstrap {
|
|
11
|
+
answerSdp: string;
|
|
12
|
+
}
|
|
13
|
+
type OpenAIRealtimeTranscriptionBootstrapFn = (request: OpenAIRealtimeTranscriptionSessionRequest) => Promise<OpenAIRealtimeTranscriptionBootstrap>;
|
|
14
|
+
interface OpenAIRealtimeSTTTranscriberConfig {
|
|
15
|
+
bootstrap: OpenAIRealtimeTranscriptionBootstrapFn;
|
|
16
|
+
}
|
|
17
|
+
declare function createOpenAIRealtimeSTTTranscriber(config: OpenAIRealtimeSTTTranscriberConfig): STTTranscriber;
|
|
18
|
+
|
|
19
|
+
export { type OpenAIRealtimeSTTTranscriberConfig, type OpenAIRealtimeTranscriptionBootstrap, type OpenAIRealtimeTranscriptionBootstrapFn, type OpenAIRealtimeTranscriptionSessionRequest, createOpenAIRealtimeSTTTranscriber };
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
import { STTTranscriber } from '@charivo/core';
|
|
2
|
+
|
|
3
|
+
interface OpenAIRealtimeTranscriptionSessionRequest {
|
|
4
|
+
sdpOffer: string;
|
|
5
|
+
session: {
|
|
6
|
+
model: string;
|
|
7
|
+
language?: string;
|
|
8
|
+
};
|
|
9
|
+
}
|
|
10
|
+
interface OpenAIRealtimeTranscriptionBootstrap {
|
|
11
|
+
answerSdp: string;
|
|
12
|
+
}
|
|
13
|
+
type OpenAIRealtimeTranscriptionBootstrapFn = (request: OpenAIRealtimeTranscriptionSessionRequest) => Promise<OpenAIRealtimeTranscriptionBootstrap>;
|
|
14
|
+
interface OpenAIRealtimeSTTTranscriberConfig {
|
|
15
|
+
bootstrap: OpenAIRealtimeTranscriptionBootstrapFn;
|
|
16
|
+
}
|
|
17
|
+
declare function createOpenAIRealtimeSTTTranscriber(config: OpenAIRealtimeSTTTranscriberConfig): STTTranscriber;
|
|
18
|
+
|
|
19
|
+
export { type OpenAIRealtimeSTTTranscriberConfig, type OpenAIRealtimeTranscriptionBootstrap, type OpenAIRealtimeTranscriptionBootstrapFn, type OpenAIRealtimeTranscriptionSessionRequest, createOpenAIRealtimeSTTTranscriber };
|
|
@@ -0,0 +1,581 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __defProp = Object.defineProperty;
|
|
3
|
+
var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
|
|
4
|
+
var __getOwnPropNames = Object.getOwnPropertyNames;
|
|
5
|
+
var __hasOwnProp = Object.prototype.hasOwnProperty;
|
|
6
|
+
var __export = (target, all) => {
|
|
7
|
+
for (var name in all)
|
|
8
|
+
__defProp(target, name, { get: all[name], enumerable: true });
|
|
9
|
+
};
|
|
10
|
+
var __copyProps = (to, from, except, desc) => {
|
|
11
|
+
if (from && typeof from === "object" || typeof from === "function") {
|
|
12
|
+
for (let key of __getOwnPropNames(from))
|
|
13
|
+
if (!__hasOwnProp.call(to, key) && key !== except)
|
|
14
|
+
__defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
|
|
15
|
+
}
|
|
16
|
+
return to;
|
|
17
|
+
};
|
|
18
|
+
var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
|
|
19
|
+
|
|
20
|
+
// src/openai-realtime/index.ts
|
|
21
|
+
var openai_realtime_exports = {};
|
|
22
|
+
__export(openai_realtime_exports, {
|
|
23
|
+
createOpenAIRealtimeSTTTranscriber: () => createOpenAIRealtimeSTTTranscriber
|
|
24
|
+
});
|
|
25
|
+
module.exports = __toCommonJS(openai_realtime_exports);
|
|
26
|
+
var import_core = require("@charivo/core");
|
|
27
|
+
var TRANSCRIPTION_MODEL = "gpt-realtime-whisper";
|
|
28
|
+
var OPEN_TIMEOUT_MS = 1e4;
|
|
29
|
+
var BOOTSTRAP_TIMEOUT_MS = 15e3;
|
|
30
|
+
var COMMIT_TIMEOUT_MS = 5e3;
|
|
31
|
+
var OpenAIRealtimeSTTTranscriber = class {
|
|
32
|
+
constructor(config) {
|
|
33
|
+
this.config = config;
|
|
34
|
+
}
|
|
35
|
+
pc = null;
|
|
36
|
+
dc = null;
|
|
37
|
+
mediaStream = null;
|
|
38
|
+
// Keyed by `item_id`. Map iteration is ARRIVAL order and must not be treated
|
|
39
|
+
// as transcript order; conversation order comes from `orderedItems()`.
|
|
40
|
+
items = /* @__PURE__ */ new Map();
|
|
41
|
+
// The stop target: the item named by the first `input_audio_buffer.committed`
|
|
42
|
+
// that arrives after our stop commit.
|
|
43
|
+
committedItemId = null;
|
|
44
|
+
awaitingStopCommit = false;
|
|
45
|
+
// Session epoch. Every per-session callback captures the value from its
|
|
46
|
+
// startRecording() call and ignores events once cleanup() has bumped it, so
|
|
47
|
+
// listeners still attached to a closing pc/dc cannot mutate a newer session.
|
|
48
|
+
generation = 0;
|
|
49
|
+
connecting = false;
|
|
50
|
+
recording = false;
|
|
51
|
+
stopPending = false;
|
|
52
|
+
isCleaningUp = false;
|
|
53
|
+
channelOpen = false;
|
|
54
|
+
terminalError = null;
|
|
55
|
+
setupError = null;
|
|
56
|
+
setupReject = null;
|
|
57
|
+
pendingStop = null;
|
|
58
|
+
openGate = null;
|
|
59
|
+
openTimer;
|
|
60
|
+
stopTimer;
|
|
61
|
+
partialSubscribers = [];
|
|
62
|
+
handlePageHide = () => this.cleanup();
|
|
63
|
+
/**
|
|
64
|
+
* Open the realtime transcription session and start streaming microphone audio
|
|
65
|
+
*/
|
|
66
|
+
async startRecording(options) {
|
|
67
|
+
if (this.recording || this.connecting) {
|
|
68
|
+
throw new import_core.CharivoStateError("already recording");
|
|
69
|
+
}
|
|
70
|
+
this.connecting = true;
|
|
71
|
+
this.items = /* @__PURE__ */ new Map();
|
|
72
|
+
this.committedItemId = null;
|
|
73
|
+
this.awaitingStopCommit = false;
|
|
74
|
+
this.stopPending = false;
|
|
75
|
+
this.channelOpen = false;
|
|
76
|
+
this.terminalError = null;
|
|
77
|
+
this.setupError = null;
|
|
78
|
+
this.pendingStop = null;
|
|
79
|
+
const gen = ++this.generation;
|
|
80
|
+
const setupFailure = new Promise((_, reject) => {
|
|
81
|
+
this.setupReject = reject;
|
|
82
|
+
});
|
|
83
|
+
void setupFailure.catch(() => {
|
|
84
|
+
});
|
|
85
|
+
try {
|
|
86
|
+
this.pc = new RTCPeerConnection();
|
|
87
|
+
this.pc.addEventListener("connectionstatechange", () => {
|
|
88
|
+
if (gen !== this.generation || this.isCleaningUp) {
|
|
89
|
+
return;
|
|
90
|
+
}
|
|
91
|
+
if (this.pc?.connectionState === "failed") {
|
|
92
|
+
this.failOpenOrTerminal(
|
|
93
|
+
new import_core.CharivoProviderError("connection failed")
|
|
94
|
+
);
|
|
95
|
+
}
|
|
96
|
+
});
|
|
97
|
+
this.pc.addEventListener("iceconnectionstatechange", () => {
|
|
98
|
+
if (gen !== this.generation || this.isCleaningUp) {
|
|
99
|
+
return;
|
|
100
|
+
}
|
|
101
|
+
if (this.pc?.iceConnectionState === "failed") {
|
|
102
|
+
this.failOpenOrTerminal(
|
|
103
|
+
new import_core.CharivoProviderError("connection failed")
|
|
104
|
+
);
|
|
105
|
+
}
|
|
106
|
+
});
|
|
107
|
+
const micPromise = this.acquireMic();
|
|
108
|
+
void micPromise.then(
|
|
109
|
+
(stream) => {
|
|
110
|
+
if (gen !== this.generation) {
|
|
111
|
+
stream.getTracks().forEach((track) => track.stop());
|
|
112
|
+
}
|
|
113
|
+
},
|
|
114
|
+
() => {
|
|
115
|
+
}
|
|
116
|
+
);
|
|
117
|
+
this.mediaStream = await Promise.race([micPromise, setupFailure]);
|
|
118
|
+
this.pc.addTrack(this.mediaStream.getTracks()[0], this.mediaStream);
|
|
119
|
+
this.dc = this.pc.createDataChannel("oai-events");
|
|
120
|
+
this.dc.onmessage = (event) => {
|
|
121
|
+
if (gen !== this.generation) {
|
|
122
|
+
return;
|
|
123
|
+
}
|
|
124
|
+
let payload;
|
|
125
|
+
try {
|
|
126
|
+
payload = JSON.parse(event.data);
|
|
127
|
+
} catch {
|
|
128
|
+
this.failTerminal(
|
|
129
|
+
new import_core.CharivoProviderError("malformed realtime event")
|
|
130
|
+
);
|
|
131
|
+
return;
|
|
132
|
+
}
|
|
133
|
+
if (typeof payload !== "object" || payload === null || typeof payload.type !== "string") {
|
|
134
|
+
this.failTerminal(new import_core.CharivoProviderError("invalid realtime event"));
|
|
135
|
+
return;
|
|
136
|
+
}
|
|
137
|
+
this.handleServerEvent(payload);
|
|
138
|
+
};
|
|
139
|
+
this.dc.onerror = () => {
|
|
140
|
+
if (gen !== this.generation || this.isCleaningUp) {
|
|
141
|
+
return;
|
|
142
|
+
}
|
|
143
|
+
this.failOpenOrTerminal(new import_core.CharivoProviderError("data channel error"));
|
|
144
|
+
};
|
|
145
|
+
this.dc.onclose = () => {
|
|
146
|
+
if (gen !== this.generation || this.isCleaningUp) {
|
|
147
|
+
return;
|
|
148
|
+
}
|
|
149
|
+
this.failOpenOrTerminal(
|
|
150
|
+
new import_core.CharivoProviderError("data channel closed")
|
|
151
|
+
);
|
|
152
|
+
};
|
|
153
|
+
this.dc.onopen = () => {
|
|
154
|
+
if (gen !== this.generation) {
|
|
155
|
+
return;
|
|
156
|
+
}
|
|
157
|
+
this.channelOpen = true;
|
|
158
|
+
this.resolveOpenGate();
|
|
159
|
+
};
|
|
160
|
+
const offer = await Promise.race([this.pc.createOffer(), setupFailure]);
|
|
161
|
+
await Promise.race([this.pc.setLocalDescription(offer), setupFailure]);
|
|
162
|
+
const { answerSdp } = await this.raceBootstrap(
|
|
163
|
+
this.config.bootstrap({
|
|
164
|
+
sdpOffer: offer.sdp,
|
|
165
|
+
session: { model: TRANSCRIPTION_MODEL, language: options?.language }
|
|
166
|
+
}),
|
|
167
|
+
setupFailure
|
|
168
|
+
);
|
|
169
|
+
await Promise.race([
|
|
170
|
+
this.pc.setRemoteDescription({ type: "answer", sdp: answerSdp }),
|
|
171
|
+
setupFailure
|
|
172
|
+
]);
|
|
173
|
+
await Promise.race([this.waitForChannelOpen(gen), setupFailure]);
|
|
174
|
+
if (typeof window !== "undefined") {
|
|
175
|
+
window.addEventListener("pagehide", this.handlePageHide);
|
|
176
|
+
}
|
|
177
|
+
this.recording = true;
|
|
178
|
+
} catch (error) {
|
|
179
|
+
const failure = this.setupError ?? error;
|
|
180
|
+
this.cleanup();
|
|
181
|
+
this.terminalError = null;
|
|
182
|
+
this.setupError = null;
|
|
183
|
+
throw failure;
|
|
184
|
+
} finally {
|
|
185
|
+
this.connecting = false;
|
|
186
|
+
this.setupReject = null;
|
|
187
|
+
}
|
|
188
|
+
}
|
|
189
|
+
/**
|
|
190
|
+
* Commit the buffered audio and resolve with the authoritative transcript
|
|
191
|
+
*/
|
|
192
|
+
async stopRecording() {
|
|
193
|
+
if (this.terminalError) {
|
|
194
|
+
const error = this.terminalError;
|
|
195
|
+
this.terminalError = null;
|
|
196
|
+
this.cleanup();
|
|
197
|
+
throw error;
|
|
198
|
+
}
|
|
199
|
+
if (this.stopPending) {
|
|
200
|
+
throw new import_core.CharivoStateError("stop already in progress");
|
|
201
|
+
}
|
|
202
|
+
if (this.connecting) {
|
|
203
|
+
this.cancelConnecting();
|
|
204
|
+
return "";
|
|
205
|
+
}
|
|
206
|
+
if (!this.recording) {
|
|
207
|
+
return "";
|
|
208
|
+
}
|
|
209
|
+
this.stopPending = true;
|
|
210
|
+
this.setMicEnabled(false);
|
|
211
|
+
return new Promise((resolve, reject) => {
|
|
212
|
+
this.pendingStop = { resolve, reject };
|
|
213
|
+
this.committedItemId = null;
|
|
214
|
+
this.awaitingStopCommit = true;
|
|
215
|
+
this.stopTimer = setTimeout(
|
|
216
|
+
() => this.rejectPendingStop(
|
|
217
|
+
new import_core.CharivoTimeoutError(
|
|
218
|
+
`stop timed out after ${COMMIT_TIMEOUT_MS}ms waiting for the final transcript`
|
|
219
|
+
)
|
|
220
|
+
),
|
|
221
|
+
COMMIT_TIMEOUT_MS
|
|
222
|
+
);
|
|
223
|
+
this.sendCommit();
|
|
224
|
+
this.maybeResolveStop();
|
|
225
|
+
}).then(
|
|
226
|
+
(text) => {
|
|
227
|
+
this.cleanup();
|
|
228
|
+
return text;
|
|
229
|
+
},
|
|
230
|
+
(error) => {
|
|
231
|
+
this.terminalError = null;
|
|
232
|
+
this.cleanup();
|
|
233
|
+
throw error;
|
|
234
|
+
}
|
|
235
|
+
);
|
|
236
|
+
}
|
|
237
|
+
/**
|
|
238
|
+
* Check if currently recording, including while a session is still
|
|
239
|
+
* connecting: lifecycle owners like Charivo.dispose() gate their stop call
|
|
240
|
+
* on this, so a connecting session must count as active.
|
|
241
|
+
*/
|
|
242
|
+
isRecording() {
|
|
243
|
+
return this.recording || this.connecting;
|
|
244
|
+
}
|
|
245
|
+
/**
|
|
246
|
+
* Subscribe to cumulative interim transcript snapshots
|
|
247
|
+
*/
|
|
248
|
+
onPartial(callback) {
|
|
249
|
+
this.partialSubscribers.push(callback);
|
|
250
|
+
}
|
|
251
|
+
async acquireMic() {
|
|
252
|
+
return navigator.mediaDevices.getUserMedia({
|
|
253
|
+
audio: {
|
|
254
|
+
echoCancellation: true,
|
|
255
|
+
noiseSuppression: true,
|
|
256
|
+
autoGainControl: true
|
|
257
|
+
}
|
|
258
|
+
});
|
|
259
|
+
}
|
|
260
|
+
async raceBootstrap(bootstrapPromise, setupFailure) {
|
|
261
|
+
let timer;
|
|
262
|
+
const timeout = new Promise((_, reject) => {
|
|
263
|
+
timer = setTimeout(
|
|
264
|
+
() => reject(
|
|
265
|
+
new import_core.CharivoTimeoutError(
|
|
266
|
+
`streaming STT bootstrap timed out after ${BOOTSTRAP_TIMEOUT_MS}ms`
|
|
267
|
+
)
|
|
268
|
+
),
|
|
269
|
+
BOOTSTRAP_TIMEOUT_MS
|
|
270
|
+
);
|
|
271
|
+
});
|
|
272
|
+
try {
|
|
273
|
+
return await Promise.race([bootstrapPromise, setupFailure, timeout]);
|
|
274
|
+
} finally {
|
|
275
|
+
clearTimeout(timer);
|
|
276
|
+
}
|
|
277
|
+
}
|
|
278
|
+
async waitForChannelOpen(gen) {
|
|
279
|
+
if (this.channelOpen || this.dc?.readyState === "open") {
|
|
280
|
+
return;
|
|
281
|
+
}
|
|
282
|
+
return new Promise((resolve, reject) => {
|
|
283
|
+
this.openGate = { resolve, reject, settled: false };
|
|
284
|
+
this.openTimer = setTimeout(() => {
|
|
285
|
+
if (gen !== this.generation) {
|
|
286
|
+
return;
|
|
287
|
+
}
|
|
288
|
+
this.rejectOpenGate(
|
|
289
|
+
new import_core.CharivoTimeoutError(
|
|
290
|
+
`streaming STT data channel did not open within ${OPEN_TIMEOUT_MS}ms`
|
|
291
|
+
)
|
|
292
|
+
);
|
|
293
|
+
}, OPEN_TIMEOUT_MS);
|
|
294
|
+
});
|
|
295
|
+
}
|
|
296
|
+
resolveOpenGate() {
|
|
297
|
+
clearTimeout(this.openTimer);
|
|
298
|
+
this.openTimer = void 0;
|
|
299
|
+
const gate = this.openGate;
|
|
300
|
+
this.openGate = null;
|
|
301
|
+
if (!gate || gate.settled) {
|
|
302
|
+
return;
|
|
303
|
+
}
|
|
304
|
+
gate.settled = true;
|
|
305
|
+
gate.resolve();
|
|
306
|
+
}
|
|
307
|
+
rejectOpenGate(error) {
|
|
308
|
+
clearTimeout(this.openTimer);
|
|
309
|
+
this.openTimer = void 0;
|
|
310
|
+
const gate = this.openGate;
|
|
311
|
+
this.openGate = null;
|
|
312
|
+
if (!gate || gate.settled) {
|
|
313
|
+
return;
|
|
314
|
+
}
|
|
315
|
+
gate.settled = true;
|
|
316
|
+
gate.reject(error);
|
|
317
|
+
}
|
|
318
|
+
failOpenOrTerminal(error) {
|
|
319
|
+
if (this.openGate && !this.openGate.settled) {
|
|
320
|
+
this.rejectOpenGate(error);
|
|
321
|
+
} else if (this.connecting) {
|
|
322
|
+
this.setupError ??= error;
|
|
323
|
+
this.setupReject?.(error);
|
|
324
|
+
} else {
|
|
325
|
+
this.failTerminal(error);
|
|
326
|
+
}
|
|
327
|
+
}
|
|
328
|
+
// Cancels the in-flight startRecording(). Runs cleanup() synchronously so
|
|
329
|
+
// a caller awaiting stopRecording() sees every acquired resource (mic, dc,
|
|
330
|
+
// pc, timers, pagehide listener) already released by the time it returns,
|
|
331
|
+
// then rejects whichever setup step startRecording() is currently
|
|
332
|
+
// awaiting so ITS promise settles too, instead of hanging or going live.
|
|
333
|
+
cancelConnecting() {
|
|
334
|
+
const error = new import_core.CharivoStateError(
|
|
335
|
+
"start canceled because stop was called while connecting"
|
|
336
|
+
);
|
|
337
|
+
this.setupError ??= error;
|
|
338
|
+
this.cleanup();
|
|
339
|
+
this.setupReject?.(error);
|
|
340
|
+
}
|
|
341
|
+
sendCommit() {
|
|
342
|
+
try {
|
|
343
|
+
this.dc.send(JSON.stringify({ type: "input_audio_buffer.commit" }));
|
|
344
|
+
} catch (error) {
|
|
345
|
+
this.failTerminal(
|
|
346
|
+
error instanceof Error ? error : new import_core.CharivoProviderError("failed to send commit")
|
|
347
|
+
);
|
|
348
|
+
}
|
|
349
|
+
}
|
|
350
|
+
ensureItem(id) {
|
|
351
|
+
let record = this.items.get(id);
|
|
352
|
+
if (!record) {
|
|
353
|
+
record = {
|
|
354
|
+
deltas: "",
|
|
355
|
+
transcript: null,
|
|
356
|
+
completed: false,
|
|
357
|
+
previousItemId: null
|
|
358
|
+
};
|
|
359
|
+
this.items.set(id, record);
|
|
360
|
+
}
|
|
361
|
+
return record;
|
|
362
|
+
}
|
|
363
|
+
handleServerEvent(payload) {
|
|
364
|
+
switch (payload.type) {
|
|
365
|
+
case "conversation.item.input_audio_transcription.delta": {
|
|
366
|
+
if (typeof payload.item_id !== "string" || typeof payload.delta !== "string") {
|
|
367
|
+
this.failTerminal(new import_core.CharivoProviderError("malformed delta event"));
|
|
368
|
+
return;
|
|
369
|
+
}
|
|
370
|
+
const record = this.ensureItem(payload.item_id);
|
|
371
|
+
record.deltas += payload.delta;
|
|
372
|
+
this.emitPartial();
|
|
373
|
+
return;
|
|
374
|
+
}
|
|
375
|
+
case "conversation.item.input_audio_transcription.completed": {
|
|
376
|
+
if (typeof payload.item_id !== "string" || typeof payload.transcript !== "string") {
|
|
377
|
+
this.failTerminal(
|
|
378
|
+
new import_core.CharivoProviderError("malformed completed event")
|
|
379
|
+
);
|
|
380
|
+
return;
|
|
381
|
+
}
|
|
382
|
+
const record = this.ensureItem(payload.item_id);
|
|
383
|
+
record.transcript = payload.transcript;
|
|
384
|
+
record.completed = true;
|
|
385
|
+
this.emitPartial();
|
|
386
|
+
this.maybeResolveStop();
|
|
387
|
+
return;
|
|
388
|
+
}
|
|
389
|
+
case "conversation.item.input_audio_transcription.failed": {
|
|
390
|
+
if (typeof payload.item_id !== "string") {
|
|
391
|
+
this.failTerminal(new import_core.CharivoProviderError("malformed failed event"));
|
|
392
|
+
return;
|
|
393
|
+
}
|
|
394
|
+
const message = this.errorMessage(payload.error);
|
|
395
|
+
if (message === null) {
|
|
396
|
+
this.failTerminal(new import_core.CharivoProviderError("malformed failed event"));
|
|
397
|
+
return;
|
|
398
|
+
}
|
|
399
|
+
this.failTerminal(new import_core.CharivoProviderError(message));
|
|
400
|
+
return;
|
|
401
|
+
}
|
|
402
|
+
case "input_audio_buffer.committed": {
|
|
403
|
+
if (typeof payload.item_id !== "string") {
|
|
404
|
+
this.failTerminal(
|
|
405
|
+
new import_core.CharivoProviderError("malformed committed event")
|
|
406
|
+
);
|
|
407
|
+
return;
|
|
408
|
+
}
|
|
409
|
+
if (payload.previous_item_id !== void 0 && payload.previous_item_id !== null && typeof payload.previous_item_id !== "string") {
|
|
410
|
+
this.failTerminal(
|
|
411
|
+
new import_core.CharivoProviderError("malformed committed event")
|
|
412
|
+
);
|
|
413
|
+
return;
|
|
414
|
+
}
|
|
415
|
+
const record = this.ensureItem(payload.item_id);
|
|
416
|
+
record.previousItemId = typeof payload.previous_item_id === "string" ? payload.previous_item_id : null;
|
|
417
|
+
if (this.awaitingStopCommit) {
|
|
418
|
+
this.committedItemId = payload.item_id;
|
|
419
|
+
this.awaitingStopCommit = false;
|
|
420
|
+
}
|
|
421
|
+
this.maybeResolveStop();
|
|
422
|
+
return;
|
|
423
|
+
}
|
|
424
|
+
case "error": {
|
|
425
|
+
const message = this.errorMessage(payload.error);
|
|
426
|
+
if (message === null) {
|
|
427
|
+
this.failTerminal(new import_core.CharivoProviderError("malformed error event"));
|
|
428
|
+
return;
|
|
429
|
+
}
|
|
430
|
+
this.failTerminal(new import_core.CharivoProviderError(message));
|
|
431
|
+
return;
|
|
432
|
+
}
|
|
433
|
+
default:
|
|
434
|
+
return;
|
|
435
|
+
}
|
|
436
|
+
}
|
|
437
|
+
/**
|
|
438
|
+
* Rebuild conversation order from the `previousItemId` linked-list, because
|
|
439
|
+
* cross-item `completed` events can arrive out of order.
|
|
440
|
+
*/
|
|
441
|
+
orderedItems() {
|
|
442
|
+
const ids = new Set(this.items.keys());
|
|
443
|
+
const next = /* @__PURE__ */ new Map();
|
|
444
|
+
let headId = null;
|
|
445
|
+
for (const [id, record] of this.items) {
|
|
446
|
+
if (record.previousItemId === null || !ids.has(record.previousItemId)) {
|
|
447
|
+
headId ??= id;
|
|
448
|
+
} else {
|
|
449
|
+
next.set(record.previousItemId, id);
|
|
450
|
+
}
|
|
451
|
+
}
|
|
452
|
+
const ordered = [];
|
|
453
|
+
const seen = /* @__PURE__ */ new Set();
|
|
454
|
+
for (let cur = headId; cur !== null && this.items.has(cur) && !seen.has(cur); cur = next.get(cur) ?? null) {
|
|
455
|
+
seen.add(cur);
|
|
456
|
+
ordered.push(this.items.get(cur));
|
|
457
|
+
}
|
|
458
|
+
for (const [id, record] of this.items) {
|
|
459
|
+
if (!seen.has(id)) {
|
|
460
|
+
ordered.push(record);
|
|
461
|
+
}
|
|
462
|
+
}
|
|
463
|
+
return ordered;
|
|
464
|
+
}
|
|
465
|
+
emitPartial() {
|
|
466
|
+
const snapshot = this.buildSnapshot();
|
|
467
|
+
for (const callback of this.partialSubscribers) {
|
|
468
|
+
callback(snapshot);
|
|
469
|
+
}
|
|
470
|
+
}
|
|
471
|
+
// Each delta carries its own spacing, so a plain "" join reconstructs the text.
|
|
472
|
+
buildSnapshot() {
|
|
473
|
+
return this.orderedItems().map((record) => record.transcript ?? record.deltas).join("");
|
|
474
|
+
}
|
|
475
|
+
buildFinal() {
|
|
476
|
+
return this.orderedItems().filter((record) => record.transcript !== null).map((record) => record.transcript).join("").trim();
|
|
477
|
+
}
|
|
478
|
+
failTerminal(error) {
|
|
479
|
+
if (this.terminalError) {
|
|
480
|
+
return;
|
|
481
|
+
}
|
|
482
|
+
this.terminalError = error;
|
|
483
|
+
this.cleanup();
|
|
484
|
+
}
|
|
485
|
+
errorMessage(error) {
|
|
486
|
+
return typeof error === "object" && error !== null && typeof error.message === "string" ? error.message : null;
|
|
487
|
+
}
|
|
488
|
+
maybeResolveStop() {
|
|
489
|
+
if (!this.stopPending || !this.pendingStop) {
|
|
490
|
+
return;
|
|
491
|
+
}
|
|
492
|
+
if (!this.committedItemId) {
|
|
493
|
+
return;
|
|
494
|
+
}
|
|
495
|
+
const committed = this.items.get(this.committedItemId);
|
|
496
|
+
if (!committed || !committed.completed) {
|
|
497
|
+
return;
|
|
498
|
+
}
|
|
499
|
+
for (const record of this.orderedItems()) {
|
|
500
|
+
if (record === committed) {
|
|
501
|
+
break;
|
|
502
|
+
}
|
|
503
|
+
if (!record.completed) {
|
|
504
|
+
return;
|
|
505
|
+
}
|
|
506
|
+
}
|
|
507
|
+
this.resolvePendingStop(this.buildFinal());
|
|
508
|
+
}
|
|
509
|
+
setMicEnabled(enabled) {
|
|
510
|
+
this.mediaStream?.getAudioTracks().forEach((track) => {
|
|
511
|
+
track.enabled = enabled;
|
|
512
|
+
});
|
|
513
|
+
}
|
|
514
|
+
resolvePendingStop(transcription) {
|
|
515
|
+
const pending = this.pendingStop;
|
|
516
|
+
if (!pending) {
|
|
517
|
+
return;
|
|
518
|
+
}
|
|
519
|
+
clearTimeout(this.stopTimer);
|
|
520
|
+
this.stopTimer = void 0;
|
|
521
|
+
this.pendingStop = null;
|
|
522
|
+
pending.resolve(transcription);
|
|
523
|
+
}
|
|
524
|
+
rejectPendingStop(error) {
|
|
525
|
+
const pending = this.pendingStop;
|
|
526
|
+
if (!pending) {
|
|
527
|
+
return;
|
|
528
|
+
}
|
|
529
|
+
clearTimeout(this.stopTimer);
|
|
530
|
+
this.stopTimer = void 0;
|
|
531
|
+
this.pendingStop = null;
|
|
532
|
+
pending.reject(error);
|
|
533
|
+
}
|
|
534
|
+
cleanup() {
|
|
535
|
+
if (this.isCleaningUp) {
|
|
536
|
+
return;
|
|
537
|
+
}
|
|
538
|
+
this.isCleaningUp = true;
|
|
539
|
+
try {
|
|
540
|
+
clearTimeout(this.stopTimer);
|
|
541
|
+
this.stopTimer = void 0;
|
|
542
|
+
clearTimeout(this.openTimer);
|
|
543
|
+
this.openTimer = void 0;
|
|
544
|
+
if (this.openGate && !this.openGate.settled) {
|
|
545
|
+
const gate = this.openGate;
|
|
546
|
+
gate.settled = true;
|
|
547
|
+
gate.reject(new import_core.CharivoStateError("recording ended"));
|
|
548
|
+
}
|
|
549
|
+
this.openGate = null;
|
|
550
|
+
if (this.pendingStop) {
|
|
551
|
+
this.rejectPendingStop(
|
|
552
|
+
this.terminalError ?? new import_core.CharivoStateError("recording ended")
|
|
553
|
+
);
|
|
554
|
+
}
|
|
555
|
+
this.mediaStream?.getTracks().forEach((track) => track.stop());
|
|
556
|
+
this.dc?.close();
|
|
557
|
+
this.pc?.close();
|
|
558
|
+
if (typeof window !== "undefined") {
|
|
559
|
+
window.removeEventListener("pagehide", this.handlePageHide);
|
|
560
|
+
}
|
|
561
|
+
this.pc = null;
|
|
562
|
+
this.dc = null;
|
|
563
|
+
this.mediaStream = null;
|
|
564
|
+
this.recording = false;
|
|
565
|
+
this.connecting = false;
|
|
566
|
+
this.stopPending = false;
|
|
567
|
+
this.channelOpen = false;
|
|
568
|
+
this.awaitingStopCommit = false;
|
|
569
|
+
this.generation++;
|
|
570
|
+
} finally {
|
|
571
|
+
this.isCleaningUp = false;
|
|
572
|
+
}
|
|
573
|
+
}
|
|
574
|
+
};
|
|
575
|
+
function createOpenAIRealtimeSTTTranscriber(config) {
|
|
576
|
+
return new OpenAIRealtimeSTTTranscriber(config);
|
|
577
|
+
}
|
|
578
|
+
// Annotate the CommonJS export names for ESM import in node:
|
|
579
|
+
0 && (module.exports = {
|
|
580
|
+
createOpenAIRealtimeSTTTranscriber
|
|
581
|
+
});
|
|
@@ -0,0 +1,560 @@
|
|
|
1
|
+
// src/openai-realtime/index.ts
|
|
2
|
+
import {
|
|
3
|
+
CharivoProviderError,
|
|
4
|
+
CharivoStateError,
|
|
5
|
+
CharivoTimeoutError
|
|
6
|
+
} from "@charivo/core";
|
|
7
|
+
var TRANSCRIPTION_MODEL = "gpt-realtime-whisper";
|
|
8
|
+
var OPEN_TIMEOUT_MS = 1e4;
|
|
9
|
+
var BOOTSTRAP_TIMEOUT_MS = 15e3;
|
|
10
|
+
var COMMIT_TIMEOUT_MS = 5e3;
|
|
11
|
+
var OpenAIRealtimeSTTTranscriber = class {
|
|
12
|
+
constructor(config) {
|
|
13
|
+
this.config = config;
|
|
14
|
+
}
|
|
15
|
+
pc = null;
|
|
16
|
+
dc = null;
|
|
17
|
+
mediaStream = null;
|
|
18
|
+
// Keyed by `item_id`. Map iteration is ARRIVAL order and must not be treated
|
|
19
|
+
// as transcript order; conversation order comes from `orderedItems()`.
|
|
20
|
+
items = /* @__PURE__ */ new Map();
|
|
21
|
+
// The stop target: the item named by the first `input_audio_buffer.committed`
|
|
22
|
+
// that arrives after our stop commit.
|
|
23
|
+
committedItemId = null;
|
|
24
|
+
awaitingStopCommit = false;
|
|
25
|
+
// Session epoch. Every per-session callback captures the value from its
|
|
26
|
+
// startRecording() call and ignores events once cleanup() has bumped it, so
|
|
27
|
+
// listeners still attached to a closing pc/dc cannot mutate a newer session.
|
|
28
|
+
generation = 0;
|
|
29
|
+
connecting = false;
|
|
30
|
+
recording = false;
|
|
31
|
+
stopPending = false;
|
|
32
|
+
isCleaningUp = false;
|
|
33
|
+
channelOpen = false;
|
|
34
|
+
terminalError = null;
|
|
35
|
+
setupError = null;
|
|
36
|
+
setupReject = null;
|
|
37
|
+
pendingStop = null;
|
|
38
|
+
openGate = null;
|
|
39
|
+
openTimer;
|
|
40
|
+
stopTimer;
|
|
41
|
+
partialSubscribers = [];
|
|
42
|
+
handlePageHide = () => this.cleanup();
|
|
43
|
+
/**
|
|
44
|
+
* Open the realtime transcription session and start streaming microphone audio
|
|
45
|
+
*/
|
|
46
|
+
async startRecording(options) {
|
|
47
|
+
if (this.recording || this.connecting) {
|
|
48
|
+
throw new CharivoStateError("already recording");
|
|
49
|
+
}
|
|
50
|
+
this.connecting = true;
|
|
51
|
+
this.items = /* @__PURE__ */ new Map();
|
|
52
|
+
this.committedItemId = null;
|
|
53
|
+
this.awaitingStopCommit = false;
|
|
54
|
+
this.stopPending = false;
|
|
55
|
+
this.channelOpen = false;
|
|
56
|
+
this.terminalError = null;
|
|
57
|
+
this.setupError = null;
|
|
58
|
+
this.pendingStop = null;
|
|
59
|
+
const gen = ++this.generation;
|
|
60
|
+
const setupFailure = new Promise((_, reject) => {
|
|
61
|
+
this.setupReject = reject;
|
|
62
|
+
});
|
|
63
|
+
void setupFailure.catch(() => {
|
|
64
|
+
});
|
|
65
|
+
try {
|
|
66
|
+
this.pc = new RTCPeerConnection();
|
|
67
|
+
this.pc.addEventListener("connectionstatechange", () => {
|
|
68
|
+
if (gen !== this.generation || this.isCleaningUp) {
|
|
69
|
+
return;
|
|
70
|
+
}
|
|
71
|
+
if (this.pc?.connectionState === "failed") {
|
|
72
|
+
this.failOpenOrTerminal(
|
|
73
|
+
new CharivoProviderError("connection failed")
|
|
74
|
+
);
|
|
75
|
+
}
|
|
76
|
+
});
|
|
77
|
+
this.pc.addEventListener("iceconnectionstatechange", () => {
|
|
78
|
+
if (gen !== this.generation || this.isCleaningUp) {
|
|
79
|
+
return;
|
|
80
|
+
}
|
|
81
|
+
if (this.pc?.iceConnectionState === "failed") {
|
|
82
|
+
this.failOpenOrTerminal(
|
|
83
|
+
new CharivoProviderError("connection failed")
|
|
84
|
+
);
|
|
85
|
+
}
|
|
86
|
+
});
|
|
87
|
+
const micPromise = this.acquireMic();
|
|
88
|
+
void micPromise.then(
|
|
89
|
+
(stream) => {
|
|
90
|
+
if (gen !== this.generation) {
|
|
91
|
+
stream.getTracks().forEach((track) => track.stop());
|
|
92
|
+
}
|
|
93
|
+
},
|
|
94
|
+
() => {
|
|
95
|
+
}
|
|
96
|
+
);
|
|
97
|
+
this.mediaStream = await Promise.race([micPromise, setupFailure]);
|
|
98
|
+
this.pc.addTrack(this.mediaStream.getTracks()[0], this.mediaStream);
|
|
99
|
+
this.dc = this.pc.createDataChannel("oai-events");
|
|
100
|
+
this.dc.onmessage = (event) => {
|
|
101
|
+
if (gen !== this.generation) {
|
|
102
|
+
return;
|
|
103
|
+
}
|
|
104
|
+
let payload;
|
|
105
|
+
try {
|
|
106
|
+
payload = JSON.parse(event.data);
|
|
107
|
+
} catch {
|
|
108
|
+
this.failTerminal(
|
|
109
|
+
new CharivoProviderError("malformed realtime event")
|
|
110
|
+
);
|
|
111
|
+
return;
|
|
112
|
+
}
|
|
113
|
+
if (typeof payload !== "object" || payload === null || typeof payload.type !== "string") {
|
|
114
|
+
this.failTerminal(new CharivoProviderError("invalid realtime event"));
|
|
115
|
+
return;
|
|
116
|
+
}
|
|
117
|
+
this.handleServerEvent(payload);
|
|
118
|
+
};
|
|
119
|
+
this.dc.onerror = () => {
|
|
120
|
+
if (gen !== this.generation || this.isCleaningUp) {
|
|
121
|
+
return;
|
|
122
|
+
}
|
|
123
|
+
this.failOpenOrTerminal(new CharivoProviderError("data channel error"));
|
|
124
|
+
};
|
|
125
|
+
this.dc.onclose = () => {
|
|
126
|
+
if (gen !== this.generation || this.isCleaningUp) {
|
|
127
|
+
return;
|
|
128
|
+
}
|
|
129
|
+
this.failOpenOrTerminal(
|
|
130
|
+
new CharivoProviderError("data channel closed")
|
|
131
|
+
);
|
|
132
|
+
};
|
|
133
|
+
this.dc.onopen = () => {
|
|
134
|
+
if (gen !== this.generation) {
|
|
135
|
+
return;
|
|
136
|
+
}
|
|
137
|
+
this.channelOpen = true;
|
|
138
|
+
this.resolveOpenGate();
|
|
139
|
+
};
|
|
140
|
+
const offer = await Promise.race([this.pc.createOffer(), setupFailure]);
|
|
141
|
+
await Promise.race([this.pc.setLocalDescription(offer), setupFailure]);
|
|
142
|
+
const { answerSdp } = await this.raceBootstrap(
|
|
143
|
+
this.config.bootstrap({
|
|
144
|
+
sdpOffer: offer.sdp,
|
|
145
|
+
session: { model: TRANSCRIPTION_MODEL, language: options?.language }
|
|
146
|
+
}),
|
|
147
|
+
setupFailure
|
|
148
|
+
);
|
|
149
|
+
await Promise.race([
|
|
150
|
+
this.pc.setRemoteDescription({ type: "answer", sdp: answerSdp }),
|
|
151
|
+
setupFailure
|
|
152
|
+
]);
|
|
153
|
+
await Promise.race([this.waitForChannelOpen(gen), setupFailure]);
|
|
154
|
+
if (typeof window !== "undefined") {
|
|
155
|
+
window.addEventListener("pagehide", this.handlePageHide);
|
|
156
|
+
}
|
|
157
|
+
this.recording = true;
|
|
158
|
+
} catch (error) {
|
|
159
|
+
const failure = this.setupError ?? error;
|
|
160
|
+
this.cleanup();
|
|
161
|
+
this.terminalError = null;
|
|
162
|
+
this.setupError = null;
|
|
163
|
+
throw failure;
|
|
164
|
+
} finally {
|
|
165
|
+
this.connecting = false;
|
|
166
|
+
this.setupReject = null;
|
|
167
|
+
}
|
|
168
|
+
}
|
|
169
|
+
/**
|
|
170
|
+
* Commit the buffered audio and resolve with the authoritative transcript
|
|
171
|
+
*/
|
|
172
|
+
async stopRecording() {
|
|
173
|
+
if (this.terminalError) {
|
|
174
|
+
const error = this.terminalError;
|
|
175
|
+
this.terminalError = null;
|
|
176
|
+
this.cleanup();
|
|
177
|
+
throw error;
|
|
178
|
+
}
|
|
179
|
+
if (this.stopPending) {
|
|
180
|
+
throw new CharivoStateError("stop already in progress");
|
|
181
|
+
}
|
|
182
|
+
if (this.connecting) {
|
|
183
|
+
this.cancelConnecting();
|
|
184
|
+
return "";
|
|
185
|
+
}
|
|
186
|
+
if (!this.recording) {
|
|
187
|
+
return "";
|
|
188
|
+
}
|
|
189
|
+
this.stopPending = true;
|
|
190
|
+
this.setMicEnabled(false);
|
|
191
|
+
return new Promise((resolve, reject) => {
|
|
192
|
+
this.pendingStop = { resolve, reject };
|
|
193
|
+
this.committedItemId = null;
|
|
194
|
+
this.awaitingStopCommit = true;
|
|
195
|
+
this.stopTimer = setTimeout(
|
|
196
|
+
() => this.rejectPendingStop(
|
|
197
|
+
new CharivoTimeoutError(
|
|
198
|
+
`stop timed out after ${COMMIT_TIMEOUT_MS}ms waiting for the final transcript`
|
|
199
|
+
)
|
|
200
|
+
),
|
|
201
|
+
COMMIT_TIMEOUT_MS
|
|
202
|
+
);
|
|
203
|
+
this.sendCommit();
|
|
204
|
+
this.maybeResolveStop();
|
|
205
|
+
}).then(
|
|
206
|
+
(text) => {
|
|
207
|
+
this.cleanup();
|
|
208
|
+
return text;
|
|
209
|
+
},
|
|
210
|
+
(error) => {
|
|
211
|
+
this.terminalError = null;
|
|
212
|
+
this.cleanup();
|
|
213
|
+
throw error;
|
|
214
|
+
}
|
|
215
|
+
);
|
|
216
|
+
}
|
|
217
|
+
/**
|
|
218
|
+
* Check if currently recording, including while a session is still
|
|
219
|
+
* connecting: lifecycle owners like Charivo.dispose() gate their stop call
|
|
220
|
+
* on this, so a connecting session must count as active.
|
|
221
|
+
*/
|
|
222
|
+
isRecording() {
|
|
223
|
+
return this.recording || this.connecting;
|
|
224
|
+
}
|
|
225
|
+
/**
|
|
226
|
+
* Subscribe to cumulative interim transcript snapshots
|
|
227
|
+
*/
|
|
228
|
+
onPartial(callback) {
|
|
229
|
+
this.partialSubscribers.push(callback);
|
|
230
|
+
}
|
|
231
|
+
async acquireMic() {
|
|
232
|
+
return navigator.mediaDevices.getUserMedia({
|
|
233
|
+
audio: {
|
|
234
|
+
echoCancellation: true,
|
|
235
|
+
noiseSuppression: true,
|
|
236
|
+
autoGainControl: true
|
|
237
|
+
}
|
|
238
|
+
});
|
|
239
|
+
}
|
|
240
|
+
async raceBootstrap(bootstrapPromise, setupFailure) {
|
|
241
|
+
let timer;
|
|
242
|
+
const timeout = new Promise((_, reject) => {
|
|
243
|
+
timer = setTimeout(
|
|
244
|
+
() => reject(
|
|
245
|
+
new CharivoTimeoutError(
|
|
246
|
+
`streaming STT bootstrap timed out after ${BOOTSTRAP_TIMEOUT_MS}ms`
|
|
247
|
+
)
|
|
248
|
+
),
|
|
249
|
+
BOOTSTRAP_TIMEOUT_MS
|
|
250
|
+
);
|
|
251
|
+
});
|
|
252
|
+
try {
|
|
253
|
+
return await Promise.race([bootstrapPromise, setupFailure, timeout]);
|
|
254
|
+
} finally {
|
|
255
|
+
clearTimeout(timer);
|
|
256
|
+
}
|
|
257
|
+
}
|
|
258
|
+
async waitForChannelOpen(gen) {
|
|
259
|
+
if (this.channelOpen || this.dc?.readyState === "open") {
|
|
260
|
+
return;
|
|
261
|
+
}
|
|
262
|
+
return new Promise((resolve, reject) => {
|
|
263
|
+
this.openGate = { resolve, reject, settled: false };
|
|
264
|
+
this.openTimer = setTimeout(() => {
|
|
265
|
+
if (gen !== this.generation) {
|
|
266
|
+
return;
|
|
267
|
+
}
|
|
268
|
+
this.rejectOpenGate(
|
|
269
|
+
new CharivoTimeoutError(
|
|
270
|
+
`streaming STT data channel did not open within ${OPEN_TIMEOUT_MS}ms`
|
|
271
|
+
)
|
|
272
|
+
);
|
|
273
|
+
}, OPEN_TIMEOUT_MS);
|
|
274
|
+
});
|
|
275
|
+
}
|
|
276
|
+
resolveOpenGate() {
|
|
277
|
+
clearTimeout(this.openTimer);
|
|
278
|
+
this.openTimer = void 0;
|
|
279
|
+
const gate = this.openGate;
|
|
280
|
+
this.openGate = null;
|
|
281
|
+
if (!gate || gate.settled) {
|
|
282
|
+
return;
|
|
283
|
+
}
|
|
284
|
+
gate.settled = true;
|
|
285
|
+
gate.resolve();
|
|
286
|
+
}
|
|
287
|
+
rejectOpenGate(error) {
|
|
288
|
+
clearTimeout(this.openTimer);
|
|
289
|
+
this.openTimer = void 0;
|
|
290
|
+
const gate = this.openGate;
|
|
291
|
+
this.openGate = null;
|
|
292
|
+
if (!gate || gate.settled) {
|
|
293
|
+
return;
|
|
294
|
+
}
|
|
295
|
+
gate.settled = true;
|
|
296
|
+
gate.reject(error);
|
|
297
|
+
}
|
|
298
|
+
failOpenOrTerminal(error) {
|
|
299
|
+
if (this.openGate && !this.openGate.settled) {
|
|
300
|
+
this.rejectOpenGate(error);
|
|
301
|
+
} else if (this.connecting) {
|
|
302
|
+
this.setupError ??= error;
|
|
303
|
+
this.setupReject?.(error);
|
|
304
|
+
} else {
|
|
305
|
+
this.failTerminal(error);
|
|
306
|
+
}
|
|
307
|
+
}
|
|
308
|
+
// Cancels the in-flight startRecording(). Runs cleanup() synchronously so
|
|
309
|
+
// a caller awaiting stopRecording() sees every acquired resource (mic, dc,
|
|
310
|
+
// pc, timers, pagehide listener) already released by the time it returns,
|
|
311
|
+
// then rejects whichever setup step startRecording() is currently
|
|
312
|
+
// awaiting so ITS promise settles too, instead of hanging or going live.
|
|
313
|
+
cancelConnecting() {
|
|
314
|
+
const error = new CharivoStateError(
|
|
315
|
+
"start canceled because stop was called while connecting"
|
|
316
|
+
);
|
|
317
|
+
this.setupError ??= error;
|
|
318
|
+
this.cleanup();
|
|
319
|
+
this.setupReject?.(error);
|
|
320
|
+
}
|
|
321
|
+
sendCommit() {
|
|
322
|
+
try {
|
|
323
|
+
this.dc.send(JSON.stringify({ type: "input_audio_buffer.commit" }));
|
|
324
|
+
} catch (error) {
|
|
325
|
+
this.failTerminal(
|
|
326
|
+
error instanceof Error ? error : new CharivoProviderError("failed to send commit")
|
|
327
|
+
);
|
|
328
|
+
}
|
|
329
|
+
}
|
|
330
|
+
ensureItem(id) {
|
|
331
|
+
let record = this.items.get(id);
|
|
332
|
+
if (!record) {
|
|
333
|
+
record = {
|
|
334
|
+
deltas: "",
|
|
335
|
+
transcript: null,
|
|
336
|
+
completed: false,
|
|
337
|
+
previousItemId: null
|
|
338
|
+
};
|
|
339
|
+
this.items.set(id, record);
|
|
340
|
+
}
|
|
341
|
+
return record;
|
|
342
|
+
}
|
|
343
|
+
handleServerEvent(payload) {
|
|
344
|
+
switch (payload.type) {
|
|
345
|
+
case "conversation.item.input_audio_transcription.delta": {
|
|
346
|
+
if (typeof payload.item_id !== "string" || typeof payload.delta !== "string") {
|
|
347
|
+
this.failTerminal(new CharivoProviderError("malformed delta event"));
|
|
348
|
+
return;
|
|
349
|
+
}
|
|
350
|
+
const record = this.ensureItem(payload.item_id);
|
|
351
|
+
record.deltas += payload.delta;
|
|
352
|
+
this.emitPartial();
|
|
353
|
+
return;
|
|
354
|
+
}
|
|
355
|
+
case "conversation.item.input_audio_transcription.completed": {
|
|
356
|
+
if (typeof payload.item_id !== "string" || typeof payload.transcript !== "string") {
|
|
357
|
+
this.failTerminal(
|
|
358
|
+
new CharivoProviderError("malformed completed event")
|
|
359
|
+
);
|
|
360
|
+
return;
|
|
361
|
+
}
|
|
362
|
+
const record = this.ensureItem(payload.item_id);
|
|
363
|
+
record.transcript = payload.transcript;
|
|
364
|
+
record.completed = true;
|
|
365
|
+
this.emitPartial();
|
|
366
|
+
this.maybeResolveStop();
|
|
367
|
+
return;
|
|
368
|
+
}
|
|
369
|
+
case "conversation.item.input_audio_transcription.failed": {
|
|
370
|
+
if (typeof payload.item_id !== "string") {
|
|
371
|
+
this.failTerminal(new CharivoProviderError("malformed failed event"));
|
|
372
|
+
return;
|
|
373
|
+
}
|
|
374
|
+
const message = this.errorMessage(payload.error);
|
|
375
|
+
if (message === null) {
|
|
376
|
+
this.failTerminal(new CharivoProviderError("malformed failed event"));
|
|
377
|
+
return;
|
|
378
|
+
}
|
|
379
|
+
this.failTerminal(new CharivoProviderError(message));
|
|
380
|
+
return;
|
|
381
|
+
}
|
|
382
|
+
case "input_audio_buffer.committed": {
|
|
383
|
+
if (typeof payload.item_id !== "string") {
|
|
384
|
+
this.failTerminal(
|
|
385
|
+
new CharivoProviderError("malformed committed event")
|
|
386
|
+
);
|
|
387
|
+
return;
|
|
388
|
+
}
|
|
389
|
+
if (payload.previous_item_id !== void 0 && payload.previous_item_id !== null && typeof payload.previous_item_id !== "string") {
|
|
390
|
+
this.failTerminal(
|
|
391
|
+
new CharivoProviderError("malformed committed event")
|
|
392
|
+
);
|
|
393
|
+
return;
|
|
394
|
+
}
|
|
395
|
+
const record = this.ensureItem(payload.item_id);
|
|
396
|
+
record.previousItemId = typeof payload.previous_item_id === "string" ? payload.previous_item_id : null;
|
|
397
|
+
if (this.awaitingStopCommit) {
|
|
398
|
+
this.committedItemId = payload.item_id;
|
|
399
|
+
this.awaitingStopCommit = false;
|
|
400
|
+
}
|
|
401
|
+
this.maybeResolveStop();
|
|
402
|
+
return;
|
|
403
|
+
}
|
|
404
|
+
case "error": {
|
|
405
|
+
const message = this.errorMessage(payload.error);
|
|
406
|
+
if (message === null) {
|
|
407
|
+
this.failTerminal(new CharivoProviderError("malformed error event"));
|
|
408
|
+
return;
|
|
409
|
+
}
|
|
410
|
+
this.failTerminal(new CharivoProviderError(message));
|
|
411
|
+
return;
|
|
412
|
+
}
|
|
413
|
+
default:
|
|
414
|
+
return;
|
|
415
|
+
}
|
|
416
|
+
}
|
|
417
|
+
/**
|
|
418
|
+
* Rebuild conversation order from the `previousItemId` linked-list, because
|
|
419
|
+
* cross-item `completed` events can arrive out of order.
|
|
420
|
+
*/
|
|
421
|
+
orderedItems() {
|
|
422
|
+
const ids = new Set(this.items.keys());
|
|
423
|
+
const next = /* @__PURE__ */ new Map();
|
|
424
|
+
let headId = null;
|
|
425
|
+
for (const [id, record] of this.items) {
|
|
426
|
+
if (record.previousItemId === null || !ids.has(record.previousItemId)) {
|
|
427
|
+
headId ??= id;
|
|
428
|
+
} else {
|
|
429
|
+
next.set(record.previousItemId, id);
|
|
430
|
+
}
|
|
431
|
+
}
|
|
432
|
+
const ordered = [];
|
|
433
|
+
const seen = /* @__PURE__ */ new Set();
|
|
434
|
+
for (let cur = headId; cur !== null && this.items.has(cur) && !seen.has(cur); cur = next.get(cur) ?? null) {
|
|
435
|
+
seen.add(cur);
|
|
436
|
+
ordered.push(this.items.get(cur));
|
|
437
|
+
}
|
|
438
|
+
for (const [id, record] of this.items) {
|
|
439
|
+
if (!seen.has(id)) {
|
|
440
|
+
ordered.push(record);
|
|
441
|
+
}
|
|
442
|
+
}
|
|
443
|
+
return ordered;
|
|
444
|
+
}
|
|
445
|
+
emitPartial() {
|
|
446
|
+
const snapshot = this.buildSnapshot();
|
|
447
|
+
for (const callback of this.partialSubscribers) {
|
|
448
|
+
callback(snapshot);
|
|
449
|
+
}
|
|
450
|
+
}
|
|
451
|
+
// Each delta carries its own spacing, so a plain "" join reconstructs the text.
|
|
452
|
+
buildSnapshot() {
|
|
453
|
+
return this.orderedItems().map((record) => record.transcript ?? record.deltas).join("");
|
|
454
|
+
}
|
|
455
|
+
buildFinal() {
|
|
456
|
+
return this.orderedItems().filter((record) => record.transcript !== null).map((record) => record.transcript).join("").trim();
|
|
457
|
+
}
|
|
458
|
+
failTerminal(error) {
|
|
459
|
+
if (this.terminalError) {
|
|
460
|
+
return;
|
|
461
|
+
}
|
|
462
|
+
this.terminalError = error;
|
|
463
|
+
this.cleanup();
|
|
464
|
+
}
|
|
465
|
+
errorMessage(error) {
|
|
466
|
+
return typeof error === "object" && error !== null && typeof error.message === "string" ? error.message : null;
|
|
467
|
+
}
|
|
468
|
+
maybeResolveStop() {
|
|
469
|
+
if (!this.stopPending || !this.pendingStop) {
|
|
470
|
+
return;
|
|
471
|
+
}
|
|
472
|
+
if (!this.committedItemId) {
|
|
473
|
+
return;
|
|
474
|
+
}
|
|
475
|
+
const committed = this.items.get(this.committedItemId);
|
|
476
|
+
if (!committed || !committed.completed) {
|
|
477
|
+
return;
|
|
478
|
+
}
|
|
479
|
+
for (const record of this.orderedItems()) {
|
|
480
|
+
if (record === committed) {
|
|
481
|
+
break;
|
|
482
|
+
}
|
|
483
|
+
if (!record.completed) {
|
|
484
|
+
return;
|
|
485
|
+
}
|
|
486
|
+
}
|
|
487
|
+
this.resolvePendingStop(this.buildFinal());
|
|
488
|
+
}
|
|
489
|
+
setMicEnabled(enabled) {
|
|
490
|
+
this.mediaStream?.getAudioTracks().forEach((track) => {
|
|
491
|
+
track.enabled = enabled;
|
|
492
|
+
});
|
|
493
|
+
}
|
|
494
|
+
resolvePendingStop(transcription) {
|
|
495
|
+
const pending = this.pendingStop;
|
|
496
|
+
if (!pending) {
|
|
497
|
+
return;
|
|
498
|
+
}
|
|
499
|
+
clearTimeout(this.stopTimer);
|
|
500
|
+
this.stopTimer = void 0;
|
|
501
|
+
this.pendingStop = null;
|
|
502
|
+
pending.resolve(transcription);
|
|
503
|
+
}
|
|
504
|
+
rejectPendingStop(error) {
|
|
505
|
+
const pending = this.pendingStop;
|
|
506
|
+
if (!pending) {
|
|
507
|
+
return;
|
|
508
|
+
}
|
|
509
|
+
clearTimeout(this.stopTimer);
|
|
510
|
+
this.stopTimer = void 0;
|
|
511
|
+
this.pendingStop = null;
|
|
512
|
+
pending.reject(error);
|
|
513
|
+
}
|
|
514
|
+
cleanup() {
|
|
515
|
+
if (this.isCleaningUp) {
|
|
516
|
+
return;
|
|
517
|
+
}
|
|
518
|
+
this.isCleaningUp = true;
|
|
519
|
+
try {
|
|
520
|
+
clearTimeout(this.stopTimer);
|
|
521
|
+
this.stopTimer = void 0;
|
|
522
|
+
clearTimeout(this.openTimer);
|
|
523
|
+
this.openTimer = void 0;
|
|
524
|
+
if (this.openGate && !this.openGate.settled) {
|
|
525
|
+
const gate = this.openGate;
|
|
526
|
+
gate.settled = true;
|
|
527
|
+
gate.reject(new CharivoStateError("recording ended"));
|
|
528
|
+
}
|
|
529
|
+
this.openGate = null;
|
|
530
|
+
if (this.pendingStop) {
|
|
531
|
+
this.rejectPendingStop(
|
|
532
|
+
this.terminalError ?? new CharivoStateError("recording ended")
|
|
533
|
+
);
|
|
534
|
+
}
|
|
535
|
+
this.mediaStream?.getTracks().forEach((track) => track.stop());
|
|
536
|
+
this.dc?.close();
|
|
537
|
+
this.pc?.close();
|
|
538
|
+
if (typeof window !== "undefined") {
|
|
539
|
+
window.removeEventListener("pagehide", this.handlePageHide);
|
|
540
|
+
}
|
|
541
|
+
this.pc = null;
|
|
542
|
+
this.dc = null;
|
|
543
|
+
this.mediaStream = null;
|
|
544
|
+
this.recording = false;
|
|
545
|
+
this.connecting = false;
|
|
546
|
+
this.stopPending = false;
|
|
547
|
+
this.channelOpen = false;
|
|
548
|
+
this.awaitingStopCommit = false;
|
|
549
|
+
this.generation++;
|
|
550
|
+
} finally {
|
|
551
|
+
this.isCleaningUp = false;
|
|
552
|
+
}
|
|
553
|
+
}
|
|
554
|
+
};
|
|
555
|
+
function createOpenAIRealtimeSTTTranscriber(config) {
|
|
556
|
+
return new OpenAIRealtimeSTTTranscriber(config);
|
|
557
|
+
}
|
|
558
|
+
export {
|
|
559
|
+
createOpenAIRealtimeSTTTranscriber
|
|
560
|
+
};
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@charivo/stt",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.6.0",
|
|
4
4
|
"description": "STT manager and browser adapters for Charivo",
|
|
5
5
|
"main": "dist/index.js",
|
|
6
6
|
"module": "dist/index.mjs",
|
|
@@ -21,6 +21,11 @@
|
|
|
21
21
|
"import": "./dist/openai/index.mjs",
|
|
22
22
|
"require": "./dist/openai/index.js"
|
|
23
23
|
},
|
|
24
|
+
"./openai-realtime": {
|
|
25
|
+
"types": "./dist/openai-realtime/index.d.ts",
|
|
26
|
+
"import": "./dist/openai-realtime/index.mjs",
|
|
27
|
+
"require": "./dist/openai-realtime/index.js"
|
|
28
|
+
},
|
|
24
29
|
"./web": {
|
|
25
30
|
"types": "./dist/web/index.d.ts",
|
|
26
31
|
"import": "./dist/web/index.mjs",
|
|
@@ -36,7 +41,7 @@
|
|
|
36
41
|
],
|
|
37
42
|
"dependencies": {
|
|
38
43
|
"openai": "^4.47.1",
|
|
39
|
-
"@charivo/core": "0.
|
|
44
|
+
"@charivo/core": "0.15.0"
|
|
40
45
|
},
|
|
41
46
|
"devDependencies": {
|
|
42
47
|
"tsup": "^8.5.0",
|
|
@@ -66,7 +71,8 @@
|
|
|
66
71
|
"scripts": {
|
|
67
72
|
"build": "tsup",
|
|
68
73
|
"dev": "tsup --watch",
|
|
69
|
-
"test": "vitest",
|
|
74
|
+
"test": "vitest run --root ../.. packages/stt",
|
|
75
|
+
"test:watch": "vitest --root ../.. packages/stt",
|
|
70
76
|
"typecheck": "tsc --noEmit",
|
|
71
77
|
"clean": "rm -rf dist"
|
|
72
78
|
}
|