@phren/cli 0.3.18 → 0.3.20
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bridge/agent-hooks.d.ts +16 -1
- package/dist/bridge/agent-hooks.js +56 -4
- package/dist/bridge/changes.d.ts +4 -0
- package/dist/bridge/changes.js +25 -2
- package/dist/bridge/command.js +24 -6
- package/dist/bridge/dispatch-returns.d.ts +11 -0
- package/dist/bridge/dispatch-returns.js +47 -7
- package/dist/bridge/dispatch.d.ts +19 -0
- package/dist/bridge/dispatch.js +164 -5
- package/dist/bridge/harnesses.d.ts +1 -1
- package/dist/bridge/harnesses.js +9 -2
- package/dist/bridge/health.d.ts +3 -2
- package/dist/bridge/health.js +4 -3
- package/dist/bridge/hook-fast.d.ts +13 -1
- package/dist/bridge/hook-fast.js +17 -2
- package/dist/bridge/models.d.ts +11 -1
- package/dist/bridge/models.js +47 -2
- package/dist/bridge/server-launch.d.ts +20 -0
- package/dist/bridge/server-launch.js +77 -3
- package/dist/bridge/server-routes.d.ts +4 -0
- package/dist/bridge/server-routes.js +2 -2
- package/dist/bridge/server.js +6 -1
- package/dist/bridge/speech-live.d.ts +45 -0
- package/dist/bridge/speech-live.js +413 -0
- package/dist/bridge/speech-transcribe.d.ts +5 -2
- package/dist/bridge/speech-transcribe.js +8 -5
- package/dist/bridge/speech-voice.d.ts +22 -1
- package/dist/bridge/speech-voice.js +20 -2
- package/dist/bridge/speech.d.ts +24 -4
- package/dist/bridge/speech.js +115 -14
- package/dist/bridge/stalls.d.ts +9 -2
- package/dist/bridge/stalls.js +20 -4
- package/dist/bridge/terminal-herdr.d.ts +7 -0
- package/dist/bridge/terminal-herdr.js +81 -7
- package/dist/bridge/terminal-tmux.js +2 -2
- package/dist/bridge/terminal.d.ts +3 -0
- package/dist/bridge/terminal.js +1 -0
- package/dist/bridge/transcript-preview.d.ts +11 -1
- package/dist/bridge/transcript-preview.js +37 -11
- package/dist/bridge/transcripts.d.ts +2 -1
- package/dist/bridge/transcripts.js +7 -3
- package/dist/bridge/transport.js +4 -3
- package/dist/bridge/turn-records.d.ts +6 -3
- package/dist/bridge/turn-records.js +9 -4
- package/dist/bridge/usage.d.ts +1 -1
- package/dist/bridge/usage.js +4 -3
- package/dist/bridge/worker-close.js +2 -1
- package/dist/bridge/worker-unfinished.d.ts +37 -0
- package/dist/bridge/worker-unfinished.js +145 -0
- package/dist/bridge-hook.mjs +2171 -1230
- package/dist/cli-registry.js +2 -1
- package/dist/modules/registry.js +2 -2
- package/package.json +1 -1
|
@@ -108,6 +108,8 @@ export declare const capabilities: {
|
|
|
108
108
|
overviewStream: boolean;
|
|
109
109
|
speech: boolean;
|
|
110
110
|
speechTimestamps: boolean;
|
|
111
|
+
speechTimestampStream: boolean;
|
|
112
|
+
speechLive: boolean;
|
|
111
113
|
speechVoices: boolean;
|
|
112
114
|
speechFormats: ("mp3_44100_128" | "mp3_44100_192" | "pcm_24000" | "pcm_44100")[];
|
|
113
115
|
transcribe: boolean;
|
|
@@ -117,6 +119,8 @@ export declare const capabilities: {
|
|
|
117
119
|
resources: boolean;
|
|
118
120
|
sudo: boolean;
|
|
119
121
|
sudoOutcome: boolean;
|
|
122
|
+
previewDeltas: boolean;
|
|
123
|
+
quickChat: boolean;
|
|
120
124
|
};
|
|
121
125
|
export declare function capabilitiesForModules(snapshot: ModuleSnapshot): Record<string, unknown>;
|
|
122
126
|
export declare function requireRoute(snapshot: ModuleSnapshot, method: string, route: string): void;
|
|
@@ -68,7 +68,7 @@ async function childActivity(source, session) {
|
|
|
68
68
|
export const capabilities = { transcript: true, progress: true, images: true, prompt: true, stop: true,
|
|
69
69
|
terminal: "ssh-pty", shell: "ssh-pty", herdr: true, sessionRename: true, diff: true, webServers: true, webPreview: "ssh-exec", activity: true,
|
|
70
70
|
approvals: true, questions: false, accountUsage: true, providers: ["codex", "claude", "copilot", "opencode"],
|
|
71
|
-
files: true, repositoryFiles: true, subagents: true, sideQuestions: true, dispatch: true, conductorSets: true, ownerInbox: true, workerReports: true, queuedHandOff: true, approvalPush: "direct-apns", simulators: process.platform === "darwin", code: true, overviewStream: true, speech: true, speechTimestamps: true, speechVoices: true, speechFormats: [...SPEECH_FORMATS], transcribe: true, memoryStore: true, promptOnce: true, promptStatus: true, resources: true, sudo: true, sudoOutcome: true };
|
|
71
|
+
files: true, repositoryFiles: true, subagents: true, sideQuestions: true, dispatch: true, conductorSets: true, ownerInbox: true, workerReports: true, queuedHandOff: true, approvalPush: "direct-apns", simulators: process.platform === "darwin", code: true, overviewStream: true, speech: true, speechTimestamps: true, speechTimestampStream: true, speechLive: true, speechVoices: true, speechFormats: [...SPEECH_FORMATS], transcribe: true, memoryStore: true, promptOnce: true, promptStatus: true, resources: true, sudo: true, sudoOutcome: true, previewDeltas: true, quickChat: true };
|
|
72
72
|
export function capabilitiesForModules(snapshot) {
|
|
73
73
|
const allowed = new Set(snapshot.modules.flatMap(module => module.capabilities));
|
|
74
74
|
const result = Object.fromEntries(Object.entries(capabilities).filter(([name]) => allowed.has(name)));
|
|
@@ -183,7 +183,7 @@ export function workspacesReader(ctx) {
|
|
|
183
183
|
markBackground(tab, background);
|
|
184
184
|
const target = targetSchema.safeParse(tab.target);
|
|
185
185
|
if (target.success && agents.length === 1)
|
|
186
|
-
Object.assign(tab, await sessionStalls.observe(target.data, { ...agents[0], agent_status: tab.agentStatus }));
|
|
186
|
+
Object.assign(tab, await sessionStalls.observe(target.data, { ...agents[0], agent_status: tab.agentStatus }, async () => background));
|
|
187
187
|
tab.title = title;
|
|
188
188
|
}));
|
|
189
189
|
let nextTab = 0;
|
package/dist/bridge/server.js
CHANGED
|
@@ -16,6 +16,7 @@ import { createServer } from "node:http";
|
|
|
16
16
|
import { cpus, hostname, loadavg } from "node:os";
|
|
17
17
|
import path from "node:path";
|
|
18
18
|
import { WebSocketServer } from "ws";
|
|
19
|
+
import { relayLiveSpeech } from "./speech-live.js";
|
|
19
20
|
import { relayTranscription } from "./speech-transcribe.js";
|
|
20
21
|
import { ActivityJournal } from "./activity.js";
|
|
21
22
|
import { countTick } from "./metrics.js";
|
|
@@ -217,7 +218,7 @@ export async function serve(version, options = {}) {
|
|
|
217
218
|
http.on("upgrade", (request, socket, head) => {
|
|
218
219
|
try {
|
|
219
220
|
const url = new URL(request.url || "/", "http://phren.local");
|
|
220
|
-
if (!["/v1/transcripts", "/v1/status", "/v1/overview", "/v1/speech/transcribe"].includes(url.pathname) || url.origin !== "http://phren.local") {
|
|
221
|
+
if (!["/v1/transcripts", "/v1/status", "/v1/overview", "/v1/speech/transcribe", "/v1/speech/live"].includes(url.pathname) || url.origin !== "http://phren.local") {
|
|
221
222
|
socket.destroy();
|
|
222
223
|
return;
|
|
223
224
|
}
|
|
@@ -239,6 +240,10 @@ export async function serve(version, options = {}) {
|
|
|
239
240
|
void relayTranscription(client, url.searchParams).catch(() => client.close(1011, "Transcription unavailable"));
|
|
240
241
|
return;
|
|
241
242
|
}
|
|
243
|
+
if (url.pathname === "/v1/speech/live") {
|
|
244
|
+
void relayLiveSpeech(client, url.searchParams).catch(() => client.close(1011, "Spoken replies unavailable"));
|
|
245
|
+
return;
|
|
246
|
+
}
|
|
242
247
|
void stream(client, url).catch(() => client.close(1011, "Conversation unavailable; refresh"));
|
|
243
248
|
});
|
|
244
249
|
}
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
import { type SpeechState } from "./speech.js";
|
|
2
|
+
export interface LiveSocket {
|
|
3
|
+
readonly readyState: number;
|
|
4
|
+
/** Bytes queued to send; the phone's socket has it, ElevenLabs' need not. */
|
|
5
|
+
readonly bufferedAmount?: number;
|
|
6
|
+
send(data: string): void;
|
|
7
|
+
close(code?: number, reason?: string): void;
|
|
8
|
+
on(event: "message", listener: (data: Buffer, isBinary: boolean) => void): unknown;
|
|
9
|
+
on(event: "open", listener: () => void): unknown;
|
|
10
|
+
on(event: "close", listener: (code?: number, reason?: Buffer) => void): unknown;
|
|
11
|
+
on(event: "error", listener: (error: Error) => void): unknown;
|
|
12
|
+
}
|
|
13
|
+
/** ElevenLabs' side: a socket that also reports a refused handshake with its
|
|
14
|
+
* HTTP status and ElevenLabs' `detail.status`. */
|
|
15
|
+
export interface LiveUpstream extends LiveSocket {
|
|
16
|
+
on(event: "message", listener: (data: Buffer, isBinary: boolean) => void): unknown;
|
|
17
|
+
on(event: "open", listener: () => void): unknown;
|
|
18
|
+
on(event: "close", listener: (code?: number, reason?: Buffer) => void): unknown;
|
|
19
|
+
on(event: "error", listener: (error: Error) => void): unknown;
|
|
20
|
+
on(event: "refused", listener: (status: number, code: string) => void): unknown;
|
|
21
|
+
}
|
|
22
|
+
export interface LiveSpeechOptions {
|
|
23
|
+
key?: () => Promise<string | undefined>;
|
|
24
|
+
connect?: (url: string, key: string) => LiveUpstream;
|
|
25
|
+
/** Defaults to this computer's model and region settings. */
|
|
26
|
+
model?: string;
|
|
27
|
+
origin?: string;
|
|
28
|
+
state?: SpeechState;
|
|
29
|
+
maxSessionMs?: number;
|
|
30
|
+
handshakeMs?: number;
|
|
31
|
+
firstAudioMs?: number;
|
|
32
|
+
}
|
|
33
|
+
/** Text-to-dialogue serves v3 and v4; text-to-speech stream-input the rest. */
|
|
34
|
+
export declare function liveEndpoint(model: string): "dialogue" | "speech";
|
|
35
|
+
/** Markdown arriving a piece at a time, released a sentence or line at a time
|
|
36
|
+
* as the words speakableText leaves of it. A code block is skipped until it
|
|
37
|
+
* closes; the unfinished sentence waits for more text or `end`. */
|
|
38
|
+
export declare class SpeakableStream {
|
|
39
|
+
private pending;
|
|
40
|
+
private inFence;
|
|
41
|
+
push(text: string): string;
|
|
42
|
+
end(): string;
|
|
43
|
+
private speak;
|
|
44
|
+
}
|
|
45
|
+
export declare function relayLiveSpeech(client: LiveSocket, query: URLSearchParams, options?: LiveSpeechOptions): Promise<void>;
|
|
@@ -0,0 +1,413 @@
|
|
|
1
|
+
import { WebSocket } from "ws";
|
|
2
|
+
import { SPEECH_FORMAT_INFO, SPEECH_FORMATS, sharedSpeechState, speakableText } from "./speech.js";
|
|
3
|
+
import { readSpeechKey } from "./speech-key.js";
|
|
4
|
+
import { FALLBACK_SPEECH_MODEL, resolveSpeechModel, resolveSpeechRegion, resolveSpeechVoice, voiceId } from "./speech-voice.js";
|
|
5
|
+
/** WS /v1/speech/live: a reply voiced while it is still being written. The
|
|
6
|
+
* phone (or, later, the Hook from the agent's partial reply) sends the text
|
|
7
|
+
* as it arrives; this computer streams it into one ElevenLabs WebSocket with
|
|
8
|
+
* its own key and sends the audio back as ElevenLabs makes it, so speech
|
|
9
|
+
* starts with the first sentence instead of after each whole one.
|
|
10
|
+
*
|
|
11
|
+
* Query: `voice` (an ElevenLabs voice id; else this computer's) and `format`,
|
|
12
|
+
* repeated, the SPEECH_FORMATS names the phone plays (else pcm_24000).
|
|
13
|
+
* Phone frames: `{"text": "<more of the reply>"}` and `{"done": true}` once
|
|
14
|
+
* the reply is complete. Hook frames: `{"type": "start", model, format,
|
|
15
|
+
* audioFormat, sampleRate}`, then `{"type": "audio", audio, alignment}` (audio
|
|
16
|
+
* base64; alignment `{characters, starts, ends}` in seconds from the start of
|
|
17
|
+
* this socket's audio, or null), then `{"type": "done"}`, or `{"type":
|
|
18
|
+
* "error", code, error}`. One socket voices one reply.
|
|
19
|
+
*
|
|
20
|
+
* v4 models are refused on ElevenLabs' text-to-speech stream-input
|
|
21
|
+
* ("unsupported_model": checked 2026-10-01) and served on text-to-dialogue
|
|
22
|
+
* stream-input, which in turn refuses Flash; so the model picks the endpoint.
|
|
23
|
+
* When the model is refused or failing, the reply falls back to Flash v2.5. */
|
|
24
|
+
const OPEN = 1;
|
|
25
|
+
/** A reply is voiced within ten minutes; the phone opens a fresh socket after. */
|
|
26
|
+
const MAX_SESSION_MS = 10 * 60_000;
|
|
27
|
+
/** Text-to-dialogue closes a socket quiet for 20 s; an agent may think longer. */
|
|
28
|
+
const KEEP_ALIVE_MS = 15_000;
|
|
29
|
+
/** ElevenLabs' socket opens in well under a second; one still connecting
|
|
30
|
+
* after this is given up on, for Flash or an error. */
|
|
31
|
+
const HANDSHAKE_MS = 5_000;
|
|
32
|
+
/** The first audio came about 140 ms after ElevenLabs had a sentence
|
|
33
|
+
* (2026-10-01). It is awaited once ElevenLabs must be voicing: the reply is
|
|
34
|
+
* done, or FORCED_TEXT of it is in. A shorter piece may sit in ElevenLabs'
|
|
35
|
+
* buffer until more text comes, which can be minutes of the agent thinking. */
|
|
36
|
+
const FIRST_AUDIO_MS = 5_000;
|
|
37
|
+
const FORCED_TEXT = 300;
|
|
38
|
+
/** Audio queued for a phone that isn't reading it: about 45 s of pcm_44100. */
|
|
39
|
+
const MAX_BUFFERED = 4 * 1024 * 1024;
|
|
40
|
+
/** The phone's frames held while this computer's settings are read. */
|
|
41
|
+
const MAX_EARLY_BYTES = 1024 * 1024;
|
|
42
|
+
const MAX_TEXT = 100_000;
|
|
43
|
+
const VOICE_SETTINGS = { stability: 0.6, similarity_boost: 0.75 };
|
|
44
|
+
function connectElevenLabs(url, key) {
|
|
45
|
+
const socket = new WebSocket(url, { headers: { "xi-api-key": key }, perMessageDeflate: false });
|
|
46
|
+
socket.on("unexpected-response", (request, response) => {
|
|
47
|
+
let body = "";
|
|
48
|
+
response.on("data", (bytes) => { body += bytes; });
|
|
49
|
+
response.on("end", () => {
|
|
50
|
+
let code = "";
|
|
51
|
+
try {
|
|
52
|
+
code = String(JSON.parse(body).detail?.status ?? "");
|
|
53
|
+
}
|
|
54
|
+
catch { /* not JSON */ }
|
|
55
|
+
socket.emit("refused", response.statusCode ?? 0, code);
|
|
56
|
+
request.destroy();
|
|
57
|
+
});
|
|
58
|
+
});
|
|
59
|
+
return socket;
|
|
60
|
+
}
|
|
61
|
+
/** Text-to-dialogue serves v3 and v4; text-to-speech stream-input the rest. */
|
|
62
|
+
export function liveEndpoint(model) {
|
|
63
|
+
return /^eleven_v[34]/.test(model) ? "dialogue" : "speech";
|
|
64
|
+
}
|
|
65
|
+
/** The longest unfinished sentence held for more text. */
|
|
66
|
+
const MAX_PENDING = 2_048;
|
|
67
|
+
/** Markdown arriving a piece at a time, released a sentence or line at a time
|
|
68
|
+
* as the words speakableText leaves of it. A code block is skipped until it
|
|
69
|
+
* closes; the unfinished sentence waits for more text or `end`. */
|
|
70
|
+
export class SpeakableStream {
|
|
71
|
+
pending = "";
|
|
72
|
+
inFence = false;
|
|
73
|
+
push(text) {
|
|
74
|
+
this.pending += text;
|
|
75
|
+
// The last line end, or sentence end followed by a space, so far.
|
|
76
|
+
let cut = this.pending.lastIndexOf("\n") + 1;
|
|
77
|
+
for (const match of this.pending.matchAll(/[.!?:;](?=\s)/g))
|
|
78
|
+
cut = Math.max(cut, match.index + 1);
|
|
79
|
+
// Text that never ends a sentence is cut at a space past MAX_PENDING, so
|
|
80
|
+
// the held text (and the rescan of it on each piece) stays small.
|
|
81
|
+
if (!cut && this.pending.length > MAX_PENDING)
|
|
82
|
+
cut = this.pending.lastIndexOf(" ", MAX_PENDING) + 1 || MAX_PENDING;
|
|
83
|
+
if (!cut)
|
|
84
|
+
return "";
|
|
85
|
+
const ready = this.pending.slice(0, cut);
|
|
86
|
+
this.pending = this.pending.slice(cut);
|
|
87
|
+
return this.speak(ready);
|
|
88
|
+
}
|
|
89
|
+
end() {
|
|
90
|
+
const rest = this.pending;
|
|
91
|
+
this.pending = "";
|
|
92
|
+
return this.speak(rest);
|
|
93
|
+
}
|
|
94
|
+
speak(text) {
|
|
95
|
+
const spoken = [];
|
|
96
|
+
for (const line of text.split("\n")) {
|
|
97
|
+
if (/^\s{0,3}(```|~~~)/.test(line)) {
|
|
98
|
+
this.inFence = !this.inFence;
|
|
99
|
+
continue;
|
|
100
|
+
}
|
|
101
|
+
if (this.inFence)
|
|
102
|
+
continue;
|
|
103
|
+
const words = speakableText(line);
|
|
104
|
+
if (words)
|
|
105
|
+
spoken.push(words);
|
|
106
|
+
}
|
|
107
|
+
return spoken.join(" ");
|
|
108
|
+
}
|
|
109
|
+
}
|
|
110
|
+
/** Seconds of audio in a chunk of the given format. */
|
|
111
|
+
function seconds(format, bytes) {
|
|
112
|
+
const mp3 = /^mp3_\d+_(\d+)$/.exec(format);
|
|
113
|
+
return mp3 ? bytes * 8 / (Number(mp3[1]) * 1_000) : bytes / (SPEECH_FORMAT_INFO[format].sampleRate * 2);
|
|
114
|
+
}
|
|
115
|
+
function liveError(text) {
|
|
116
|
+
if (/quota/i.test(text))
|
|
117
|
+
return { code: "speech-quota", error: "The ElevenLabs quota on this computer's account is used up." };
|
|
118
|
+
if (/auth|api_key|unauthori[sz]ed/i.test(text))
|
|
119
|
+
return { code: "speech-rejected", error: "ElevenLabs refused this computer's key." };
|
|
120
|
+
return { code: "speech-failed", error: "ElevenLabs couldn't voice this reply." };
|
|
121
|
+
}
|
|
122
|
+
/** A failure Flash would share: the key, the quota or the rate limit. */
|
|
123
|
+
function accountFailure(status, code) {
|
|
124
|
+
return status === 401 || status === 429 || /quota|auth|api_key|unauthori[sz]ed|rate_limit|too_many/i.test(code);
|
|
125
|
+
}
|
|
126
|
+
/** The HTTP status an error frame's code stands for, so a frame is judged as
|
|
127
|
+
* a refused handshake would be: a refused format or model, or the request,
|
|
128
|
+
* is a 400; anything else (internal_error, overloaded) ElevenLabs failing. */
|
|
129
|
+
function frameStatus(code) {
|
|
130
|
+
return /output_format|model|invalid|validation/.test(code) ? 400 : 500;
|
|
131
|
+
}
|
|
132
|
+
export function relayLiveSpeech(client, query, options = {}) {
|
|
133
|
+
const connect = options.connect ?? connectElevenLabs;
|
|
134
|
+
const state = options.state ?? sharedSpeechState;
|
|
135
|
+
// Attached before the first await: frames the phone sends while this
|
|
136
|
+
// computer's settings are read wait here, and a phone that hangs up by
|
|
137
|
+
// then never opens ElevenLabs' socket.
|
|
138
|
+
let early = [], earlyBytes = 0, overflow = false, hungUp = false;
|
|
139
|
+
let receive = (data) => {
|
|
140
|
+
earlyBytes += data.length;
|
|
141
|
+
if (earlyBytes > MAX_EARLY_BYTES)
|
|
142
|
+
overflow = true;
|
|
143
|
+
else
|
|
144
|
+
early.push(data);
|
|
145
|
+
};
|
|
146
|
+
let gone = () => { hungUp = true; };
|
|
147
|
+
client.on("message", (data, isBinary) => { if (!isBinary)
|
|
148
|
+
receive(data); });
|
|
149
|
+
client.on("close", () => gone());
|
|
150
|
+
client.on("error", () => gone());
|
|
151
|
+
return (async () => {
|
|
152
|
+
const tell = (frame) => { if (client.readyState === OPEN)
|
|
153
|
+
client.send(JSON.stringify(frame)); };
|
|
154
|
+
const key = await (options.key ?? readSpeechKey)();
|
|
155
|
+
if (!key) {
|
|
156
|
+
tell({ type: "error", code: "speech-unconfigured", error: "Spoken replies aren't set up on this computer: it has no ElevenLabs key." });
|
|
157
|
+
client.close(1008, "No ElevenLabs key");
|
|
158
|
+
return;
|
|
159
|
+
}
|
|
160
|
+
const requested = voiceId.safeParse(query.get("voice") ?? undefined);
|
|
161
|
+
const { voice } = await resolveSpeechVoice(requested.success ? requested.data : undefined);
|
|
162
|
+
const chosen = options.model ?? (await resolveSpeechModel()).model;
|
|
163
|
+
const origin = (options.origin ?? (await resolveSpeechRegion()).origin).replace(/^http/, "ws");
|
|
164
|
+
if (hungUp || client.readyState !== OPEN)
|
|
165
|
+
return;
|
|
166
|
+
const accepted = new Set(query.getAll("format"));
|
|
167
|
+
const formats = SPEECH_FORMATS.filter(format => format === "pcm_24000" || (accepted.has(format) && state.formatAllowed(format)));
|
|
168
|
+
let models = chosen === FALLBACK_SPEECH_MODEL || state.benchedReason(chosen) ? [FALLBACK_SPEECH_MODEL] : [chosen, FALLBACK_SPEECH_MODEL];
|
|
169
|
+
const text = new SpeakableStream();
|
|
170
|
+
// Everything said so far, replayed into a fallback's socket.
|
|
171
|
+
const said = [];
|
|
172
|
+
let saidLength = 0, done = false, closed = false, started = false;
|
|
173
|
+
let upstream, endpoint = "speech", format = "pcm_24000", elapsed = 0;
|
|
174
|
+
/** The current socket's handshake and first-audio timers. */
|
|
175
|
+
let watch;
|
|
176
|
+
const finish = (code = 1000, reason = "") => {
|
|
177
|
+
if (closed)
|
|
178
|
+
return;
|
|
179
|
+
closed = true;
|
|
180
|
+
clearTimeout(limit);
|
|
181
|
+
clearInterval(keepAlive);
|
|
182
|
+
watch?.stop();
|
|
183
|
+
try {
|
|
184
|
+
upstream?.close();
|
|
185
|
+
}
|
|
186
|
+
catch { /* already closed */ }
|
|
187
|
+
if (client.readyState === OPEN)
|
|
188
|
+
client.close(code, reason);
|
|
189
|
+
};
|
|
190
|
+
const fail = (frame, code = 1011) => { tell({ type: "error", ...frame }); finish(code); };
|
|
191
|
+
const limit = setTimeout(() => fail({ code: "speech-limit", error: "This reply ran past the ten minutes one spoken reply can take." }, 1000), options.maxSessionMs ?? MAX_SESSION_MS);
|
|
192
|
+
limit.unref?.();
|
|
193
|
+
const keepAlive = setInterval(() => { if (upstream?.readyState === OPEN && endpoint === "dialogue")
|
|
194
|
+
upstream.send(JSON.stringify({ keep_alive: true })); }, KEEP_ALIVE_MS);
|
|
195
|
+
keepAlive.unref?.();
|
|
196
|
+
const textFrame = (words) => JSON.stringify(endpoint === "dialogue" ? { inputs: [{ text: `${words} `, voice_id: voice }] } : { text: `${words} ` });
|
|
197
|
+
const closeFrame = () => JSON.stringify(endpoint === "dialogue" ? { close_socket: true } : { text: "" });
|
|
198
|
+
const send = (words) => {
|
|
199
|
+
if (!words)
|
|
200
|
+
return;
|
|
201
|
+
saidLength += words.length;
|
|
202
|
+
if (saidLength > MAX_TEXT) {
|
|
203
|
+
fail({ code: "speech-invalid", error: "This reply is too long to voice." });
|
|
204
|
+
return;
|
|
205
|
+
}
|
|
206
|
+
said.push(words);
|
|
207
|
+
if (upstream?.readyState === OPEN) {
|
|
208
|
+
upstream.send(textFrame(words));
|
|
209
|
+
if (saidLength >= FORCED_TEXT)
|
|
210
|
+
watch?.voicing();
|
|
211
|
+
}
|
|
212
|
+
};
|
|
213
|
+
const open = () => {
|
|
214
|
+
const model = models[0];
|
|
215
|
+
endpoint = liveEndpoint(model);
|
|
216
|
+
format = formats[0];
|
|
217
|
+
const params = new URLSearchParams({ model_id: model, output_format: format, sync_alignment: "true" });
|
|
218
|
+
if (endpoint === "speech") {
|
|
219
|
+
params.set("auto_mode", "true");
|
|
220
|
+
params.set("inactivity_timeout", "180");
|
|
221
|
+
}
|
|
222
|
+
const path = endpoint === "dialogue" ? "/v1/text-to-dialogue/stream-input" : `/v1/text-to-speech/${encodeURIComponent(voice)}/stream-input`;
|
|
223
|
+
const socket = connect(`${origin}${path}?${params}`, key);
|
|
224
|
+
upstream = socket;
|
|
225
|
+
const current = () => upstream === socket && !closed;
|
|
226
|
+
let voiced = false, firstAudio;
|
|
227
|
+
const handshake = setTimeout(() => { if (current())
|
|
228
|
+
refused(504, "handshake_timeout"); }, options.handshakeMs ?? HANDSHAKE_MS);
|
|
229
|
+
handshake.unref?.();
|
|
230
|
+
watch = {
|
|
231
|
+
voicing: () => {
|
|
232
|
+
if (voiced || firstAudio || !current())
|
|
233
|
+
return;
|
|
234
|
+
firstAudio = setTimeout(() => { if (current() && !voiced)
|
|
235
|
+
refused(504, "first_audio_timeout"); }, options.firstAudioMs ?? FIRST_AUDIO_MS);
|
|
236
|
+
firstAudio.unref?.();
|
|
237
|
+
},
|
|
238
|
+
stop: () => { clearTimeout(handshake); clearTimeout(firstAudio); },
|
|
239
|
+
};
|
|
240
|
+
const detach = () => {
|
|
241
|
+
clearTimeout(handshake);
|
|
242
|
+
clearTimeout(firstAudio);
|
|
243
|
+
// Detached first, so its closing is not taken for a failure.
|
|
244
|
+
upstream = undefined;
|
|
245
|
+
try {
|
|
246
|
+
socket.close();
|
|
247
|
+
}
|
|
248
|
+
catch { /* already closed */ }
|
|
249
|
+
};
|
|
250
|
+
// Before the first audio, the next model (Flash) takes the reply over;
|
|
251
|
+
// with none left, the phone gets `failure`.
|
|
252
|
+
const fallBack = (failure) => {
|
|
253
|
+
detach();
|
|
254
|
+
if (models.length > 1) {
|
|
255
|
+
state.bench(model, "failed");
|
|
256
|
+
models = models.slice(1);
|
|
257
|
+
open();
|
|
258
|
+
return;
|
|
259
|
+
}
|
|
260
|
+
fail(failure);
|
|
261
|
+
};
|
|
262
|
+
// A refused format or model: ElevenLabs answers some in the handshake
|
|
263
|
+
// (v4 on stream-input) and the rest in an error frame once open. A
|
|
264
|
+
// stalled socket is refused here too, as a 504.
|
|
265
|
+
const refused = (status, code) => {
|
|
266
|
+
if (format !== "pcm_24000" && (status === 400 || status === 403 || status === 422) && /output_format/.test(code)) {
|
|
267
|
+
detach();
|
|
268
|
+
state.refuseFormat(format);
|
|
269
|
+
formats.shift();
|
|
270
|
+
open();
|
|
271
|
+
return;
|
|
272
|
+
}
|
|
273
|
+
if (!accountFailure(status, code) && (status >= 500 || /model/.test(code))) {
|
|
274
|
+
fallBack(status === 504 ? { code: "speech-failed", error: "ElevenLabs didn't answer." } : liveError(`${status} ${code}`));
|
|
275
|
+
return;
|
|
276
|
+
}
|
|
277
|
+
detach();
|
|
278
|
+
fail(liveError(`${status} ${code}`));
|
|
279
|
+
};
|
|
280
|
+
socket.on("refused", (status, code) => { if (current())
|
|
281
|
+
refused(status, code); });
|
|
282
|
+
socket.on("open", () => {
|
|
283
|
+
if (!current())
|
|
284
|
+
return;
|
|
285
|
+
clearTimeout(handshake);
|
|
286
|
+
socket.send(JSON.stringify(endpoint === "dialogue" ? { voices: [voice], voice_settings: VOICE_SETTINGS } : { text: " ", voice_settings: VOICE_SETTINGS }));
|
|
287
|
+
for (const words of said)
|
|
288
|
+
socket.send(textFrame(words));
|
|
289
|
+
if (done)
|
|
290
|
+
socket.send(closeFrame());
|
|
291
|
+
if (said.length && (done || saidLength >= FORCED_TEXT))
|
|
292
|
+
watch?.voicing();
|
|
293
|
+
});
|
|
294
|
+
socket.on("message", data => {
|
|
295
|
+
if (!current())
|
|
296
|
+
return;
|
|
297
|
+
let message;
|
|
298
|
+
try {
|
|
299
|
+
message = JSON.parse(data.toString("utf8"));
|
|
300
|
+
}
|
|
301
|
+
catch {
|
|
302
|
+
return;
|
|
303
|
+
}
|
|
304
|
+
if (message.error && !voiced) {
|
|
305
|
+
refused(frameStatus(String(message.error)), String(message.error));
|
|
306
|
+
return;
|
|
307
|
+
}
|
|
308
|
+
if (typeof message.audio === "string" && message.audio) {
|
|
309
|
+
// A phone that stopped reading would hold the rest of the reply in this computer's memory.
|
|
310
|
+
if ((client.bufferedAmount ?? 0) > MAX_BUFFERED) {
|
|
311
|
+
fail({ code: "speech-failed", error: "The phone stopped taking the audio." });
|
|
312
|
+
return;
|
|
313
|
+
}
|
|
314
|
+
// Announced with the first audio, so a fallback before it is never seen.
|
|
315
|
+
if (!started) {
|
|
316
|
+
const info = SPEECH_FORMAT_INFO[format];
|
|
317
|
+
tell({ type: "start", model, format, audioFormat: info.audio, sampleRate: info.sampleRate });
|
|
318
|
+
started = true;
|
|
319
|
+
}
|
|
320
|
+
voiced = true;
|
|
321
|
+
clearTimeout(firstAudio);
|
|
322
|
+
const audio = message.audio, length = Buffer.from(audio, "base64").length;
|
|
323
|
+
tell({ type: "audio", audio, alignment: alignment(message.alignment) });
|
|
324
|
+
elapsed += seconds(format, length);
|
|
325
|
+
}
|
|
326
|
+
if (message.isFinal === true || message.is_final === true) {
|
|
327
|
+
tell({ type: "done" });
|
|
328
|
+
finish();
|
|
329
|
+
return;
|
|
330
|
+
}
|
|
331
|
+
if (message.error)
|
|
332
|
+
fail(liveError(String(message.error)));
|
|
333
|
+
});
|
|
334
|
+
socket.on("error", () => { if (current()) {
|
|
335
|
+
detach();
|
|
336
|
+
fail({ code: "speech-unreachable", error: "Couldn't reach ElevenLabs from this computer." });
|
|
337
|
+
} });
|
|
338
|
+
socket.on("close", (_code, reason) => {
|
|
339
|
+
if (!current())
|
|
340
|
+
return;
|
|
341
|
+
if (voiced) {
|
|
342
|
+
detach();
|
|
343
|
+
fail({ code: "speech-failed", error: "ElevenLabs stopped before the reply was voiced." });
|
|
344
|
+
return;
|
|
345
|
+
}
|
|
346
|
+
// Closed before any audio: the reason is read only to tell the
|
|
347
|
+
// account's failures (which Flash shares) from the model's.
|
|
348
|
+
const why = reason?.toString("utf8") ?? "";
|
|
349
|
+
if (accountFailure(0, why)) {
|
|
350
|
+
detach();
|
|
351
|
+
fail(liveError(why));
|
|
352
|
+
return;
|
|
353
|
+
}
|
|
354
|
+
fallBack({ code: "speech-failed", error: "ElevenLabs didn't answer." });
|
|
355
|
+
});
|
|
356
|
+
};
|
|
357
|
+
/** In seconds from the start of this socket's audio. Stream-input counts
|
|
358
|
+
* from the start of its stream; text-to-dialogue from each chunk. */
|
|
359
|
+
const alignment = (raw) => {
|
|
360
|
+
const value = raw;
|
|
361
|
+
const characters = value?.chars, starts = value?.charStartTimesMs ?? value?.char_start_times_ms, durations = value?.charDurationsMs ?? value?.char_durations_ms;
|
|
362
|
+
if (!Array.isArray(characters) || !Array.isArray(starts) || !Array.isArray(durations) || !characters.length)
|
|
363
|
+
return null;
|
|
364
|
+
if (characters.length !== starts.length || characters.length !== durations.length)
|
|
365
|
+
return null;
|
|
366
|
+
if (!characters.every(c => typeof c === "string") || ![...starts, ...durations].every(t => typeof t === "number" && Number.isFinite(t)))
|
|
367
|
+
return null;
|
|
368
|
+
const offset = endpoint === "dialogue" ? elapsed : 0;
|
|
369
|
+
const round = (t) => Math.round(t * 1_000) / 1_000;
|
|
370
|
+
return { characters, starts: starts.map(t => round(offset + t / 1_000)), ends: starts.map((t, i) => round(offset + (t + durations[i]) / 1_000)) };
|
|
371
|
+
};
|
|
372
|
+
const handle = (data) => {
|
|
373
|
+
if (closed || done)
|
|
374
|
+
return;
|
|
375
|
+
let frame;
|
|
376
|
+
try {
|
|
377
|
+
frame = JSON.parse(data.toString("utf8"));
|
|
378
|
+
}
|
|
379
|
+
catch {
|
|
380
|
+
return;
|
|
381
|
+
}
|
|
382
|
+
if (typeof frame.text === "string")
|
|
383
|
+
send(text.push(frame.text));
|
|
384
|
+
if (frame.done === true && !closed) {
|
|
385
|
+
done = true;
|
|
386
|
+
send(text.end());
|
|
387
|
+
if (closed)
|
|
388
|
+
return;
|
|
389
|
+
// Nothing speakable (a reply of only code, say): no audio is coming.
|
|
390
|
+
if (!saidLength) {
|
|
391
|
+
tell({ type: "done" });
|
|
392
|
+
finish();
|
|
393
|
+
return;
|
|
394
|
+
}
|
|
395
|
+
if (upstream?.readyState === OPEN) {
|
|
396
|
+
upstream.send(closeFrame());
|
|
397
|
+
watch?.voicing();
|
|
398
|
+
}
|
|
399
|
+
}
|
|
400
|
+
};
|
|
401
|
+
receive = handle;
|
|
402
|
+
gone = () => finish();
|
|
403
|
+
if (overflow) {
|
|
404
|
+
fail({ code: "speech-invalid", error: "This reply is too long to voice." });
|
|
405
|
+
return;
|
|
406
|
+
}
|
|
407
|
+
for (const data of early)
|
|
408
|
+
handle(data);
|
|
409
|
+
early = [];
|
|
410
|
+
if (!closed)
|
|
411
|
+
open();
|
|
412
|
+
})();
|
|
413
|
+
}
|
|
@@ -17,8 +17,11 @@ export interface RelaySocket {
|
|
|
17
17
|
export interface TranscribeOptions {
|
|
18
18
|
key?: () => Promise<string | undefined>;
|
|
19
19
|
connect?: (url: string, key: string) => RelaySocket;
|
|
20
|
+
/** Defaults to this computer's region setting (speech-voice.ts). */
|
|
21
|
+
origin?: string;
|
|
20
22
|
maxSessionMs?: number;
|
|
21
23
|
}
|
|
22
|
-
/** The upstream URL: VAD commits, the phone's language
|
|
23
|
-
|
|
24
|
+
/** The upstream URL in the given region: VAD commits, the phone's language
|
|
25
|
+
* and its vocabulary as keyterms. */
|
|
26
|
+
export declare function transcribeURL(query: URLSearchParams, origin?: string): string;
|
|
24
27
|
export declare function relayTranscription(client: RelaySocket, query: URLSearchParams, options?: TranscribeOptions): Promise<void>;
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { WebSocket } from "ws";
|
|
2
2
|
import { readSpeechKey } from "./speech-key.js";
|
|
3
|
+
import { resolveSpeechRegion, SPEECH_REGIONS } from "./speech-voice.js";
|
|
3
4
|
/** Dictation through ElevenLabs Scribe v2 Realtime, for the phone's opt-in
|
|
4
5
|
* Scribe input. The phone streams 16 kHz mono PCM as binary frames; this
|
|
5
6
|
* computer forwards it to ElevenLabs with its own key, which never leaves
|
|
@@ -7,16 +8,17 @@ import { readSpeechKey } from "./speech-key.js";
|
|
|
7
8
|
* are fixed messages by code: ElevenLabs' own text is never passed on. */
|
|
8
9
|
export const TRANSCRIBE_MODEL = "scribe_v2_realtime";
|
|
9
10
|
export const TRANSCRIBE_RATE = 16_000;
|
|
10
|
-
const ENDPOINT = "
|
|
11
|
+
const ENDPOINT = "/v1/speech-to-text/realtime";
|
|
11
12
|
/** Ten minutes of dictation per socket; the phone opens a fresh one after. */
|
|
12
13
|
const MAX_SESSION_MS = 10 * 60_000;
|
|
13
14
|
/** Audio held while the upstream socket opens: about eight seconds. */
|
|
14
15
|
const MAX_PENDING_BYTES = TRANSCRIBE_RATE * 2 * 8;
|
|
15
16
|
const MAX_KEYTERMS = 50;
|
|
16
17
|
const OPEN = 1;
|
|
17
|
-
/** The upstream URL: VAD commits, the phone's language
|
|
18
|
-
|
|
19
|
-
|
|
18
|
+
/** The upstream URL in the given region: VAD commits, the phone's language
|
|
19
|
+
* and its vocabulary as keyterms. */
|
|
20
|
+
export function transcribeURL(query, origin = SPEECH_REGIONS.global) {
|
|
21
|
+
const url = new URL(ENDPOINT, origin.replace(/^http/, "ws"));
|
|
20
22
|
url.searchParams.set("model_id", TRANSCRIBE_MODEL);
|
|
21
23
|
url.searchParams.set("audio_format", "pcm_16000");
|
|
22
24
|
url.searchParams.set("commit_strategy", "vad");
|
|
@@ -47,7 +49,8 @@ export function relayTranscription(client, query, options = {}) {
|
|
|
47
49
|
client.close(1008, "No ElevenLabs key");
|
|
48
50
|
return;
|
|
49
51
|
}
|
|
50
|
-
const
|
|
52
|
+
const origin = options.origin ?? (await resolveSpeechRegion()).origin;
|
|
53
|
+
const upstream = connect(transcribeURL(query, origin), key);
|
|
51
54
|
let pending = [], pendingBytes = 0, closed = false;
|
|
52
55
|
const finish = (code = 1000, reason = "") => {
|
|
53
56
|
if (closed)
|
|
@@ -32,6 +32,27 @@ export declare function resolveSpeechModel(file?: string): Promise<{
|
|
|
32
32
|
model: string;
|
|
33
33
|
source: "setting" | "default";
|
|
34
34
|
}>;
|
|
35
|
+
/** Where this computer reaches ElevenLabs. `global` is api.elevenlabs.io,
|
|
36
|
+
* which routes to the nearest region; `us` is ElevenLabs' US-only endpoint.
|
|
37
|
+
* `phren bridge speech-region us|global` stores it in the same `speech.json`
|
|
38
|
+
* as the voice and model; unset is global. Read on every reply. */
|
|
39
|
+
export declare const SPEECH_REGIONS: {
|
|
40
|
+
readonly global: "https://api.elevenlabs.io";
|
|
41
|
+
readonly us: "https://api.us.elevenlabs.io";
|
|
42
|
+
};
|
|
43
|
+
export type SpeechRegion = keyof typeof SPEECH_REGIONS;
|
|
44
|
+
export declare const DEFAULT_SPEECH_REGION: SpeechRegion;
|
|
45
|
+
export declare const speechRegion: z.ZodEnum<{
|
|
46
|
+
global: "global";
|
|
47
|
+
us: "us";
|
|
48
|
+
}>;
|
|
49
|
+
export declare function resolveSpeechRegion(file?: string): Promise<{
|
|
50
|
+
region: SpeechRegion;
|
|
51
|
+
origin: string;
|
|
52
|
+
source: "setting" | "default";
|
|
53
|
+
}>;
|
|
54
|
+
/** Global is the default, so choosing it removes the key. */
|
|
55
|
+
export declare function writeSpeechRegion(region: string, file?: string): Promise<SpeechRegion>;
|
|
35
56
|
export type SpeechVoiceSource = "request" | "setting" | "environment" | "default";
|
|
36
57
|
/** The voice to speak with: the request's own, the stored setting, the old
|
|
37
58
|
* environment override (copied into the setting on first use), else the
|
|
@@ -52,4 +73,4 @@ export interface SpeechVoiceChoice {
|
|
|
52
73
|
/** The voices this computer's ElevenLabs account can speak with, for the
|
|
53
74
|
* phone's picker: id, name and a short description, never the account's
|
|
54
75
|
* other details. */
|
|
55
|
-
export declare function listSpeechVoices(key: string, fetcher?: typeof fetch, signal?: AbortSignal): Promise<SpeechVoiceChoice[]>;
|
|
76
|
+
export declare function listSpeechVoices(key: string, fetcher?: typeof fetch, signal?: AbortSignal, origin?: string): Promise<SpeechVoiceChoice[]>;
|
|
@@ -79,6 +79,24 @@ export async function resolveSpeechModel(file = speechVoiceFile()) {
|
|
|
79
79
|
const stored = await readStoredModel(file);
|
|
80
80
|
return stored ? { model: stored, source: "setting" } : { model: DEFAULT_SPEECH_MODEL, source: "default" };
|
|
81
81
|
}
|
|
82
|
+
/** Where this computer reaches ElevenLabs. `global` is api.elevenlabs.io,
|
|
83
|
+
* which routes to the nearest region; `us` is ElevenLabs' US-only endpoint.
|
|
84
|
+
* `phren bridge speech-region us|global` stores it in the same `speech.json`
|
|
85
|
+
* as the voice and model; unset is global. Read on every reply. */
|
|
86
|
+
export const SPEECH_REGIONS = { global: "https://api.elevenlabs.io", us: "https://api.us.elevenlabs.io" };
|
|
87
|
+
export const DEFAULT_SPEECH_REGION = "global";
|
|
88
|
+
export const speechRegion = z.enum(Object.keys(SPEECH_REGIONS));
|
|
89
|
+
export async function resolveSpeechRegion(file = speechVoiceFile()) {
|
|
90
|
+
const parsed = speechRegion.safeParse((await readSettings(file)).region);
|
|
91
|
+
const region = parsed.success ? parsed.data : DEFAULT_SPEECH_REGION;
|
|
92
|
+
return { region, origin: SPEECH_REGIONS[region], source: parsed.success ? "setting" : "default" };
|
|
93
|
+
}
|
|
94
|
+
/** Global is the default, so choosing it removes the key. */
|
|
95
|
+
export async function writeSpeechRegion(region, file = speechVoiceFile()) {
|
|
96
|
+
const value = speechRegion.parse(region);
|
|
97
|
+
await updateSettings(file, "region", value === DEFAULT_SPEECH_REGION ? undefined : value);
|
|
98
|
+
return value;
|
|
99
|
+
}
|
|
82
100
|
/** The voice to speak with: the request's own, the stored setting, the old
|
|
83
101
|
* environment override (copied into the setting on first use), else the
|
|
84
102
|
* default. */
|
|
@@ -99,10 +117,10 @@ export async function resolveSpeechVoice(requested, paths = {}) {
|
|
|
99
117
|
/** The voices this computer's ElevenLabs account can speak with, for the
|
|
100
118
|
* phone's picker: id, name and a short description, never the account's
|
|
101
119
|
* other details. */
|
|
102
|
-
export async function listSpeechVoices(key, fetcher = fetch, signal) {
|
|
120
|
+
export async function listSpeechVoices(key, fetcher = fetch, signal, origin) {
|
|
103
121
|
let upstream;
|
|
104
122
|
try {
|
|
105
|
-
upstream = await fetcher(
|
|
123
|
+
upstream = await fetcher(`${origin ?? (await resolveSpeechRegion()).origin}/v1/voices`, { headers: { "xi-api-key": key }, signal });
|
|
106
124
|
}
|
|
107
125
|
catch {
|
|
108
126
|
throw new BridgeError(502, "Couldn't reach ElevenLabs from this computer.", { code: "speech-unreachable" });
|