@phren/cli 0.3.1 → 0.3.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -195,6 +195,11 @@ export declare class AgentHooks {
195
195
  * phone with its own keys: let the callback fall back to the terminal at
196
196
  * once instead of leaving the card up until the 55-second timer. */
197
197
  releaseChoice(target: Target): void;
198
+ /** The phone's Esc or Cancel on a question the Hook holds. A held
199
+ * AskUserQuestion has no terminal choice, so `releaseChoice` never lets it
200
+ * go and the terminal stays frozen until the hold timer; decline it now so
201
+ * Claude moves on. True when a held question was cancelled. */
202
+ cancelHeldQuestion(target: Target): boolean;
198
203
  answerPush(binding: string, decision: unknown): Promise<void>;
199
204
  /** Where a pushed ask lives, without answering it: the phone opens that
200
205
  * session's details when the notification itself is tapped. */
@@ -809,6 +809,25 @@ export class AgentHooks {
809
809
  entry.response.end("{}");
810
810
  }
811
811
  }
812
+ /** The phone's Esc or Cancel on a question the Hook holds. A held
813
+ * AskUserQuestion has no terminal choice, so `releaseChoice` never lets it
814
+ * go and the terminal stays frozen until the hold timer; decline it now so
815
+ * Claude moves on. True when a held question was cancelled. */
816
+ cancelHeldQuestion(target) {
817
+ let cancelled = false;
818
+ for (const [id, entry] of this.pending) {
819
+ if (entry.tool !== "AskUserQuestion" || JSON.stringify(entry.target) !== JSON.stringify(target))
820
+ continue;
821
+ this.pending.delete(id);
822
+ this.dropPushBindings(id);
823
+ clearTimeout(entry.timer);
824
+ if (!entry.response.destroyed)
825
+ entry.response.end(JSON.stringify({ hookSpecificOutput: { hookEventName: "PermissionRequest",
826
+ decision: { behavior: "deny", message: "The user cancelled this question in Phren without answering." } } }));
827
+ cancelled = true;
828
+ }
829
+ return cancelled;
830
+ }
812
831
  async answerPush(binding, decision) {
813
832
  if (!["approve", "deny"].includes(String(decision)))
814
833
  throw new BridgeError(400, "The approval answer is not valid.");
@@ -1,7 +1,8 @@
1
1
  import { type Json } from "./protocol.js";
2
2
  export declare function runBridge(args: string[], version: string): Promise<number>;
3
- /** Doctor's push check, from the running Hook's own capability: only a Hook
4
- * that loaded apns.json and its key offers `approvalPush`. */
3
+ /** Doctor's push check, from the running Hook's own capability: a Hook that
4
+ * loaded apns.json and its key, or one whose phones registered through the
5
+ * push relay, offers `approvalPush`. */
5
6
  export declare function approvalPushCheck(helper: Json): {
6
7
  configured: boolean;
7
8
  warning?: string;
@@ -131,10 +131,12 @@ export async function runBridge(args, version) {
131
131
  }
132
132
  return 0;
133
133
  }
134
- /** Doctor's push check, from the running Hook's own capability: only a Hook
135
- * that loaded apns.json and its key offers `approvalPush`. */
134
+ /** Doctor's push check, from the running Hook's own capability: a Hook that
135
+ * loaded apns.json and its key, or one whose phones registered through the
136
+ * push relay, offers `approvalPush`. */
136
137
  export function approvalPushCheck(helper) {
137
- const configured = object(helper.capabilities).approvalPush === "direct-apns";
138
+ const capability = object(helper.capabilities).approvalPush;
139
+ const configured = capability === "direct-apns" || capability === "relay";
138
140
  return configured ? { configured } : { configured, warning: apnsSetupSteps() };
139
141
  }
140
142
  const SPEECH_KEY_USAGE = "Usage: phren bridge speech-key set (paste the key when asked, or pipe it on stdin)";
@@ -1,8 +1,23 @@
1
1
  import { z } from "zod";
2
+ /** A phone registered through the phren push relay: the relay knows where to
3
+ * deliver, and `key` (32 bytes, base64url) encrypts what the notification says
4
+ * so only the phone reads it. The phone made the key and sent it over SSH. */
5
+ declare const relaySchema: z.ZodObject<{
6
+ url: z.ZodString;
7
+ relayId: z.ZodString;
8
+ secret: z.ZodString;
9
+ key: z.ZodString;
10
+ }, z.core.$strip>;
2
11
  declare const deviceSchema: z.ZodObject<{
3
12
  deviceID: z.ZodString;
4
13
  hostID: z.ZodString;
5
- token: z.ZodString;
14
+ token: z.ZodOptional<z.ZodString>;
15
+ relay: z.ZodOptional<z.ZodObject<{
16
+ url: z.ZodString;
17
+ relayId: z.ZodString;
18
+ secret: z.ZodString;
19
+ key: z.ZodString;
20
+ }, z.core.$strip>>;
6
21
  environment: z.ZodEnum<{
7
22
  development: "development";
8
23
  production: "production";
@@ -54,11 +69,29 @@ export declare function fanoutBlockedPushPayload(value: FanoutBlockedPush): Reco
54
69
  export declare function schedulePushPayload(value: SchedulePush): Record<string, unknown>;
55
70
  /** What `phren bridge doctor` prints when the Hook has no APNs sender. */
56
71
  export declare function apnsSetupSteps(configFile?: string): string;
57
- /** The capability a phone reads: only a Hook with a loaded APNs sender offers push. */
72
+ /** The capability a phone reads: a Hook with its own APNs key sends direct;
73
+ * one whose phones registered through the relay sends through it. */
58
74
  export declare function approvalPushCapability(status: {
59
75
  configured: boolean;
60
- }): "direct-apns" | undefined;
76
+ direct?: boolean;
77
+ }): "direct-apns" | "relay" | undefined;
61
78
  export declare function upsertPushDevice(devices: PushDevice[], value: unknown): PushDevice[];
79
+ /** The relay refuses ciphertext longer than this (it keeps the APNs payload under 4 KB). */
80
+ export declare const RELAY_MAX_CIPHERTEXT = 2800;
81
+ /** What the phone's notification extension shows, encrypted with the phone's
82
+ * key: ChaCha20-Poly1305, base64url of nonce (12) + ciphertext + tag (16), the
83
+ * layout CryptoKit's `ChaChaPoly.SealedBox(combined:)` reads. A long body is
84
+ * shortened until it fits the relay's limit. */
85
+ export declare function relayCiphertext(key: string, payload: Record<string, unknown>, nonce?: NonSharedBuffer): string;
86
+ /** Signs a relay send: base64url HMAC-SHA256 of `${timestamp}.${body}` under the phone's send secret. */
87
+ export declare function relaySignature(secret: string, timestamp: string, body: string): string;
88
+ type RelayRegistration = z.infer<typeof relaySchema>;
89
+ /** "gone" means the relay (Apple) no longer knows the phone: stop sending until it registers again. */
90
+ export type RelayResult = "sent" | "failed" | "gone";
91
+ export declare function sendThroughRelay(relay: RelayRegistration, payload: Record<string, unknown>, headers: {
92
+ expiration: string;
93
+ collapseId: string;
94
+ }, fetcher?: typeof fetch, now?: () => number): Promise<RelayResult>;
62
95
  /** Direct Hook -> APNs delivery. Apple receives only a short-lived opaque
63
96
  * binding; conversation identity, action id, command and SSH key stay local. */
64
97
  export declare class ApprovalPushService {
@@ -66,13 +99,19 @@ export declare class ApprovalPushService {
66
99
  private sender?;
67
100
  private readonly devicesFile;
68
101
  start(): Promise<void>;
102
+ /** Phones this Hook can reach: through the relay, or direct with its own key. */
103
+ private get reachable();
69
104
  get available(): boolean;
70
105
  get status(): {
71
106
  supported: boolean;
72
107
  configured: boolean;
108
+ direct: boolean;
73
109
  devices: number;
110
+ relay: number;
74
111
  };
75
112
  register(value: unknown): Promise<void>;
113
+ /** One phone: the relay when it registered through one, otherwise direct. */
114
+ private send;
76
115
  notify(value: ApprovalPush): Promise<boolean>;
77
116
  /** A headless worker whose permission the plugin refused. Approval-registered
78
117
  * phones already accept agent alerts; the payload carries the reason so the
@@ -1,16 +1,27 @@
1
- import { createPrivateKey, sign } from "node:crypto";
1
+ import { createCipheriv, createHmac, createPrivateKey, randomBytes, sign } from "node:crypto";
2
2
  import { readFile, stat } from "node:fs/promises";
3
3
  import { connect } from "node:http2";
4
4
  import path from "node:path";
5
5
  import { z } from "zod";
6
6
  import { atomic, bridgeRoot } from "./protocol.js";
7
+ /** A phone registered through the phren push relay: the relay knows where to
8
+ * deliver, and `key` (32 bytes, base64url) encrypts what the notification says
9
+ * so only the phone reads it. The phone made the key and sent it over SSH. */
10
+ const relaySchema = z.object({
11
+ url: z.string().url().refine(value => /^https:\/\//.test(value) || /^http:\/\/(127\.0\.0\.1|localhost)(:\d+)?\//.test(`${value}/`), "The relay must use https."),
12
+ relayId: z.string().regex(/^[A-Za-z0-9_-]{40,400}$/),
13
+ secret: z.string().regex(/^[A-Za-z0-9_-]{20,100}$/),
14
+ key: z.string().regex(/^[A-Za-z0-9_-]{43}$/),
15
+ });
7
16
  const deviceSchema = z.object({
8
17
  deviceID: z.string().uuid(),
9
18
  hostID: z.string().uuid(),
10
- token: z.string().regex(/^[0-9a-f]{64}$/),
19
+ /** The APNs token, for a Hook with its own key (apns.json). */
20
+ token: z.string().regex(/^[0-9a-f]{64}$/).optional(),
21
+ relay: relaySchema.optional(),
11
22
  environment: z.enum(["development", "production"]),
12
23
  kinds: z.array(z.enum(["approval", "scheduleStarted", "scheduleFinished", "scheduleFailed", "scheduleBlocked"])).max(5).default(["approval"]),
13
- });
24
+ }).refine(device => device.token !== undefined || device.relay !== undefined, "A phone needs an APNs token or a relay registration.");
14
25
  const configSchema = z.object({
15
26
  keyId: z.string().regex(/^[A-Z0-9]{10}$/),
16
27
  teamId: z.string().regex(/^[A-Z0-9]{10}$/),
@@ -83,15 +94,59 @@ export function apnsSetupSteps(configFile = process.env.PHREN_APNS_CONFIG || pat
83
94
  "4. Restart the Hook (phren bridge install), then run phren bridge doctor again.",
84
95
  ].join("\n");
85
96
  }
86
- /** The capability a phone reads: only a Hook with a loaded APNs sender offers push. */
97
+ /** The capability a phone reads: a Hook with its own APNs key sends direct;
98
+ * one whose phones registered through the relay sends through it. */
87
99
  export function approvalPushCapability(status) {
88
- return status.configured ? "direct-apns" : undefined;
100
+ if (!status.configured)
101
+ return undefined;
102
+ return status.direct === false ? "relay" : "direct-apns";
89
103
  }
90
104
  export function upsertPushDevice(devices, value) {
91
105
  const device = deviceSchema.parse(value);
92
106
  return [...devices.filter(item => item.deviceID !== device.deviceID), device].slice(-16);
93
107
  }
94
108
  const base64url = (value) => Buffer.from(value).toString("base64url");
109
+ /** The relay refuses ciphertext longer than this (it keeps the APNs payload under 4 KB). */
110
+ export const RELAY_MAX_CIPHERTEXT = 2_800;
111
+ /** What the phone's notification extension shows, encrypted with the phone's
112
+ * key: ChaCha20-Poly1305, base64url of nonce (12) + ciphertext + tag (16), the
113
+ * layout CryptoKit's `ChaChaPoly.SealedBox(combined:)` reads. A long body is
114
+ * shortened until it fits the relay's limit. */
115
+ export function relayCiphertext(key, payload, nonce = randomBytes(12)) {
116
+ const aps = (payload.aps ?? {});
117
+ let body = aps.alert?.body ?? "";
118
+ for (;;) {
119
+ const content = JSON.stringify({ t: aps.alert?.title ?? "phren", b: body, ...(aps.category ? { c: aps.category } : {}),
120
+ ...(aps["interruption-level"] ? { i: aps["interruption-level"] } : {}), ...(payload.phren ? { p: payload.phren } : {}) });
121
+ const cipher = createCipheriv("chacha20-poly1305", Buffer.from(key, "base64url"), nonce, { authTagLength: 16 });
122
+ const sealed = Buffer.concat([nonce, cipher.update(content, "utf8"), cipher.final(), cipher.getAuthTag()]).toString("base64url");
123
+ if (sealed.length <= RELAY_MAX_CIPHERTEXT || !body)
124
+ return sealed;
125
+ body = body.length > 40 ? `${body.slice(0, Math.floor(body.length * 0.8))}…` : "";
126
+ }
127
+ }
128
+ /** Signs a relay send: base64url HMAC-SHA256 of `${timestamp}.${body}` under the phone's send secret. */
129
+ export function relaySignature(secret, timestamp, body) {
130
+ return createHmac("sha256", secret).update(`${timestamp}.${body}`).digest("base64url");
131
+ }
132
+ export async function sendThroughRelay(relay, payload, headers, fetcher = fetch, now = Date.now) {
133
+ const aps = (payload.aps ?? {});
134
+ const body = JSON.stringify({ kind: "alert", ciphertext: relayCiphertext(relay.key, payload),
135
+ collapseId: headers.collapseId.slice(0, 64), expiration: Number(headers.expiration) || 0,
136
+ ...(aps.category ? { category: aps.category } : {}) });
137
+ const timestamp = String(Math.floor(now() / 1000));
138
+ try {
139
+ const response = await fetcher(`${relay.url.replace(/\/+$/, "")}/v1/send`, {
140
+ method: "POST", body, signal: AbortSignal.timeout(10_000),
141
+ headers: { "content-type": "application/json", "x-phren-relay": relay.relayId, "x-phren-timestamp": timestamp,
142
+ "x-phren-signature": relaySignature(relay.secret, timestamp, body) },
143
+ });
144
+ return response.status === 410 ? "gone" : response.ok ? "sent" : "failed";
145
+ }
146
+ catch {
147
+ return "failed";
148
+ }
149
+ }
95
150
  class APNsSender {
96
151
  config;
97
152
  key;
@@ -149,19 +204,35 @@ export class ApprovalPushService {
149
204
  if (key)
150
205
  this.sender = new APNsSender(config, key);
151
206
  }
152
- get available() { return this.sender !== undefined && this.devices.length > 0; }
153
- get status() { return { supported: true, configured: this.sender !== undefined, devices: this.devices.length }; }
207
+ /** Phones this Hook can reach: through the relay, or direct with its own key. */
208
+ get reachable() { return this.devices.filter(device => device.relay || (this.sender && device.token)); }
209
+ get available() { return this.reachable.length > 0; }
210
+ get status() {
211
+ const relay = this.devices.filter(device => device.relay).length;
212
+ return { supported: true, configured: this.sender !== undefined || relay > 0, direct: this.sender !== undefined,
213
+ devices: this.devices.length, relay };
214
+ }
154
215
  async register(value) {
155
216
  this.devices = upsertPushDevice(this.devices, value);
156
217
  await atomic(this.devicesFile, JSON.stringify(this.devices));
157
218
  }
219
+ /** One phone: the relay when it registered through one, otherwise direct. */
220
+ async send(device, payload, headers) {
221
+ if (device.relay) {
222
+ const result = await sendThroughRelay(device.relay, payload, headers);
223
+ if (result === "gone") {
224
+ this.devices = this.devices.filter(item => item.deviceID !== device.deviceID);
225
+ await atomic(this.devicesFile, JSON.stringify(this.devices)).catch(() => { });
226
+ }
227
+ return result === "sent";
228
+ }
229
+ return this.sender && device.token ? this.sender.send(device, payload, headers) : false;
230
+ }
158
231
  async notify(value) {
159
- if (!this.sender || !this.devices.length)
160
- return false;
161
- const devices = this.devices.filter(device => device.kinds.includes("approval"));
232
+ const devices = this.reachable.filter(device => device.kinds.includes("approval"));
162
233
  if (!devices.length)
163
234
  return false;
164
- return (await Promise.all(devices.map(device => this.sender.send(device, approvalPushPayload(value, device.hostID), {
235
+ return (await Promise.all(devices.map(device => this.send(device, approvalPushPayload(value, device.hostID), {
165
236
  expiration: String(Math.floor(Date.parse(value.expiresAt) / 1000)), collapseId: value.binding,
166
237
  })))).some(Boolean);
167
238
  }
@@ -169,25 +240,24 @@ export class ApprovalPushService {
169
240
  * phones already accept agent alerts; the payload carries the reason so the
170
241
  * notification is actionable on its own. */
171
242
  async notifyFanoutBlocked(value) {
172
- if (!this.sender || !this.devices.length)
173
- return false;
174
- const devices = this.devices.filter(device => device.kinds.includes("approval"));
243
+ const devices = this.reachable.filter(device => device.kinds.includes("approval"));
175
244
  if (!devices.length)
176
245
  return false;
177
- return (await Promise.all(devices.map(device => this.sender.send(device, fanoutBlockedPushPayload(value), {
246
+ return (await Promise.all(devices.map(device => this.send(device, fanoutBlockedPushPayload(value), {
178
247
  expiration: "0", collapseId: `fanout-${value.job}`,
179
248
  })))).some(Boolean);
180
249
  }
181
250
  async notifySchedule(value) {
182
- if (!this.sender)
183
- return { notified: false, reason: "no push config" };
184
- let devices = this.devices.filter(device => device.kinds.includes(value.kind));
251
+ const reachable = this.reachable;
252
+ if (!reachable.length)
253
+ return { notified: false, reason: this.sender || this.devices.length ? "no registered devices" : "no push config" };
254
+ let devices = reachable.filter(device => device.kinds.includes(value.kind));
185
255
  if (!devices.length && value.kind === "scheduleBlocked")
186
- devices = this.devices.filter(device => device.kinds.includes("scheduleFailed"));
256
+ devices = reachable.filter(device => device.kinds.includes("scheduleFailed"));
187
257
  if (!devices.length)
188
258
  return { notified: false, reason: "no registered devices" };
189
259
  try {
190
- const notified = (await Promise.all(devices.map(device => this.sender.send(device, schedulePushPayload(value), {
260
+ const notified = (await Promise.all(devices.map(device => this.send(device, schedulePushPayload(value), {
191
261
  expiration: "0", collapseId: scheduleCollapseId(value.kind, value.runId),
192
262
  })))).some(Boolean);
193
263
  return notified ? { notified: true } : { notified: false, reason: "push delivery failed" };
@@ -364,15 +364,21 @@ export async function paneRoute(ctx, url, data, response) {
364
364
  // not hold) is being answered even when Herdr reads the pane as
365
365
  // working or idle; Herdr's status lags the agent's own dialog.
366
366
  const holding = menu || !!agentHooks.terminalPrompt(target);
367
+ // Esc on a question the Hook still holds declines it at once, whatever
368
+ // Herdr reads the pane as: an esc typed into the held pane does nothing.
369
+ const cancelled = keys.every(key => key === "Escape") && agentHooks.cancelHeldQuestion(target);
367
370
  // Escape interrupts a working agent. Everything else answers a
368
371
  // prompt the agent is holding: a menu, a y/n, a trust question.
369
- if (!holding && (keys.every(key => key === "Escape") ? !["working", "blocked", "waiting", "unknown"].includes(status)
372
+ if (!cancelled && !holding && (keys.every(key => key === "Escape") ? !["working", "blocked", "waiting", "unknown"].includes(status)
370
373
  : !["blocked", "waiting", "unknown"].includes(status)))
371
374
  throw new BridgeError(409, keys.every(key => key === "Escape") ? "This agent is no longer working." : "This agent is not waiting for an answer.");
372
- // An older phone answers a released AskUserQuestion one question at
373
- // a time with its digits; the Hook walks that question in the pane
374
- // and submits the set after the last.
375
- if (await agentHooks.answerReleasedQuestion(target, keys)) {
375
+ if (cancelled) {
376
+ result = { ok: true };
377
+ // An older phone answers a released AskUserQuestion one question at
378
+ // a time with its digits; the Hook walks that question in the pane
379
+ // and submits the set after the last.
380
+ }
381
+ else if (await agentHooks.answerReleasedQuestion(target, keys)) {
376
382
  result = { ok: true };
377
383
  }
378
384
  else {
@@ -92,6 +92,7 @@ export declare const capabilities: {
92
92
  code: boolean;
93
93
  overviewStream: boolean;
94
94
  speech: boolean;
95
+ speechTimestamps: boolean;
95
96
  transcribe: boolean;
96
97
  memoryStore: boolean;
97
98
  };
@@ -51,7 +51,7 @@ async function childActivity(source, session) {
51
51
  export const capabilities = { transcript: true, progress: true, images: true, prompt: true, stop: true,
52
52
  terminal: "ssh-pty", shell: "ssh-pty", herdr: true, diff: true, webServers: true, webPreview: "ssh-exec", activity: true,
53
53
  approvals: true, questions: false, accountUsage: true, providers: ["codex", "claude", "copilot", "opencode"],
54
- files: true, repositoryFiles: true, subagents: true, sideQuestions: true, dispatch: true, approvalPush: "direct-apns", simulators: process.platform === "darwin", code: true, overviewStream: true, speech: true, transcribe: true, memoryStore: true };
54
+ files: true, repositoryFiles: true, subagents: true, sideQuestions: true, dispatch: true, approvalPush: "direct-apns", simulators: process.platform === "darwin", code: true, overviewStream: true, speech: true, speechTimestamps: true, transcribe: true, memoryStore: true };
55
55
  export function capabilitiesForModules(snapshot) {
56
56
  const allowed = new Set(snapshot.modules.flatMap(module => module.capabilities));
57
57
  const result = Object.fromEntries(Object.entries(capabilities).filter(([name]) => allowed.has(name)));
@@ -14,7 +14,15 @@ export declare const SPEECH_MODEL = "eleven_flash_v2_5";
14
14
  export declare const DEFAULT_SPEECH_VOICE = "SAz9YHcvj6GT2YYXdXww";
15
15
  export declare const speechRequest: z.ZodObject<{
16
16
  text: z.ZodString;
17
+ timestamps: z.ZodOptional<z.ZodBoolean>;
17
18
  }, z.core.$strip>;
19
+ /** When each character of the voiced text starts and ends, in seconds from
20
+ * the start of the audio. */
21
+ export interface SpeechAlignment {
22
+ characters: string[];
23
+ starts: number[];
24
+ ends: number[];
25
+ }
18
26
  export interface SpeechOptions {
19
27
  fetch?: typeof fetch;
20
28
  /** Resolves the ElevenLabs key; defaults to ELEVENLABS_API_KEY, then the stored key (speech-key.ts). */
@@ -24,9 +32,29 @@ export interface SpeechOptions {
24
32
  /** A fixed message per ElevenLabs failure: its own response text is never
25
33
  * passed on. */
26
34
  export declare function speechError(upstream: Response): Promise<BridgeError>;
35
+ /** A reply as it should sound: markdown the agent wrote for the chat bubble
36
+ * (emphasis, headings, bullets, links, inline code, tables) is read aloud
37
+ * literally by ElevenLabs, so it is reduced to its words. A code block is
38
+ * skipped, a link keeps its text and a bare URL becomes "a link". Empty when
39
+ * nothing speakable is left. */
40
+ export declare function speakableText(text: string): string;
27
41
  /** Starts ElevenLabs' streaming synthesis and returns its audio body. */
28
42
  export declare function synthesizeSpeech(text: string, signal: AbortSignal, options?: SpeechOptions): Promise<ReadableStream<Uint8Array>>;
43
+ /** Voices the text in one piece with ElevenLabs' character alignment. */
44
+ export declare function synthesizeTimedSpeech(text: string, signal: AbortSignal, options?: SpeechOptions): Promise<{
45
+ audio: string;
46
+ alignment: SpeechAlignment | null;
47
+ }>;
48
+ /** ElevenLabs' alignment, kept only when its three lists line up. */
49
+ export declare function alignmentOf(raw: {
50
+ characters?: unknown;
51
+ character_start_times_seconds?: unknown;
52
+ character_end_times_seconds?: unknown;
53
+ } | null | undefined): SpeechAlignment | null;
29
54
  /** POST /v1/speech: writes the audio to the phone as ElevenLabs produces it.
55
+ * With `timestamps`, answers JSON instead: `{ audio, audioFormat, alignment }`,
56
+ * the audio base64 in the same PCM format and the alignment null when
57
+ * ElevenLabs sent none.
30
58
  * Failures before the first byte are thrown for the route's JSON error; a
31
59
  * failure mid-stream cuts the response off, which the phone treats as an
32
60
  * error. The phone hanging up cancels the ElevenLabs request. */
@@ -15,7 +15,12 @@ export const SPEECH_MODEL = "eleven_flash_v2_5";
15
15
  /** River: relaxed, neutral and informative. PHREN_SPEECH_VOICE overrides it. */
16
16
  export const DEFAULT_SPEECH_VOICE = "SAz9YHcvj6GT2YYXdXww";
17
17
  const MAX_TEXT = 2_000;
18
- export const speechRequest = z.object({ text: z.string().trim().min(1).max(MAX_TEXT) });
18
+ export const speechRequest = z.object({
19
+ text: z.string().trim().min(1).max(MAX_TEXT),
20
+ /** Answer JSON with the audio and when each character is spoken, so the
21
+ * phone can highlight the word being read (talk mode's karaoke). */
22
+ timestamps: z.boolean().optional(),
23
+ });
19
24
  /** A fixed message per ElevenLabs failure: its own response text is never
20
25
  * passed on. */
21
26
  export async function speechError(upstream) {
@@ -36,13 +41,75 @@ export async function speechError(upstream) {
36
41
  default: return new BridgeError(502, `ElevenLabs failed (HTTP ${upstream.status}).`, { code: "speech-failed" });
37
42
  }
38
43
  }
44
+ /** A reply as it should sound: markdown the agent wrote for the chat bubble
45
+ * (emphasis, headings, bullets, links, inline code, tables) is read aloud
46
+ * literally by ElevenLabs, so it is reduced to its words. A code block is
47
+ * skipped, a link keeps its text and a bare URL becomes "a link". Empty when
48
+ * nothing speakable is left. */
49
+ export function speakableText(text) {
50
+ return text
51
+ .replace(/```[\s\S]*?(?:```|$)/g, " ")
52
+ .replace(/!\[([^\]]*)\]\([^)]*\)/g, "$1")
53
+ .replace(/\[([^\]]+)\]\([^)]*\)/g, "$1")
54
+ .replace(/https?:\/\/\S+/g, "a link")
55
+ .replace(/`([^`]*)`/g, "$1")
56
+ .split(/\r?\n/)
57
+ .map(line => {
58
+ // A heading, bullet or table row ends where its line does; a
59
+ // paragraph's soft-wrapped lines run on.
60
+ const block = /^\s{0,3}(?:#{1,6}\s|[-*+]\s|\d+[.)]\s|\|)/.test(line);
61
+ const words = line
62
+ .replace(/^\s*\|?[\s:|-]*-[\s:|-]*\|?\s*$/, "")
63
+ .replace(/^\s{0,3}(?:#{1,6}\s+|>\s?|[-*+]\s+(?:\[[ xX]\]\s+)?)/, "")
64
+ .replace(/\s*\|\s*/g, (bar, offset, whole) => offset === 0 || offset + bar.length === whole.length ? "" : ", ")
65
+ .replace(/(\*\*|__|~~)(.+?)\1/g, "$2")
66
+ .replace(/(^|[^\w*])[*_](\S(?:.*?\S)?)[*_](?![\w*])/g, "$1$2")
67
+ .trim();
68
+ return block && words && !/[.!?:;,]$/.test(words) ? `${words}.` : words;
69
+ })
70
+ .filter(Boolean)
71
+ .join(" ")
72
+ .replace(/\s+/g, " ")
73
+ .trim();
74
+ }
39
75
  /** Starts ElevenLabs' streaming synthesis and returns its audio body. */
40
76
  export async function synthesizeSpeech(text, signal, options = {}) {
77
+ const upstream = await elevenLabs("stream", text, signal, options);
78
+ if (!upstream.body)
79
+ throw await speechError(upstream);
80
+ return upstream.body;
81
+ }
82
+ /** Voices the text in one piece with ElevenLabs' character alignment. */
83
+ export async function synthesizeTimedSpeech(text, signal, options = {}) {
84
+ const upstream = await elevenLabs("with-timestamps", text, signal, options);
85
+ let body;
86
+ try {
87
+ body = (await upstream.json());
88
+ }
89
+ catch {
90
+ throw new BridgeError(502, "ElevenLabs sent an unreadable reply.", { code: "speech-failed" });
91
+ }
92
+ if (typeof body.audio_base64 !== "string")
93
+ throw new BridgeError(502, "ElevenLabs sent an unreadable reply.", { code: "speech-failed" });
94
+ return { audio: body.audio_base64, alignment: alignmentOf(body.alignment) };
95
+ }
96
+ /** ElevenLabs' alignment, kept only when its three lists line up. */
97
+ export function alignmentOf(raw) {
98
+ const characters = raw?.characters, starts = raw?.character_start_times_seconds, ends = raw?.character_end_times_seconds;
99
+ if (!Array.isArray(characters) || !Array.isArray(starts) || !Array.isArray(ends))
100
+ return null;
101
+ if (characters.length !== starts.length || characters.length !== ends.length)
102
+ return null;
103
+ if (!characters.every(c => typeof c === "string") || ![...starts, ...ends].every(t => typeof t === "number" && Number.isFinite(t)))
104
+ return null;
105
+ return { characters: characters, starts: starts, ends: ends };
106
+ }
107
+ async function elevenLabs(endpoint, text, signal, options) {
41
108
  const key = await (options.key ?? readSpeechKey)();
42
109
  if (!key)
43
110
  throw new BridgeError(503, "Spoken replies aren't set up on this computer: it has no ElevenLabs key.", { code: "speech-unconfigured" });
44
111
  const voice = options.voice ?? process.env.PHREN_SPEECH_VOICE ?? DEFAULT_SPEECH_VOICE;
45
- const url = `https://api.elevenlabs.io/v1/text-to-speech/${encodeURIComponent(voice)}/stream?output_format=${OUTPUT_FORMAT}`;
112
+ const url = `https://api.elevenlabs.io/v1/text-to-speech/${encodeURIComponent(voice)}/${endpoint}?output_format=${OUTPUT_FORMAT}`;
46
113
  let upstream;
47
114
  try {
48
115
  upstream = await (options.fetch ?? fetch)(url, {
@@ -54,19 +121,34 @@ export async function synthesizeSpeech(text, signal, options = {}) {
54
121
  catch {
55
122
  throw new BridgeError(502, "Couldn't reach ElevenLabs from this computer.", { code: "speech-unreachable" });
56
123
  }
57
- if (!upstream.ok || !upstream.body)
124
+ if (!upstream.ok)
58
125
  throw await speechError(upstream);
59
- return upstream.body;
126
+ return upstream;
60
127
  }
61
128
  /** POST /v1/speech: writes the audio to the phone as ElevenLabs produces it.
129
+ * With `timestamps`, answers JSON instead: `{ audio, audioFormat, alignment }`,
130
+ * the audio base64 in the same PCM format and the alignment null when
131
+ * ElevenLabs sent none.
62
132
  * Failures before the first byte are thrown for the route's JSON error; a
63
133
  * failure mid-stream cuts the response off, which the phone treats as an
64
134
  * error. The phone hanging up cancels the ElevenLabs request. */
65
135
  export async function streamSpeech(data, response, options = {}) {
66
- const { text } = speechRequest.parse(data);
136
+ const request = speechRequest.parse(data);
137
+ const text = speakableText(request.text);
138
+ if (!text)
139
+ throw new BridgeError(400, "There is nothing to say in this reply.", { code: "speech-invalid" });
140
+ // With timestamps, the alignment covers these spoken words, not the markdown.
141
+ const { timestamps } = request;
67
142
  const abort = new AbortController();
68
143
  response.once("close", () => { if (!response.writableEnded)
69
144
  abort.abort(); });
145
+ if (timestamps) {
146
+ const { audio, alignment } = await synthesizeTimedSpeech(text, abort.signal, options);
147
+ response.statusCode = 200;
148
+ response.setHeader("Content-Type", "application/json");
149
+ response.end(JSON.stringify({ audio, audioFormat: SPEECH_AUDIO, alignment }));
150
+ return;
151
+ }
70
152
  const audio = await synthesizeSpeech(text, abort.signal, options);
71
153
  response.statusCode = 200;
72
154
  response.setHeader("Content-Type", "application/octet-stream");