@phren/cli 0.3.1 → 0.3.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bridge/agent-hooks.d.ts +5 -0
- package/dist/bridge/agent-hooks.js +19 -0
- package/dist/bridge/command.d.ts +3 -2
- package/dist/bridge/command.js +5 -3
- package/dist/bridge/push.d.ts +42 -3
- package/dist/bridge/push.js +90 -20
- package/dist/bridge/server-pane-routes.js +11 -5
- package/dist/bridge/server-routes.d.ts +1 -0
- package/dist/bridge/server-routes.js +1 -1
- package/dist/bridge/speech.d.ts +28 -0
- package/dist/bridge/speech.js +87 -5
- package/dist/bridge-hook.mjs +206 -31
- package/dist/cli/namespaces-store.js +83 -2
- package/dist/cli-registry.js +1 -1
- package/dist/init/init-walkthrough.js +1 -1
- package/dist/link/link.js +9 -3
- package/dist/link/refresh.js +11 -10
- package/dist/modules/registry.js +1 -1
- package/dist/sync/conflict-resolve.d.ts +1 -1
- package/dist/sync/conflict-resolve.js +44 -7
- package/dist/tools/search.js +1 -1
- package/package.json +1 -1
|
@@ -195,6 +195,11 @@ export declare class AgentHooks {
|
|
|
195
195
|
* phone with its own keys: let the callback fall back to the terminal at
|
|
196
196
|
* once instead of leaving the card up until the 55-second timer. */
|
|
197
197
|
releaseChoice(target: Target): void;
|
|
198
|
+
/** The phone's Esc or Cancel on a question the Hook holds. A held
|
|
199
|
+
* AskUserQuestion has no terminal choice, so `releaseChoice` never lets it
|
|
200
|
+
* go and the terminal stays frozen until the hold timer; decline it now so
|
|
201
|
+
* Claude moves on. True when a held question was cancelled. */
|
|
202
|
+
cancelHeldQuestion(target: Target): boolean;
|
|
198
203
|
answerPush(binding: string, decision: unknown): Promise<void>;
|
|
199
204
|
/** Where a pushed ask lives, without answering it: the phone opens that
|
|
200
205
|
* session's details when the notification itself is tapped. */
|
|
@@ -809,6 +809,25 @@ export class AgentHooks {
|
|
|
809
809
|
entry.response.end("{}");
|
|
810
810
|
}
|
|
811
811
|
}
|
|
812
|
+
/** The phone's Esc or Cancel on a question the Hook holds. A held
|
|
813
|
+
* AskUserQuestion has no terminal choice, so `releaseChoice` never lets it
|
|
814
|
+
* go and the terminal stays frozen until the hold timer; decline it now so
|
|
815
|
+
* Claude moves on. True when a held question was cancelled. */
|
|
816
|
+
cancelHeldQuestion(target) {
|
|
817
|
+
let cancelled = false;
|
|
818
|
+
for (const [id, entry] of this.pending) {
|
|
819
|
+
if (entry.tool !== "AskUserQuestion" || JSON.stringify(entry.target) !== JSON.stringify(target))
|
|
820
|
+
continue;
|
|
821
|
+
this.pending.delete(id);
|
|
822
|
+
this.dropPushBindings(id);
|
|
823
|
+
clearTimeout(entry.timer);
|
|
824
|
+
if (!entry.response.destroyed)
|
|
825
|
+
entry.response.end(JSON.stringify({ hookSpecificOutput: { hookEventName: "PermissionRequest",
|
|
826
|
+
decision: { behavior: "deny", message: "The user cancelled this question in Phren without answering." } } }));
|
|
827
|
+
cancelled = true;
|
|
828
|
+
}
|
|
829
|
+
return cancelled;
|
|
830
|
+
}
|
|
812
831
|
async answerPush(binding, decision) {
|
|
813
832
|
if (!["approve", "deny"].includes(String(decision)))
|
|
814
833
|
throw new BridgeError(400, "The approval answer is not valid.");
|
package/dist/bridge/command.d.ts
CHANGED
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
import { type Json } from "./protocol.js";
|
|
2
2
|
export declare function runBridge(args: string[], version: string): Promise<number>;
|
|
3
|
-
/** Doctor's push check, from the running Hook's own capability:
|
|
4
|
-
*
|
|
3
|
+
/** Doctor's push check, from the running Hook's own capability: a Hook that
|
|
4
|
+
* loaded apns.json and its key, or one whose phones registered through the
|
|
5
|
+
* push relay, offers `approvalPush`. */
|
|
5
6
|
export declare function approvalPushCheck(helper: Json): {
|
|
6
7
|
configured: boolean;
|
|
7
8
|
warning?: string;
|
package/dist/bridge/command.js
CHANGED
|
@@ -131,10 +131,12 @@ export async function runBridge(args, version) {
|
|
|
131
131
|
}
|
|
132
132
|
return 0;
|
|
133
133
|
}
|
|
134
|
-
/** Doctor's push check, from the running Hook's own capability:
|
|
135
|
-
*
|
|
134
|
+
/** Doctor's push check, from the running Hook's own capability: a Hook that
|
|
135
|
+
* loaded apns.json and its key, or one whose phones registered through the
|
|
136
|
+
* push relay, offers `approvalPush`. */
|
|
136
137
|
export function approvalPushCheck(helper) {
|
|
137
|
-
const
|
|
138
|
+
const capability = object(helper.capabilities).approvalPush;
|
|
139
|
+
const configured = capability === "direct-apns" || capability === "relay";
|
|
138
140
|
return configured ? { configured } : { configured, warning: apnsSetupSteps() };
|
|
139
141
|
}
|
|
140
142
|
const SPEECH_KEY_USAGE = "Usage: phren bridge speech-key set (paste the key when asked, or pipe it on stdin)";
|
package/dist/bridge/push.d.ts
CHANGED
|
@@ -1,8 +1,23 @@
|
|
|
1
1
|
import { z } from "zod";
|
|
2
|
+
/** A phone registered through the phren push relay: the relay knows where to
|
|
3
|
+
* deliver, and `key` (32 bytes, base64url) encrypts what the notification says
|
|
4
|
+
* so only the phone reads it. The phone made the key and sent it over SSH. */
|
|
5
|
+
declare const relaySchema: z.ZodObject<{
|
|
6
|
+
url: z.ZodString;
|
|
7
|
+
relayId: z.ZodString;
|
|
8
|
+
secret: z.ZodString;
|
|
9
|
+
key: z.ZodString;
|
|
10
|
+
}, z.core.$strip>;
|
|
2
11
|
declare const deviceSchema: z.ZodObject<{
|
|
3
12
|
deviceID: z.ZodString;
|
|
4
13
|
hostID: z.ZodString;
|
|
5
|
-
token: z.ZodString
|
|
14
|
+
token: z.ZodOptional<z.ZodString>;
|
|
15
|
+
relay: z.ZodOptional<z.ZodObject<{
|
|
16
|
+
url: z.ZodString;
|
|
17
|
+
relayId: z.ZodString;
|
|
18
|
+
secret: z.ZodString;
|
|
19
|
+
key: z.ZodString;
|
|
20
|
+
}, z.core.$strip>>;
|
|
6
21
|
environment: z.ZodEnum<{
|
|
7
22
|
development: "development";
|
|
8
23
|
production: "production";
|
|
@@ -54,11 +69,29 @@ export declare function fanoutBlockedPushPayload(value: FanoutBlockedPush): Reco
|
|
|
54
69
|
export declare function schedulePushPayload(value: SchedulePush): Record<string, unknown>;
|
|
55
70
|
/** What `phren bridge doctor` prints when the Hook has no APNs sender. */
|
|
56
71
|
export declare function apnsSetupSteps(configFile?: string): string;
|
|
57
|
-
/** The capability a phone reads:
|
|
72
|
+
/** The capability a phone reads: a Hook with its own APNs key sends direct;
|
|
73
|
+
* one whose phones registered through the relay sends through it. */
|
|
58
74
|
export declare function approvalPushCapability(status: {
|
|
59
75
|
configured: boolean;
|
|
60
|
-
|
|
76
|
+
direct?: boolean;
|
|
77
|
+
}): "direct-apns" | "relay" | undefined;
|
|
61
78
|
export declare function upsertPushDevice(devices: PushDevice[], value: unknown): PushDevice[];
|
|
79
|
+
/** The relay refuses ciphertext longer than this (it keeps the APNs payload under 4 KB). */
|
|
80
|
+
export declare const RELAY_MAX_CIPHERTEXT = 2800;
|
|
81
|
+
/** What the phone's notification extension shows, encrypted with the phone's
|
|
82
|
+
* key: ChaCha20-Poly1305, base64url of nonce (12) + ciphertext + tag (16), the
|
|
83
|
+
* layout CryptoKit's `ChaChaPoly.SealedBox(combined:)` reads. A long body is
|
|
84
|
+
* shortened until it fits the relay's limit. */
|
|
85
|
+
export declare function relayCiphertext(key: string, payload: Record<string, unknown>, nonce?: NonSharedBuffer): string;
|
|
86
|
+
/** Signs a relay send: base64url HMAC-SHA256 of `${timestamp}.${body}` under the phone's send secret. */
|
|
87
|
+
export declare function relaySignature(secret: string, timestamp: string, body: string): string;
|
|
88
|
+
type RelayRegistration = z.infer<typeof relaySchema>;
|
|
89
|
+
/** "gone" means the relay (Apple) no longer knows the phone: stop sending until it registers again. */
|
|
90
|
+
export type RelayResult = "sent" | "failed" | "gone";
|
|
91
|
+
export declare function sendThroughRelay(relay: RelayRegistration, payload: Record<string, unknown>, headers: {
|
|
92
|
+
expiration: string;
|
|
93
|
+
collapseId: string;
|
|
94
|
+
}, fetcher?: typeof fetch, now?: () => number): Promise<RelayResult>;
|
|
62
95
|
/** Direct Hook -> APNs delivery. Apple receives only a short-lived opaque
|
|
63
96
|
* binding; conversation identity, action id, command and SSH key stay local. */
|
|
64
97
|
export declare class ApprovalPushService {
|
|
@@ -66,13 +99,19 @@ export declare class ApprovalPushService {
|
|
|
66
99
|
private sender?;
|
|
67
100
|
private readonly devicesFile;
|
|
68
101
|
start(): Promise<void>;
|
|
102
|
+
/** Phones this Hook can reach: through the relay, or direct with its own key. */
|
|
103
|
+
private get reachable();
|
|
69
104
|
get available(): boolean;
|
|
70
105
|
get status(): {
|
|
71
106
|
supported: boolean;
|
|
72
107
|
configured: boolean;
|
|
108
|
+
direct: boolean;
|
|
73
109
|
devices: number;
|
|
110
|
+
relay: number;
|
|
74
111
|
};
|
|
75
112
|
register(value: unknown): Promise<void>;
|
|
113
|
+
/** One phone: the relay when it registered through one, otherwise direct. */
|
|
114
|
+
private send;
|
|
76
115
|
notify(value: ApprovalPush): Promise<boolean>;
|
|
77
116
|
/** A headless worker whose permission the plugin refused. Approval-registered
|
|
78
117
|
* phones already accept agent alerts; the payload carries the reason so the
|
package/dist/bridge/push.js
CHANGED
|
@@ -1,16 +1,27 @@
|
|
|
1
|
-
import { createPrivateKey, sign } from "node:crypto";
|
|
1
|
+
import { createCipheriv, createHmac, createPrivateKey, randomBytes, sign } from "node:crypto";
|
|
2
2
|
import { readFile, stat } from "node:fs/promises";
|
|
3
3
|
import { connect } from "node:http2";
|
|
4
4
|
import path from "node:path";
|
|
5
5
|
import { z } from "zod";
|
|
6
6
|
import { atomic, bridgeRoot } from "./protocol.js";
|
|
7
|
+
/** A phone registered through the phren push relay: the relay knows where to
|
|
8
|
+
* deliver, and `key` (32 bytes, base64url) encrypts what the notification says
|
|
9
|
+
* so only the phone reads it. The phone made the key and sent it over SSH. */
|
|
10
|
+
const relaySchema = z.object({
|
|
11
|
+
url: z.string().url().refine(value => /^https:\/\//.test(value) || /^http:\/\/(127\.0\.0\.1|localhost)(:\d+)?\//.test(`${value}/`), "The relay must use https."),
|
|
12
|
+
relayId: z.string().regex(/^[A-Za-z0-9_-]{40,400}$/),
|
|
13
|
+
secret: z.string().regex(/^[A-Za-z0-9_-]{20,100}$/),
|
|
14
|
+
key: z.string().regex(/^[A-Za-z0-9_-]{43}$/),
|
|
15
|
+
});
|
|
7
16
|
const deviceSchema = z.object({
|
|
8
17
|
deviceID: z.string().uuid(),
|
|
9
18
|
hostID: z.string().uuid(),
|
|
10
|
-
token
|
|
19
|
+
/** The APNs token, for a Hook with its own key (apns.json). */
|
|
20
|
+
token: z.string().regex(/^[0-9a-f]{64}$/).optional(),
|
|
21
|
+
relay: relaySchema.optional(),
|
|
11
22
|
environment: z.enum(["development", "production"]),
|
|
12
23
|
kinds: z.array(z.enum(["approval", "scheduleStarted", "scheduleFinished", "scheduleFailed", "scheduleBlocked"])).max(5).default(["approval"]),
|
|
13
|
-
});
|
|
24
|
+
}).refine(device => device.token !== undefined || device.relay !== undefined, "A phone needs an APNs token or a relay registration.");
|
|
14
25
|
const configSchema = z.object({
|
|
15
26
|
keyId: z.string().regex(/^[A-Z0-9]{10}$/),
|
|
16
27
|
teamId: z.string().regex(/^[A-Z0-9]{10}$/),
|
|
@@ -83,15 +94,59 @@ export function apnsSetupSteps(configFile = process.env.PHREN_APNS_CONFIG || pat
|
|
|
83
94
|
"4. Restart the Hook (phren bridge install), then run phren bridge doctor again.",
|
|
84
95
|
].join("\n");
|
|
85
96
|
}
|
|
86
|
-
/** The capability a phone reads:
|
|
97
|
+
/** The capability a phone reads: a Hook with its own APNs key sends direct;
|
|
98
|
+
* one whose phones registered through the relay sends through it. */
|
|
87
99
|
export function approvalPushCapability(status) {
|
|
88
|
-
|
|
100
|
+
if (!status.configured)
|
|
101
|
+
return undefined;
|
|
102
|
+
return status.direct === false ? "relay" : "direct-apns";
|
|
89
103
|
}
|
|
90
104
|
export function upsertPushDevice(devices, value) {
|
|
91
105
|
const device = deviceSchema.parse(value);
|
|
92
106
|
return [...devices.filter(item => item.deviceID !== device.deviceID), device].slice(-16);
|
|
93
107
|
}
|
|
94
108
|
const base64url = (value) => Buffer.from(value).toString("base64url");
|
|
109
|
+
/** The relay refuses ciphertext longer than this (it keeps the APNs payload under 4 KB). */
|
|
110
|
+
export const RELAY_MAX_CIPHERTEXT = 2_800;
|
|
111
|
+
/** What the phone's notification extension shows, encrypted with the phone's
|
|
112
|
+
* key: ChaCha20-Poly1305, base64url of nonce (12) + ciphertext + tag (16), the
|
|
113
|
+
* layout CryptoKit's `ChaChaPoly.SealedBox(combined:)` reads. A long body is
|
|
114
|
+
* shortened until it fits the relay's limit. */
|
|
115
|
+
export function relayCiphertext(key, payload, nonce = randomBytes(12)) {
|
|
116
|
+
const aps = (payload.aps ?? {});
|
|
117
|
+
let body = aps.alert?.body ?? "";
|
|
118
|
+
for (;;) {
|
|
119
|
+
const content = JSON.stringify({ t: aps.alert?.title ?? "phren", b: body, ...(aps.category ? { c: aps.category } : {}),
|
|
120
|
+
...(aps["interruption-level"] ? { i: aps["interruption-level"] } : {}), ...(payload.phren ? { p: payload.phren } : {}) });
|
|
121
|
+
const cipher = createCipheriv("chacha20-poly1305", Buffer.from(key, "base64url"), nonce, { authTagLength: 16 });
|
|
122
|
+
const sealed = Buffer.concat([nonce, cipher.update(content, "utf8"), cipher.final(), cipher.getAuthTag()]).toString("base64url");
|
|
123
|
+
if (sealed.length <= RELAY_MAX_CIPHERTEXT || !body)
|
|
124
|
+
return sealed;
|
|
125
|
+
body = body.length > 40 ? `${body.slice(0, Math.floor(body.length * 0.8))}…` : "";
|
|
126
|
+
}
|
|
127
|
+
}
|
|
128
|
+
/** Signs a relay send: base64url HMAC-SHA256 of `${timestamp}.${body}` under the phone's send secret. */
|
|
129
|
+
export function relaySignature(secret, timestamp, body) {
|
|
130
|
+
return createHmac("sha256", secret).update(`${timestamp}.${body}`).digest("base64url");
|
|
131
|
+
}
|
|
132
|
+
export async function sendThroughRelay(relay, payload, headers, fetcher = fetch, now = Date.now) {
|
|
133
|
+
const aps = (payload.aps ?? {});
|
|
134
|
+
const body = JSON.stringify({ kind: "alert", ciphertext: relayCiphertext(relay.key, payload),
|
|
135
|
+
collapseId: headers.collapseId.slice(0, 64), expiration: Number(headers.expiration) || 0,
|
|
136
|
+
...(aps.category ? { category: aps.category } : {}) });
|
|
137
|
+
const timestamp = String(Math.floor(now() / 1000));
|
|
138
|
+
try {
|
|
139
|
+
const response = await fetcher(`${relay.url.replace(/\/+$/, "")}/v1/send`, {
|
|
140
|
+
method: "POST", body, signal: AbortSignal.timeout(10_000),
|
|
141
|
+
headers: { "content-type": "application/json", "x-phren-relay": relay.relayId, "x-phren-timestamp": timestamp,
|
|
142
|
+
"x-phren-signature": relaySignature(relay.secret, timestamp, body) },
|
|
143
|
+
});
|
|
144
|
+
return response.status === 410 ? "gone" : response.ok ? "sent" : "failed";
|
|
145
|
+
}
|
|
146
|
+
catch {
|
|
147
|
+
return "failed";
|
|
148
|
+
}
|
|
149
|
+
}
|
|
95
150
|
class APNsSender {
|
|
96
151
|
config;
|
|
97
152
|
key;
|
|
@@ -149,19 +204,35 @@ export class ApprovalPushService {
|
|
|
149
204
|
if (key)
|
|
150
205
|
this.sender = new APNsSender(config, key);
|
|
151
206
|
}
|
|
152
|
-
|
|
153
|
-
get
|
|
207
|
+
/** Phones this Hook can reach: through the relay, or direct with its own key. */
|
|
208
|
+
get reachable() { return this.devices.filter(device => device.relay || (this.sender && device.token)); }
|
|
209
|
+
get available() { return this.reachable.length > 0; }
|
|
210
|
+
get status() {
|
|
211
|
+
const relay = this.devices.filter(device => device.relay).length;
|
|
212
|
+
return { supported: true, configured: this.sender !== undefined || relay > 0, direct: this.sender !== undefined,
|
|
213
|
+
devices: this.devices.length, relay };
|
|
214
|
+
}
|
|
154
215
|
async register(value) {
|
|
155
216
|
this.devices = upsertPushDevice(this.devices, value);
|
|
156
217
|
await atomic(this.devicesFile, JSON.stringify(this.devices));
|
|
157
218
|
}
|
|
219
|
+
/** One phone: the relay when it registered through one, otherwise direct. */
|
|
220
|
+
async send(device, payload, headers) {
|
|
221
|
+
if (device.relay) {
|
|
222
|
+
const result = await sendThroughRelay(device.relay, payload, headers);
|
|
223
|
+
if (result === "gone") {
|
|
224
|
+
this.devices = this.devices.filter(item => item.deviceID !== device.deviceID);
|
|
225
|
+
await atomic(this.devicesFile, JSON.stringify(this.devices)).catch(() => { });
|
|
226
|
+
}
|
|
227
|
+
return result === "sent";
|
|
228
|
+
}
|
|
229
|
+
return this.sender && device.token ? this.sender.send(device, payload, headers) : false;
|
|
230
|
+
}
|
|
158
231
|
async notify(value) {
|
|
159
|
-
|
|
160
|
-
return false;
|
|
161
|
-
const devices = this.devices.filter(device => device.kinds.includes("approval"));
|
|
232
|
+
const devices = this.reachable.filter(device => device.kinds.includes("approval"));
|
|
162
233
|
if (!devices.length)
|
|
163
234
|
return false;
|
|
164
|
-
return (await Promise.all(devices.map(device => this.
|
|
235
|
+
return (await Promise.all(devices.map(device => this.send(device, approvalPushPayload(value, device.hostID), {
|
|
165
236
|
expiration: String(Math.floor(Date.parse(value.expiresAt) / 1000)), collapseId: value.binding,
|
|
166
237
|
})))).some(Boolean);
|
|
167
238
|
}
|
|
@@ -169,25 +240,24 @@ export class ApprovalPushService {
|
|
|
169
240
|
* phones already accept agent alerts; the payload carries the reason so the
|
|
170
241
|
* notification is actionable on its own. */
|
|
171
242
|
async notifyFanoutBlocked(value) {
|
|
172
|
-
|
|
173
|
-
return false;
|
|
174
|
-
const devices = this.devices.filter(device => device.kinds.includes("approval"));
|
|
243
|
+
const devices = this.reachable.filter(device => device.kinds.includes("approval"));
|
|
175
244
|
if (!devices.length)
|
|
176
245
|
return false;
|
|
177
|
-
return (await Promise.all(devices.map(device => this.
|
|
246
|
+
return (await Promise.all(devices.map(device => this.send(device, fanoutBlockedPushPayload(value), {
|
|
178
247
|
expiration: "0", collapseId: `fanout-${value.job}`,
|
|
179
248
|
})))).some(Boolean);
|
|
180
249
|
}
|
|
181
250
|
async notifySchedule(value) {
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
251
|
+
const reachable = this.reachable;
|
|
252
|
+
if (!reachable.length)
|
|
253
|
+
return { notified: false, reason: this.sender || this.devices.length ? "no registered devices" : "no push config" };
|
|
254
|
+
let devices = reachable.filter(device => device.kinds.includes(value.kind));
|
|
185
255
|
if (!devices.length && value.kind === "scheduleBlocked")
|
|
186
|
-
devices =
|
|
256
|
+
devices = reachable.filter(device => device.kinds.includes("scheduleFailed"));
|
|
187
257
|
if (!devices.length)
|
|
188
258
|
return { notified: false, reason: "no registered devices" };
|
|
189
259
|
try {
|
|
190
|
-
const notified = (await Promise.all(devices.map(device => this.
|
|
260
|
+
const notified = (await Promise.all(devices.map(device => this.send(device, schedulePushPayload(value), {
|
|
191
261
|
expiration: "0", collapseId: scheduleCollapseId(value.kind, value.runId),
|
|
192
262
|
})))).some(Boolean);
|
|
193
263
|
return notified ? { notified: true } : { notified: false, reason: "push delivery failed" };
|
|
@@ -364,15 +364,21 @@ export async function paneRoute(ctx, url, data, response) {
|
|
|
364
364
|
// not hold) is being answered even when Herdr reads the pane as
|
|
365
365
|
// working or idle; Herdr's status lags the agent's own dialog.
|
|
366
366
|
const holding = menu || !!agentHooks.terminalPrompt(target);
|
|
367
|
+
// Esc on a question the Hook still holds declines it at once, whatever
|
|
368
|
+
// Herdr reads the pane as: an esc typed into the held pane does nothing.
|
|
369
|
+
const cancelled = keys.every(key => key === "Escape") && agentHooks.cancelHeldQuestion(target);
|
|
367
370
|
// Escape interrupts a working agent. Everything else answers a
|
|
368
371
|
// prompt the agent is holding: a menu, a y/n, a trust question.
|
|
369
|
-
if (!holding && (keys.every(key => key === "Escape") ? !["working", "blocked", "waiting", "unknown"].includes(status)
|
|
372
|
+
if (!cancelled && !holding && (keys.every(key => key === "Escape") ? !["working", "blocked", "waiting", "unknown"].includes(status)
|
|
370
373
|
: !["blocked", "waiting", "unknown"].includes(status)))
|
|
371
374
|
throw new BridgeError(409, keys.every(key => key === "Escape") ? "This agent is no longer working." : "This agent is not waiting for an answer.");
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
375
|
+
if (cancelled) {
|
|
376
|
+
result = { ok: true };
|
|
377
|
+
// An older phone answers a released AskUserQuestion one question at
|
|
378
|
+
// a time with its digits; the Hook walks that question in the pane
|
|
379
|
+
// and submits the set after the last.
|
|
380
|
+
}
|
|
381
|
+
else if (await agentHooks.answerReleasedQuestion(target, keys)) {
|
|
376
382
|
result = { ok: true };
|
|
377
383
|
}
|
|
378
384
|
else {
|
|
@@ -51,7 +51,7 @@ async function childActivity(source, session) {
|
|
|
51
51
|
export const capabilities = { transcript: true, progress: true, images: true, prompt: true, stop: true,
|
|
52
52
|
terminal: "ssh-pty", shell: "ssh-pty", herdr: true, diff: true, webServers: true, webPreview: "ssh-exec", activity: true,
|
|
53
53
|
approvals: true, questions: false, accountUsage: true, providers: ["codex", "claude", "copilot", "opencode"],
|
|
54
|
-
files: true, repositoryFiles: true, subagents: true, sideQuestions: true, dispatch: true, approvalPush: "direct-apns", simulators: process.platform === "darwin", code: true, overviewStream: true, speech: true, transcribe: true, memoryStore: true };
|
|
54
|
+
files: true, repositoryFiles: true, subagents: true, sideQuestions: true, dispatch: true, approvalPush: "direct-apns", simulators: process.platform === "darwin", code: true, overviewStream: true, speech: true, speechTimestamps: true, transcribe: true, memoryStore: true };
|
|
55
55
|
export function capabilitiesForModules(snapshot) {
|
|
56
56
|
const allowed = new Set(snapshot.modules.flatMap(module => module.capabilities));
|
|
57
57
|
const result = Object.fromEntries(Object.entries(capabilities).filter(([name]) => allowed.has(name)));
|
package/dist/bridge/speech.d.ts
CHANGED
|
@@ -14,7 +14,15 @@ export declare const SPEECH_MODEL = "eleven_flash_v2_5";
|
|
|
14
14
|
export declare const DEFAULT_SPEECH_VOICE = "SAz9YHcvj6GT2YYXdXww";
|
|
15
15
|
export declare const speechRequest: z.ZodObject<{
|
|
16
16
|
text: z.ZodString;
|
|
17
|
+
timestamps: z.ZodOptional<z.ZodBoolean>;
|
|
17
18
|
}, z.core.$strip>;
|
|
19
|
+
/** When each character of the voiced text starts and ends, in seconds from
|
|
20
|
+
* the start of the audio. */
|
|
21
|
+
export interface SpeechAlignment {
|
|
22
|
+
characters: string[];
|
|
23
|
+
starts: number[];
|
|
24
|
+
ends: number[];
|
|
25
|
+
}
|
|
18
26
|
export interface SpeechOptions {
|
|
19
27
|
fetch?: typeof fetch;
|
|
20
28
|
/** Resolves the ElevenLabs key; defaults to ELEVENLABS_API_KEY, then the stored key (speech-key.ts). */
|
|
@@ -24,9 +32,29 @@ export interface SpeechOptions {
|
|
|
24
32
|
/** A fixed message per ElevenLabs failure: its own response text is never
|
|
25
33
|
* passed on. */
|
|
26
34
|
export declare function speechError(upstream: Response): Promise<BridgeError>;
|
|
35
|
+
/** A reply as it should sound: markdown the agent wrote for the chat bubble
|
|
36
|
+
* (emphasis, headings, bullets, links, inline code, tables) is read aloud
|
|
37
|
+
* literally by ElevenLabs, so it is reduced to its words. A code block is
|
|
38
|
+
* skipped, a link keeps its text and a bare URL becomes "a link". Empty when
|
|
39
|
+
* nothing speakable is left. */
|
|
40
|
+
export declare function speakableText(text: string): string;
|
|
27
41
|
/** Starts ElevenLabs' streaming synthesis and returns its audio body. */
|
|
28
42
|
export declare function synthesizeSpeech(text: string, signal: AbortSignal, options?: SpeechOptions): Promise<ReadableStream<Uint8Array>>;
|
|
43
|
+
/** Voices the text in one piece with ElevenLabs' character alignment. */
|
|
44
|
+
export declare function synthesizeTimedSpeech(text: string, signal: AbortSignal, options?: SpeechOptions): Promise<{
|
|
45
|
+
audio: string;
|
|
46
|
+
alignment: SpeechAlignment | null;
|
|
47
|
+
}>;
|
|
48
|
+
/** ElevenLabs' alignment, kept only when its three lists line up. */
|
|
49
|
+
export declare function alignmentOf(raw: {
|
|
50
|
+
characters?: unknown;
|
|
51
|
+
character_start_times_seconds?: unknown;
|
|
52
|
+
character_end_times_seconds?: unknown;
|
|
53
|
+
} | null | undefined): SpeechAlignment | null;
|
|
29
54
|
/** POST /v1/speech: writes the audio to the phone as ElevenLabs produces it.
|
|
55
|
+
* With `timestamps`, answers JSON instead: `{ audio, audioFormat, alignment }`,
|
|
56
|
+
* the audio base64 in the same PCM format and the alignment null when
|
|
57
|
+
* ElevenLabs sent none.
|
|
30
58
|
* Failures before the first byte are thrown for the route's JSON error; a
|
|
31
59
|
* failure mid-stream cuts the response off, which the phone treats as an
|
|
32
60
|
* error. The phone hanging up cancels the ElevenLabs request. */
|
package/dist/bridge/speech.js
CHANGED
|
@@ -15,7 +15,12 @@ export const SPEECH_MODEL = "eleven_flash_v2_5";
|
|
|
15
15
|
/** River: relaxed, neutral and informative. PHREN_SPEECH_VOICE overrides it. */
|
|
16
16
|
export const DEFAULT_SPEECH_VOICE = "SAz9YHcvj6GT2YYXdXww";
|
|
17
17
|
const MAX_TEXT = 2_000;
|
|
18
|
-
export const speechRequest = z.object({
|
|
18
|
+
export const speechRequest = z.object({
|
|
19
|
+
text: z.string().trim().min(1).max(MAX_TEXT),
|
|
20
|
+
/** Answer JSON with the audio and when each character is spoken, so the
|
|
21
|
+
* phone can highlight the word being read (talk mode's karaoke). */
|
|
22
|
+
timestamps: z.boolean().optional(),
|
|
23
|
+
});
|
|
19
24
|
/** A fixed message per ElevenLabs failure: its own response text is never
|
|
20
25
|
* passed on. */
|
|
21
26
|
export async function speechError(upstream) {
|
|
@@ -36,13 +41,75 @@ export async function speechError(upstream) {
|
|
|
36
41
|
default: return new BridgeError(502, `ElevenLabs failed (HTTP ${upstream.status}).`, { code: "speech-failed" });
|
|
37
42
|
}
|
|
38
43
|
}
|
|
44
|
+
/** A reply as it should sound: markdown the agent wrote for the chat bubble
|
|
45
|
+
* (emphasis, headings, bullets, links, inline code, tables) is read aloud
|
|
46
|
+
* literally by ElevenLabs, so it is reduced to its words. A code block is
|
|
47
|
+
* skipped, a link keeps its text and a bare URL becomes "a link". Empty when
|
|
48
|
+
* nothing speakable is left. */
|
|
49
|
+
export function speakableText(text) {
|
|
50
|
+
return text
|
|
51
|
+
.replace(/```[\s\S]*?(?:```|$)/g, " ")
|
|
52
|
+
.replace(/!\[([^\]]*)\]\([^)]*\)/g, "$1")
|
|
53
|
+
.replace(/\[([^\]]+)\]\([^)]*\)/g, "$1")
|
|
54
|
+
.replace(/https?:\/\/\S+/g, "a link")
|
|
55
|
+
.replace(/`([^`]*)`/g, "$1")
|
|
56
|
+
.split(/\r?\n/)
|
|
57
|
+
.map(line => {
|
|
58
|
+
// A heading, bullet or table row ends where its line does; a
|
|
59
|
+
// paragraph's soft-wrapped lines run on.
|
|
60
|
+
const block = /^\s{0,3}(?:#{1,6}\s|[-*+]\s|\d+[.)]\s|\|)/.test(line);
|
|
61
|
+
const words = line
|
|
62
|
+
.replace(/^\s*\|?[\s:|-]*-[\s:|-]*\|?\s*$/, "")
|
|
63
|
+
.replace(/^\s{0,3}(?:#{1,6}\s+|>\s?|[-*+]\s+(?:\[[ xX]\]\s+)?)/, "")
|
|
64
|
+
.replace(/\s*\|\s*/g, (bar, offset, whole) => offset === 0 || offset + bar.length === whole.length ? "" : ", ")
|
|
65
|
+
.replace(/(\*\*|__|~~)(.+?)\1/g, "$2")
|
|
66
|
+
.replace(/(^|[^\w*])[*_](\S(?:.*?\S)?)[*_](?![\w*])/g, "$1$2")
|
|
67
|
+
.trim();
|
|
68
|
+
return block && words && !/[.!?:;,]$/.test(words) ? `${words}.` : words;
|
|
69
|
+
})
|
|
70
|
+
.filter(Boolean)
|
|
71
|
+
.join(" ")
|
|
72
|
+
.replace(/\s+/g, " ")
|
|
73
|
+
.trim();
|
|
74
|
+
}
|
|
39
75
|
/** Starts ElevenLabs' streaming synthesis and returns its audio body. */
|
|
40
76
|
export async function synthesizeSpeech(text, signal, options = {}) {
|
|
77
|
+
const upstream = await elevenLabs("stream", text, signal, options);
|
|
78
|
+
if (!upstream.body)
|
|
79
|
+
throw await speechError(upstream);
|
|
80
|
+
return upstream.body;
|
|
81
|
+
}
|
|
82
|
+
/** Voices the text in one piece with ElevenLabs' character alignment. */
|
|
83
|
+
export async function synthesizeTimedSpeech(text, signal, options = {}) {
|
|
84
|
+
const upstream = await elevenLabs("with-timestamps", text, signal, options);
|
|
85
|
+
let body;
|
|
86
|
+
try {
|
|
87
|
+
body = (await upstream.json());
|
|
88
|
+
}
|
|
89
|
+
catch {
|
|
90
|
+
throw new BridgeError(502, "ElevenLabs sent an unreadable reply.", { code: "speech-failed" });
|
|
91
|
+
}
|
|
92
|
+
if (typeof body.audio_base64 !== "string")
|
|
93
|
+
throw new BridgeError(502, "ElevenLabs sent an unreadable reply.", { code: "speech-failed" });
|
|
94
|
+
return { audio: body.audio_base64, alignment: alignmentOf(body.alignment) };
|
|
95
|
+
}
|
|
96
|
+
/** ElevenLabs' alignment, kept only when its three lists line up. */
|
|
97
|
+
export function alignmentOf(raw) {
|
|
98
|
+
const characters = raw?.characters, starts = raw?.character_start_times_seconds, ends = raw?.character_end_times_seconds;
|
|
99
|
+
if (!Array.isArray(characters) || !Array.isArray(starts) || !Array.isArray(ends))
|
|
100
|
+
return null;
|
|
101
|
+
if (characters.length !== starts.length || characters.length !== ends.length)
|
|
102
|
+
return null;
|
|
103
|
+
if (!characters.every(c => typeof c === "string") || ![...starts, ...ends].every(t => typeof t === "number" && Number.isFinite(t)))
|
|
104
|
+
return null;
|
|
105
|
+
return { characters: characters, starts: starts, ends: ends };
|
|
106
|
+
}
|
|
107
|
+
async function elevenLabs(endpoint, text, signal, options) {
|
|
41
108
|
const key = await (options.key ?? readSpeechKey)();
|
|
42
109
|
if (!key)
|
|
43
110
|
throw new BridgeError(503, "Spoken replies aren't set up on this computer: it has no ElevenLabs key.", { code: "speech-unconfigured" });
|
|
44
111
|
const voice = options.voice ?? process.env.PHREN_SPEECH_VOICE ?? DEFAULT_SPEECH_VOICE;
|
|
45
|
-
const url = `https://api.elevenlabs.io/v1/text-to-speech/${encodeURIComponent(voice)}
|
|
112
|
+
const url = `https://api.elevenlabs.io/v1/text-to-speech/${encodeURIComponent(voice)}/${endpoint}?output_format=${OUTPUT_FORMAT}`;
|
|
46
113
|
let upstream;
|
|
47
114
|
try {
|
|
48
115
|
upstream = await (options.fetch ?? fetch)(url, {
|
|
@@ -54,19 +121,34 @@ export async function synthesizeSpeech(text, signal, options = {}) {
|
|
|
54
121
|
catch {
|
|
55
122
|
throw new BridgeError(502, "Couldn't reach ElevenLabs from this computer.", { code: "speech-unreachable" });
|
|
56
123
|
}
|
|
57
|
-
if (!upstream.ok
|
|
124
|
+
if (!upstream.ok)
|
|
58
125
|
throw await speechError(upstream);
|
|
59
|
-
return upstream
|
|
126
|
+
return upstream;
|
|
60
127
|
}
|
|
61
128
|
/** POST /v1/speech: writes the audio to the phone as ElevenLabs produces it.
|
|
129
|
+
* With `timestamps`, answers JSON instead: `{ audio, audioFormat, alignment }`,
|
|
130
|
+
* the audio base64 in the same PCM format and the alignment null when
|
|
131
|
+
* ElevenLabs sent none.
|
|
62
132
|
* Failures before the first byte are thrown for the route's JSON error; a
|
|
63
133
|
* failure mid-stream cuts the response off, which the phone treats as an
|
|
64
134
|
* error. The phone hanging up cancels the ElevenLabs request. */
|
|
65
135
|
export async function streamSpeech(data, response, options = {}) {
|
|
66
|
-
const
|
|
136
|
+
const request = speechRequest.parse(data);
|
|
137
|
+
const text = speakableText(request.text);
|
|
138
|
+
if (!text)
|
|
139
|
+
throw new BridgeError(400, "There is nothing to say in this reply.", { code: "speech-invalid" });
|
|
140
|
+
// With timestamps, the alignment covers these spoken words, not the markdown.
|
|
141
|
+
const { timestamps } = request;
|
|
67
142
|
const abort = new AbortController();
|
|
68
143
|
response.once("close", () => { if (!response.writableEnded)
|
|
69
144
|
abort.abort(); });
|
|
145
|
+
if (timestamps) {
|
|
146
|
+
const { audio, alignment } = await synthesizeTimedSpeech(text, abort.signal, options);
|
|
147
|
+
response.statusCode = 200;
|
|
148
|
+
response.setHeader("Content-Type", "application/json");
|
|
149
|
+
response.end(JSON.stringify({ audio, audioFormat: SPEECH_AUDIO, alignment }));
|
|
150
|
+
return;
|
|
151
|
+
}
|
|
70
152
|
const audio = await synthesizeSpeech(text, abort.signal, options);
|
|
71
153
|
response.statusCode = 200;
|
|
72
154
|
response.setHeader("Content-Type", "application/octet-stream");
|