@phren/cli 0.3.11 → 0.3.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bridge/account-usage.d.ts +124 -0
- package/dist/bridge/account-usage.js +258 -0
- package/dist/bridge/agent-hook-stores.d.ts +3 -0
- package/dist/bridge/agent-hook-stores.js +5 -0
- package/dist/bridge/agent-hooks.d.ts +162 -5
- package/dist/bridge/agent-hooks.js +682 -49
- package/dist/bridge/approval-summary.d.ts +16 -0
- package/dist/bridge/approval-summary.js +133 -0
- package/dist/bridge/authority-command.d.ts +20 -0
- package/dist/bridge/authority-command.js +120 -0
- package/dist/bridge/authority.d.ts +223 -0
- package/dist/bridge/authority.js +231 -0
- package/dist/bridge/claude-account-setup.d.ts +14 -0
- package/dist/bridge/claude-account-setup.js +69 -0
- package/dist/bridge/claude-accounts.d.ts +49 -0
- package/dist/bridge/claude-accounts.js +195 -0
- package/dist/bridge/code-routes.js +2 -2
- package/dist/bridge/codex-app-server.d.ts +144 -0
- package/dist/bridge/codex-app-server.js +319 -0
- package/dist/bridge/codex-auth-refresh.d.ts +33 -0
- package/dist/bridge/codex-auth-refresh.js +71 -0
- package/dist/bridge/codex-hook-trust.d.ts +27 -0
- package/dist/bridge/codex-hook-trust.js +237 -0
- package/dist/bridge/codex-question-reply.d.ts +16 -0
- package/dist/bridge/codex-question-reply.js +57 -0
- package/dist/bridge/codex-review-mode.d.ts +10 -0
- package/dist/bridge/codex-review-mode.js +74 -0
- package/dist/bridge/codex-servers.d.ts +217 -0
- package/dist/bridge/codex-servers.js +636 -0
- package/dist/bridge/command.js +97 -4
- package/dist/bridge/computer-identity.d.ts +75 -0
- package/dist/bridge/computer-identity.js +166 -0
- package/dist/bridge/conductor-group.d.ts +91 -8
- package/dist/bridge/conductor-group.js +105 -9
- package/dist/bridge/conductor-role.d.ts +94 -0
- package/dist/bridge/conductor-role.js +176 -0
- package/dist/bridge/conductor-sets.d.ts +20 -0
- package/dist/bridge/conductor-sets.js +59 -0
- package/dist/bridge/dispatch-command.d.ts +3 -0
- package/dist/bridge/dispatch-command.js +82 -12
- package/dist/bridge/dispatch-returns.d.ts +128 -19
- package/dist/bridge/dispatch-returns.js +371 -54
- package/dist/bridge/dispatch.d.ts +128 -3
- package/dist/bridge/dispatch.js +303 -42
- package/dist/bridge/fanouts.d.ts +7 -1
- package/dist/bridge/fanouts.js +39 -4
- package/dist/bridge/folder-trust.d.ts +56 -0
- package/dist/bridge/folder-trust.js +195 -0
- package/dist/bridge/grants.d.ts +22 -1
- package/dist/bridge/grants.js +57 -9
- package/dist/bridge/hand-off-queue.d.ts +74 -0
- package/dist/bridge/hand-off-queue.js +235 -0
- package/dist/bridge/hand-off.d.ts +30 -10
- package/dist/bridge/hand-off.js +79 -46
- package/dist/bridge/harness-status.d.ts +11 -0
- package/dist/bridge/harness-status.js +32 -0
- package/dist/bridge/harnesses.d.ts +39 -0
- package/dist/bridge/harnesses.js +65 -0
- package/dist/bridge/health.d.ts +2 -0
- package/dist/bridge/health.js +1 -1
- package/dist/bridge/herdr.d.ts +11 -2
- package/dist/bridge/herdr.js +98 -23
- package/dist/bridge/hook-fast.d.ts +4 -0
- package/dist/bridge/hook-fast.js +68 -0
- package/dist/bridge/install.d.ts +16 -1
- package/dist/bridge/install.js +102 -21
- package/dist/bridge/launch-brief.d.ts +96 -0
- package/dist/bridge/launch-brief.js +164 -0
- package/dist/bridge/model-switch.d.ts +3 -0
- package/dist/bridge/model-switch.js +17 -1
- package/dist/bridge/models.d.ts +8 -4
- package/dist/bridge/models.js +24 -8
- package/dist/bridge/mux-wire.d.ts +9 -0
- package/dist/bridge/mux-wire.js +19 -0
- package/dist/bridge/opencode-pane-server.d.ts +113 -0
- package/dist/bridge/opencode-pane-server.js +332 -0
- package/dist/bridge/opencode-panes.d.ts +111 -0
- package/dist/bridge/opencode-panes.js +295 -0
- package/dist/bridge/pane-accounts.d.ts +8 -0
- package/dist/bridge/pane-accounts.js +46 -0
- package/dist/bridge/pane-status.d.ts +12 -0
- package/dist/bridge/pane-status.js +50 -1
- package/dist/bridge/prompt-startup.d.ts +5 -0
- package/dist/bridge/prompt-startup.js +24 -0
- package/dist/bridge/protocol.d.ts +5 -0
- package/dist/bridge/protocol.js +4 -0
- package/dist/bridge/push.d.ts +11 -4
- package/dist/bridge/push.js +27 -4
- package/dist/bridge/questions.d.ts +37 -2
- package/dist/bridge/questions.js +158 -15
- package/dist/bridge/resources.d.ts +9 -3
- package/dist/bridge/resources.js +24 -8
- package/dist/bridge/schedule-format.d.ts +2 -0
- package/dist/bridge/schedule-format.js +3 -0
- package/dist/bridge/schedule-launch.d.ts +2 -1
- package/dist/bridge/schedule-launch.js +27 -53
- package/dist/bridge/schedule-watch.d.ts +24 -1
- package/dist/bridge/schedule-watch.js +89 -2
- package/dist/bridge/server-launch.d.ts +22 -5
- package/dist/bridge/server-launch.js +238 -32
- package/dist/bridge/server-overview.d.ts +10 -3
- package/dist/bridge/server-overview.js +32 -15
- package/dist/bridge/server-pane-routes.d.ts +3 -0
- package/dist/bridge/server-pane-routes.js +150 -18
- package/dist/bridge/server-routes.d.ts +11 -0
- package/dist/bridge/server-routes.js +214 -36
- package/dist/bridge/server-stream.d.ts +2 -0
- package/dist/bridge/server-stream.js +22 -8
- package/dist/bridge/server.d.ts +6 -1
- package/dist/bridge/server.js +58 -13
- package/dist/bridge/session-activity.d.ts +93 -0
- package/dist/bridge/session-activity.js +139 -0
- package/dist/bridge/session-rename.d.ts +48 -0
- package/dist/bridge/session-rename.js +190 -0
- package/dist/bridge/session-servers.js +2 -2
- package/dist/bridge/settings-switch.d.ts +60 -0
- package/dist/bridge/settings-switch.js +287 -0
- package/dist/bridge/speech-voice.d.ts +55 -0
- package/dist/bridge/speech-voice.js +132 -0
- package/dist/bridge/speech.d.ts +70 -11
- package/dist/bridge/speech.js +169 -41
- package/dist/bridge/stalls.d.ts +22 -0
- package/dist/bridge/stalls.js +57 -0
- package/dist/bridge/sudo.d.ts +209 -0
- package/dist/bridge/sudo.js +501 -0
- package/dist/bridge/terminal-herdr.js +9 -3
- package/dist/bridge/terminal-tmux.d.ts +15 -3
- package/dist/bridge/terminal-tmux.js +103 -13
- package/dist/bridge/terminal.d.ts +27 -4
- package/dist/bridge/terminal.js +8 -1
- package/dist/bridge/transcript-claude.d.ts +4 -0
- package/dist/bridge/transcript-claude.js +186 -16
- package/dist/bridge/transcript-codex.d.ts +0 -2
- package/dist/bridge/transcript-codex.js +29 -4
- package/dist/bridge/transcript-preview.d.ts +1 -0
- package/dist/bridge/transcript-preview.js +24 -4
- package/dist/bridge/transcripts.d.ts +15 -2
- package/dist/bridge/transcripts.js +47 -16
- package/dist/bridge/turn-records.d.ts +85 -0
- package/dist/bridge/turn-records.js +143 -0
- package/dist/bridge/uploads.d.ts +3 -0
- package/dist/bridge/uploads.js +14 -0
- package/dist/bridge/usage.d.ts +56 -10
- package/dist/bridge/usage.js +190 -45
- package/dist/bridge-hook.mjs +27975 -20636
- package/dist/cli/claude-unmanaged.d.ts +12 -0
- package/dist/cli/claude-unmanaged.js +105 -0
- package/dist/cli/schedules.js +2 -1
- package/dist/cli/session-start.js +10 -1
- package/dist/cli/session-stop.js +9 -1
- package/dist/cli-registry.js +31 -6
- package/dist/data/task-receipts.d.ts +19 -0
- package/dist/data/task-receipts.js +68 -0
- package/dist/data/tasks.js +4 -1
- package/dist/generated/memory-ui-graph.browser.js +63 -63
- package/dist/graph-core/model.js +24 -3
- package/dist/graph-core/types.d.ts +4 -1
- package/dist/graph-core/types.js +1 -0
- package/dist/init/init-preset.js +7 -3
- package/dist/init/init-uninstall.js +2 -11
- package/dist/init/teardown.d.ts +3 -0
- package/dist/init/teardown.js +47 -0
- package/dist/link/context.js +21 -9
- package/dist/mcp/plugin-mode.js +1 -1
- package/dist/memory-ui-graph.runtime.js +63 -63
- package/dist/modules/code-package.d.ts +1 -0
- package/dist/modules/code-package.js +36 -7
- package/dist/modules/registry.js +17 -16
- package/dist/profile-store.d.ts +6 -0
- package/dist/profile-store.js +25 -1
- package/dist/shell/graph/glyphs.js +4 -0
- package/dist/shell/graph/graph-view.js +2 -0
- package/dist/status.js +7 -1
- package/dist/sync/task-claim.js +3 -2
- package/dist/task/lifecycle.js +25 -4
- package/dist/test-global-setup.js +5 -0
- package/dist/tools/dispatch.js +52 -4
- package/dist/tools/tasks.js +10 -1
- package/package.json +1 -1
- package/plugins/opencode/phren-transcript.js +87 -4
- package/starter/global/AGENTS.md +4 -0
- package/starter/global/skills/conductor/SKILL.md +51 -4
package/dist/bridge/speech.d.ts
CHANGED
|
@@ -1,21 +1,64 @@
|
|
|
1
1
|
import type { ServerResponse } from "node:http";
|
|
2
2
|
import { z } from "zod";
|
|
3
3
|
import { BridgeError, type Json } from "./protocol.js";
|
|
4
|
+
export { DEFAULT_SPEECH_MODEL, DEFAULT_SPEECH_VOICE, FALLBACK_SPEECH_MODEL } from "./speech-voice.js";
|
|
4
5
|
/** Spoken replies for the phone's talk mode. The phone sends a sentence; the
|
|
5
6
|
* Hook voices it with ElevenLabs and streams the audio back. The API key is
|
|
6
7
|
* read on this computer, used only in the request to ElevenLabs, and never
|
|
7
8
|
* appears in a response, including errors. */
|
|
8
|
-
/**
|
|
9
|
-
*
|
|
9
|
+
/** What a phone that names no `formats` gets, as every phone did before
|
|
10
|
+
* `speechFormats`: raw signed 16-bit little-endian mono PCM at 24 kHz, which
|
|
11
|
+
* it queues straight into its audio engine without decoding. */
|
|
10
12
|
export declare const SPEECH_AUDIO = "pcm_s16le;rate=24000;channels=1";
|
|
11
|
-
/** ElevenLabs
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
13
|
+
/** ElevenLabs output formats the Hook can serve, best first. Each account
|
|
14
|
+
* plan allows some of them (Creator refuses pcm_44100, which is Pro and
|
|
15
|
+
* above); the Hook learns which from ElevenLabs' refusal rather than a
|
|
16
|
+
* hard-coded plan table. pcm_24000 works on every plan and every phone. */
|
|
17
|
+
export declare const SPEECH_FORMATS: readonly ["pcm_44100", "mp3_44100_192", "pcm_24000"];
|
|
18
|
+
export type SpeechFormat = typeof SPEECH_FORMATS[number];
|
|
19
|
+
/** How each format is described to the phone: `audioFormat` in the JSON reply
|
|
20
|
+
* and `X-Phren-Audio` on the streamed one, plus the sample rate on its own. */
|
|
21
|
+
export declare const SPEECH_FORMAT_INFO: Record<SpeechFormat, {
|
|
22
|
+
audio: string;
|
|
23
|
+
sampleRate: number;
|
|
24
|
+
contentType: string;
|
|
25
|
+
}>;
|
|
26
|
+
/** Talk mode waits for the whole sentence on the timestamped path, so a model
|
|
27
|
+
* whose recent replies took longer than this to start playing is set aside
|
|
28
|
+
* for FALLBACK_HOLD_MS. Measured 2026-09-28 on a 95-character reply with the
|
|
29
|
+
* owner's voice: v4 Turbo ~200-400 ms to first streamed byte and ~1.1 s for
|
|
30
|
+
* the timestamped reply; Flash v2.5 ~180-460 ms and ~0.4 s. 1.5 s leaves v4
|
|
31
|
+
* Turbo headroom on a normal day and catches a slow one. */
|
|
32
|
+
export declare const SPEECH_SLOW_MS = 1500;
|
|
33
|
+
/** How long a failing or slow model, or a refused format, is set aside. */
|
|
34
|
+
export declare const FALLBACK_HOLD_MS: number;
|
|
15
35
|
export declare const speechRequest: z.ZodObject<{
|
|
16
36
|
text: z.ZodString;
|
|
17
37
|
timestamps: z.ZodOptional<z.ZodBoolean>;
|
|
38
|
+
voice: z.ZodOptional<z.ZodString>;
|
|
39
|
+
formats: z.ZodPreprocess<z.ZodOptional<z.ZodArray<z.ZodEnum<{
|
|
40
|
+
mp3_44100_192: "mp3_44100_192";
|
|
41
|
+
pcm_24000: "pcm_24000";
|
|
42
|
+
pcm_44100: "pcm_44100";
|
|
43
|
+
}>>>, unknown>;
|
|
18
44
|
}, z.core.$strip>;
|
|
45
|
+
/** What the Hook learned about this account: formats its plan refused and
|
|
46
|
+
* models set aside for failing or being slow, each until a time, and the
|
|
47
|
+
* recent start-of-audio times per model. One per Hook process. */
|
|
48
|
+
export declare class SpeechState {
|
|
49
|
+
private readonly now;
|
|
50
|
+
private readonly refused;
|
|
51
|
+
private readonly benched;
|
|
52
|
+
private readonly latency;
|
|
53
|
+
constructor(now?: () => number);
|
|
54
|
+
formatAllowed(format: SpeechFormat): boolean;
|
|
55
|
+
refuseFormat(format: SpeechFormat): void;
|
|
56
|
+
benchedReason(model: string): "failed" | "slow" | undefined;
|
|
57
|
+
bench(model: string, reason: "failed" | "slow"): void;
|
|
58
|
+
/** Records how long a reply took to start; benches the model when the median
|
|
59
|
+
* of its last three is over SPEECH_SLOW_MS. */
|
|
60
|
+
recordLatency(model: string, ms: number): void;
|
|
61
|
+
}
|
|
19
62
|
/** When each character of the voiced text starts and ends, in seconds from
|
|
20
63
|
* the start of the audio. */
|
|
21
64
|
export interface SpeechAlignment {
|
|
@@ -28,6 +71,18 @@ export interface SpeechOptions {
|
|
|
28
71
|
/** Resolves the ElevenLabs key; defaults to ELEVENLABS_API_KEY, then the stored key (speech-key.ts). */
|
|
29
72
|
key?: () => Promise<string | undefined>;
|
|
30
73
|
voice?: string;
|
|
74
|
+
/** Formats the phone plays, from the request; pcm_24000 always. */
|
|
75
|
+
formats?: readonly string[];
|
|
76
|
+
/** Defaults to this computer's setting (speech-voice.ts). */
|
|
77
|
+
model?: string;
|
|
78
|
+
/** Defaults to the Hook's own. */
|
|
79
|
+
state?: SpeechState;
|
|
80
|
+
now?: () => number;
|
|
81
|
+
}
|
|
82
|
+
/** What was voiced: the model and format actually used. */
|
|
83
|
+
export interface SpeechResult {
|
|
84
|
+
model: string;
|
|
85
|
+
format: SpeechFormat;
|
|
31
86
|
}
|
|
32
87
|
/** A fixed message per ElevenLabs failure: its own response text is never
|
|
33
88
|
* passed on. */
|
|
@@ -39,9 +94,11 @@ export declare function speechError(upstream: Response): Promise<BridgeError>;
|
|
|
39
94
|
* nothing speakable is left. */
|
|
40
95
|
export declare function speakableText(text: string): string;
|
|
41
96
|
/** Starts ElevenLabs' streaming synthesis and returns its audio body. */
|
|
42
|
-
export declare function synthesizeSpeech(text: string, signal: AbortSignal, options?: SpeechOptions): Promise<
|
|
97
|
+
export declare function synthesizeSpeech(text: string, signal: AbortSignal, options?: SpeechOptions): Promise<SpeechResult & {
|
|
98
|
+
body: ReadableStream<Uint8Array>;
|
|
99
|
+
}>;
|
|
43
100
|
/** Voices the text in one piece with ElevenLabs' character alignment. */
|
|
44
|
-
export declare function synthesizeTimedSpeech(text: string, signal: AbortSignal, options?: SpeechOptions): Promise<{
|
|
101
|
+
export declare function synthesizeTimedSpeech(text: string, signal: AbortSignal, options?: SpeechOptions): Promise<SpeechResult & {
|
|
45
102
|
audio: string;
|
|
46
103
|
alignment: SpeechAlignment | null;
|
|
47
104
|
}>;
|
|
@@ -52,9 +109,11 @@ export declare function alignmentOf(raw: {
|
|
|
52
109
|
character_end_times_seconds?: unknown;
|
|
53
110
|
} | null | undefined): SpeechAlignment | null;
|
|
54
111
|
/** POST /v1/speech: writes the audio to the phone as ElevenLabs produces it.
|
|
55
|
-
* With `timestamps`, answers JSON instead: `{ audio, audioFormat,
|
|
56
|
-
* the audio base64 in the
|
|
57
|
-
* ElevenLabs sent none.
|
|
112
|
+
* With `timestamps`, answers JSON instead: `{ audio, audioFormat, sampleRate,
|
|
113
|
+
* format, model, alignment }`, the audio base64 in the format served and the
|
|
114
|
+
* alignment null when ElevenLabs sent none. The streamed reply names its
|
|
115
|
+
* format in `X-Phren-Audio` and `X-Phren-Audio-Rate`. A phone that sends no
|
|
116
|
+
* `formats` always gets pcm_24000.
|
|
58
117
|
* Failures before the first byte are thrown for the route's JSON error; a
|
|
59
118
|
* failure mid-stream cuts the response off, which the phone treats as an
|
|
60
119
|
* error. The phone hanging up cancels the ElevenLabs request. */
|
package/dist/bridge/speech.js
CHANGED
|
@@ -2,43 +2,112 @@ import { once } from "node:events";
|
|
|
2
2
|
import { z } from "zod";
|
|
3
3
|
import { BridgeError } from "./protocol.js";
|
|
4
4
|
import { readSpeechKey } from "./speech-key.js";
|
|
5
|
+
import { FALLBACK_SPEECH_MODEL, resolveSpeechModel, resolveSpeechVoice, voiceId } from "./speech-voice.js";
|
|
6
|
+
export { DEFAULT_SPEECH_MODEL, DEFAULT_SPEECH_VOICE, FALLBACK_SPEECH_MODEL } from "./speech-voice.js";
|
|
5
7
|
/** Spoken replies for the phone's talk mode. The phone sends a sentence; the
|
|
6
8
|
* Hook voices it with ElevenLabs and streams the audio back. The API key is
|
|
7
9
|
* read on this computer, used only in the request to ElevenLabs, and never
|
|
8
10
|
* appears in a response, including errors. */
|
|
9
|
-
/**
|
|
10
|
-
*
|
|
11
|
+
/** What a phone that names no `formats` gets, as every phone did before
|
|
12
|
+
* `speechFormats`: raw signed 16-bit little-endian mono PCM at 24 kHz, which
|
|
13
|
+
* it queues straight into its audio engine without decoding. */
|
|
11
14
|
export const SPEECH_AUDIO = "pcm_s16le;rate=24000;channels=1";
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
export const
|
|
15
|
+
/** ElevenLabs output formats the Hook can serve, best first. Each account
|
|
16
|
+
* plan allows some of them (Creator refuses pcm_44100, which is Pro and
|
|
17
|
+
* above); the Hook learns which from ElevenLabs' refusal rather than a
|
|
18
|
+
* hard-coded plan table. pcm_24000 works on every plan and every phone. */
|
|
19
|
+
export const SPEECH_FORMATS = ["pcm_44100", "mp3_44100_192", "pcm_24000"];
|
|
20
|
+
const BASE_FORMAT = "pcm_24000";
|
|
21
|
+
/** How each format is described to the phone: `audioFormat` in the JSON reply
|
|
22
|
+
* and `X-Phren-Audio` on the streamed one, plus the sample rate on its own. */
|
|
23
|
+
export const SPEECH_FORMAT_INFO = {
|
|
24
|
+
pcm_44100: { audio: "pcm_s16le;rate=44100;channels=1", sampleRate: 44_100, contentType: "application/octet-stream" },
|
|
25
|
+
mp3_44100_192: { audio: "mp3;rate=44100;bitrate=192000;channels=1", sampleRate: 44_100, contentType: "audio/mpeg" },
|
|
26
|
+
pcm_24000: { audio: SPEECH_AUDIO, sampleRate: 24_000, contentType: "application/octet-stream" },
|
|
27
|
+
};
|
|
28
|
+
/** Talk mode waits for the whole sentence on the timestamped path, so a model
|
|
29
|
+
* whose recent replies took longer than this to start playing is set aside
|
|
30
|
+
* for FALLBACK_HOLD_MS. Measured 2026-09-28 on a 95-character reply with the
|
|
31
|
+
* owner's voice: v4 Turbo ~200-400 ms to first streamed byte and ~1.1 s for
|
|
32
|
+
* the timestamped reply; Flash v2.5 ~180-460 ms and ~0.4 s. 1.5 s leaves v4
|
|
33
|
+
* Turbo headroom on a normal day and catches a slow one. */
|
|
34
|
+
export const SPEECH_SLOW_MS = 1_500;
|
|
35
|
+
/** Only replies this short are timed: a long one is slow for its length, not
|
|
36
|
+
* because the model is. */
|
|
37
|
+
const TIMED_TEXT = 200;
|
|
38
|
+
/** How long a failing or slow model, or a refused format, is set aside. */
|
|
39
|
+
export const FALLBACK_HOLD_MS = 10 * 60_000;
|
|
40
|
+
const FORMAT_HOLD_MS = 6 * 60 * 60_000;
|
|
17
41
|
const MAX_TEXT = 2_000;
|
|
18
42
|
export const speechRequest = z.object({
|
|
19
43
|
text: z.string().trim().min(1).max(MAX_TEXT),
|
|
20
44
|
/** Answer JSON with the audio and when each character is spoken, so the
|
|
21
45
|
* phone can highlight the word being read (talk mode's karaoke). */
|
|
22
46
|
timestamps: z.boolean().optional(),
|
|
47
|
+
/** A voice the phone picked; otherwise this computer's setting. */
|
|
48
|
+
voice: voiceId.optional(),
|
|
49
|
+
/** The output formats the phone plays (`SPEECH_FORMATS` names). Without
|
|
50
|
+
* it, pcm_24000, which older phones assume. Names this Hook doesn't know
|
|
51
|
+
* (a newer phone's) are dropped before validation, never refused. */
|
|
52
|
+
formats: z.preprocess(value => Array.isArray(value) ? [...new Set(value)].filter(name => SPEECH_FORMATS.includes(name)) : undefined, z.array(z.enum(SPEECH_FORMATS)).max(SPEECH_FORMATS.length).optional()),
|
|
23
53
|
});
|
|
54
|
+
/** What the Hook learned about this account: formats its plan refused and
|
|
55
|
+
* models set aside for failing or being slow, each until a time, and the
|
|
56
|
+
* recent start-of-audio times per model. One per Hook process. */
|
|
57
|
+
export class SpeechState {
|
|
58
|
+
now;
|
|
59
|
+
refused = new Map();
|
|
60
|
+
benched = new Map();
|
|
61
|
+
latency = new Map();
|
|
62
|
+
constructor(now = Date.now) {
|
|
63
|
+
this.now = now;
|
|
64
|
+
}
|
|
65
|
+
formatAllowed(format) { return (this.refused.get(format) ?? 0) <= this.now(); }
|
|
66
|
+
refuseFormat(format) { this.refused.set(format, this.now() + FORMAT_HOLD_MS); }
|
|
67
|
+
benchedReason(model) {
|
|
68
|
+
const entry = this.benched.get(model);
|
|
69
|
+
return entry && entry.until > this.now() ? entry.reason : undefined;
|
|
70
|
+
}
|
|
71
|
+
bench(model, reason) {
|
|
72
|
+
this.benched.set(model, { until: this.now() + FALLBACK_HOLD_MS, reason });
|
|
73
|
+
this.latency.delete(model);
|
|
74
|
+
}
|
|
75
|
+
/** Records how long a reply took to start; benches the model when the median
|
|
76
|
+
* of its last three is over SPEECH_SLOW_MS. */
|
|
77
|
+
recordLatency(model, ms) {
|
|
78
|
+
const samples = [...(this.latency.get(model) ?? []), ms].slice(-3);
|
|
79
|
+
this.latency.set(model, samples);
|
|
80
|
+
const median = [...samples].sort((a, b) => a - b)[Math.floor(samples.length / 2)];
|
|
81
|
+
if (samples.length === 3 && median > SPEECH_SLOW_MS)
|
|
82
|
+
this.bench(model, "slow");
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
const sharedState = new SpeechState();
|
|
86
|
+
/** ElevenLabs' machine-readable reason (`detail.status`), never its text. */
|
|
87
|
+
async function upstreamCode(upstream) {
|
|
88
|
+
try {
|
|
89
|
+
return String((await upstream.json()).detail?.status ?? "");
|
|
90
|
+
}
|
|
91
|
+
catch {
|
|
92
|
+
return "";
|
|
93
|
+
}
|
|
94
|
+
}
|
|
24
95
|
/** A fixed message per ElevenLabs failure: its own response text is never
|
|
25
96
|
* passed on. */
|
|
26
97
|
export async function speechError(upstream) {
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
}
|
|
31
|
-
catch { /* not JSON */ }
|
|
98
|
+
return speechErrorFor(upstream.status, await upstreamCode(upstream));
|
|
99
|
+
}
|
|
100
|
+
function speechErrorFor(status, detail) {
|
|
32
101
|
if (detail === "quota_exceeded")
|
|
33
102
|
return new BridgeError(402, "The ElevenLabs quota on this computer's account is used up.", { code: "speech-quota" });
|
|
34
|
-
switch (
|
|
103
|
+
switch (status) {
|
|
35
104
|
case 401:
|
|
36
105
|
case 403: return new BridgeError(502, "ElevenLabs refused this computer's key.", { code: "speech-rejected" });
|
|
37
106
|
case 404: return new BridgeError(502, "ElevenLabs doesn't know the configured voice.", { code: "speech-voice" });
|
|
38
107
|
case 400:
|
|
39
108
|
case 422: return new BridgeError(400, "ElevenLabs couldn't voice this text.", { code: "speech-invalid" });
|
|
40
109
|
case 429: return new BridgeError(429, "ElevenLabs is busy. Try again shortly.", { code: "speech-busy" });
|
|
41
|
-
default: return new BridgeError(502, `ElevenLabs failed (HTTP ${
|
|
110
|
+
default: return new BridgeError(502, `ElevenLabs failed (HTTP ${status}).`, { code: "speech-failed" });
|
|
42
111
|
}
|
|
43
112
|
}
|
|
44
113
|
/** A reply as it should sound: markdown the agent wrote for the chat bubble
|
|
@@ -74,14 +143,15 @@ export function speakableText(text) {
|
|
|
74
143
|
}
|
|
75
144
|
/** Starts ElevenLabs' streaming synthesis and returns its audio body. */
|
|
76
145
|
export async function synthesizeSpeech(text, signal, options = {}) {
|
|
77
|
-
const upstream = await elevenLabs("stream", text, signal, options);
|
|
146
|
+
const { upstream, model, format, started } = await elevenLabs("stream", text, signal, options);
|
|
78
147
|
if (!upstream.body)
|
|
79
148
|
throw await speechError(upstream);
|
|
80
|
-
|
|
149
|
+
timed(text, model, started, options);
|
|
150
|
+
return { body: upstream.body, model, format };
|
|
81
151
|
}
|
|
82
152
|
/** Voices the text in one piece with ElevenLabs' character alignment. */
|
|
83
153
|
export async function synthesizeTimedSpeech(text, signal, options = {}) {
|
|
84
|
-
const upstream = await elevenLabs("with-timestamps", text, signal, options);
|
|
154
|
+
const { upstream, model, format, started } = await elevenLabs("with-timestamps", text, signal, options);
|
|
85
155
|
let body;
|
|
86
156
|
try {
|
|
87
157
|
body = (await upstream.json());
|
|
@@ -91,7 +161,16 @@ export async function synthesizeTimedSpeech(text, signal, options = {}) {
|
|
|
91
161
|
}
|
|
92
162
|
if (typeof body.audio_base64 !== "string")
|
|
93
163
|
throw new BridgeError(502, "ElevenLabs sent an unreadable reply.", { code: "speech-failed" });
|
|
94
|
-
|
|
164
|
+
// Talk mode plays nothing until this whole reply is in.
|
|
165
|
+
timed(text, model, started, options);
|
|
166
|
+
return { audio: body.audio_base64, alignment: alignmentOf(body.alignment), model, format };
|
|
167
|
+
}
|
|
168
|
+
/** Times a short reply from the chosen model (never the fallback, which has
|
|
169
|
+
* nothing to fall back to). */
|
|
170
|
+
function timed(text, model, started, options) {
|
|
171
|
+
if (model === FALLBACK_SPEECH_MODEL || text.length > TIMED_TEXT)
|
|
172
|
+
return;
|
|
173
|
+
(options.state ?? sharedState).recordLatency(model, (options.now ?? Date.now)() - started);
|
|
95
174
|
}
|
|
96
175
|
/** ElevenLabs' alignment, kept only when its three lists line up. */
|
|
97
176
|
export function alignmentOf(raw) {
|
|
@@ -104,31 +183,73 @@ export function alignmentOf(raw) {
|
|
|
104
183
|
return null;
|
|
105
184
|
return { characters: characters, starts: starts, ends: ends };
|
|
106
185
|
}
|
|
186
|
+
/** The plan doesn't allow this output format (Creator: "Output format
|
|
187
|
+
* 'pcm_44100' is only available on the Pro tier and above", 403
|
|
188
|
+
* `output_format_not_allowed`). */
|
|
189
|
+
function formatRefused(status, code) {
|
|
190
|
+
return (status === 400 || status === 403 || status === 422) && /output_format/.test(code);
|
|
191
|
+
}
|
|
192
|
+
/** A failure another model may not have: the model itself refused or
|
|
193
|
+
* unknown, or ElevenLabs erroring. Not the key, quota, voice or rate limit. */
|
|
194
|
+
function modelFailed(status, code) {
|
|
195
|
+
if (status === 429 || code === "quota_exceeded" || code === "invalid_api_key")
|
|
196
|
+
return false;
|
|
197
|
+
return status >= 500 || /model/.test(code);
|
|
198
|
+
}
|
|
199
|
+
/** Asks ElevenLabs for the best format the phone plays and the plan allows,
|
|
200
|
+
* with the chosen model, then Flash v2.5 when that model fails or has been
|
|
201
|
+
* slow. A refused format is skipped for FORMAT_HOLD_MS, a failed or slow
|
|
202
|
+
* model for FALLBACK_HOLD_MS. */
|
|
107
203
|
async function elevenLabs(endpoint, text, signal, options) {
|
|
108
204
|
const key = await (options.key ?? readSpeechKey)();
|
|
109
205
|
if (!key)
|
|
110
206
|
throw new BridgeError(503, "Spoken replies aren't set up on this computer: it has no ElevenLabs key.", { code: "speech-unconfigured" });
|
|
111
|
-
const voice = options.voice
|
|
112
|
-
const
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
207
|
+
const { voice } = await resolveSpeechVoice(options.voice);
|
|
208
|
+
const state = options.state ?? sharedState, now = options.now ?? Date.now;
|
|
209
|
+
const chosen = options.model ?? (await resolveSpeechModel()).model;
|
|
210
|
+
const models = chosen === FALLBACK_SPEECH_MODEL ? [chosen] : state.benchedReason(chosen) ? [FALLBACK_SPEECH_MODEL] : [chosen, FALLBACK_SPEECH_MODEL];
|
|
211
|
+
const accepted = new Set(options.formats ?? []);
|
|
212
|
+
let failure;
|
|
213
|
+
models: for (const model of models) {
|
|
214
|
+
// Per model, so a format refused under one isn't tried again under the next.
|
|
215
|
+
const formats = SPEECH_FORMATS.filter(format => format === BASE_FORMAT || (accepted.has(format) && state.formatAllowed(format)));
|
|
216
|
+
for (const format of formats) {
|
|
217
|
+
const url = `https://api.elevenlabs.io/v1/text-to-speech/${encodeURIComponent(voice)}/${endpoint}?output_format=${format}`;
|
|
218
|
+
const started = now();
|
|
219
|
+
let upstream;
|
|
220
|
+
try {
|
|
221
|
+
upstream = await (options.fetch ?? fetch)(url, {
|
|
222
|
+
method: "POST", signal,
|
|
223
|
+
headers: { "xi-api-key": key, "Content-Type": "application/json" },
|
|
224
|
+
body: JSON.stringify({ text, model_id: model, voice_settings: { stability: 0.6, similarity_boost: 0.75 } }),
|
|
225
|
+
});
|
|
226
|
+
}
|
|
227
|
+
catch {
|
|
228
|
+
throw new BridgeError(502, "Couldn't reach ElevenLabs from this computer.", { code: "speech-unreachable" });
|
|
229
|
+
}
|
|
230
|
+
if (upstream.ok)
|
|
231
|
+
return { upstream, model, format, started };
|
|
232
|
+
const code = await upstreamCode(upstream);
|
|
233
|
+
if (format !== BASE_FORMAT && formatRefused(upstream.status, code)) {
|
|
234
|
+
state.refuseFormat(format);
|
|
235
|
+
continue;
|
|
236
|
+
}
|
|
237
|
+
failure = speechErrorFor(upstream.status, code);
|
|
238
|
+
if (model !== FALLBACK_SPEECH_MODEL && modelFailed(upstream.status, code)) {
|
|
239
|
+
state.bench(model, "failed");
|
|
240
|
+
continue models;
|
|
241
|
+
}
|
|
242
|
+
throw failure;
|
|
243
|
+
}
|
|
123
244
|
}
|
|
124
|
-
|
|
125
|
-
throw await speechError(upstream);
|
|
126
|
-
return upstream;
|
|
245
|
+
throw failure ?? new BridgeError(502, "ElevenLabs failed.", { code: "speech-failed" });
|
|
127
246
|
}
|
|
128
247
|
/** POST /v1/speech: writes the audio to the phone as ElevenLabs produces it.
|
|
129
|
-
* With `timestamps`, answers JSON instead: `{ audio, audioFormat,
|
|
130
|
-
* the audio base64 in the
|
|
131
|
-
* ElevenLabs sent none.
|
|
248
|
+
* With `timestamps`, answers JSON instead: `{ audio, audioFormat, sampleRate,
|
|
249
|
+
* format, model, alignment }`, the audio base64 in the format served and the
|
|
250
|
+
* alignment null when ElevenLabs sent none. The streamed reply names its
|
|
251
|
+
* format in `X-Phren-Audio` and `X-Phren-Audio-Rate`. A phone that sends no
|
|
252
|
+
* `formats` always gets pcm_24000.
|
|
132
253
|
* Failures before the first byte are thrown for the route's JSON error; a
|
|
133
254
|
* failure mid-stream cuts the response off, which the phone treats as an
|
|
134
255
|
* error. The phone hanging up cancels the ElevenLabs request. */
|
|
@@ -139,20 +260,27 @@ export async function streamSpeech(data, response, options = {}) {
|
|
|
139
260
|
throw new BridgeError(400, "There is nothing to say in this reply.", { code: "speech-invalid" });
|
|
140
261
|
// With timestamps, the alignment covers these spoken words, not the markdown.
|
|
141
262
|
const { timestamps } = request;
|
|
263
|
+
if (request.voice)
|
|
264
|
+
options = { ...options, voice: request.voice };
|
|
265
|
+
options = { ...options, formats: request.formats ?? [] };
|
|
142
266
|
const abort = new AbortController();
|
|
143
267
|
response.once("close", () => { if (!response.writableEnded)
|
|
144
268
|
abort.abort(); });
|
|
145
269
|
if (timestamps) {
|
|
146
|
-
const { audio, alignment } = await synthesizeTimedSpeech(text, abort.signal, options);
|
|
270
|
+
const { audio, alignment, model, format } = await synthesizeTimedSpeech(text, abort.signal, options);
|
|
271
|
+
const info = SPEECH_FORMAT_INFO[format];
|
|
147
272
|
response.statusCode = 200;
|
|
148
273
|
response.setHeader("Content-Type", "application/json");
|
|
149
|
-
response.end(JSON.stringify({ audio, audioFormat:
|
|
274
|
+
response.end(JSON.stringify({ audio, audioFormat: info.audio, sampleRate: info.sampleRate, format, model, alignment }));
|
|
150
275
|
return;
|
|
151
276
|
}
|
|
152
|
-
const audio = await synthesizeSpeech(text, abort.signal, options);
|
|
277
|
+
const { body: audio, model, format } = await synthesizeSpeech(text, abort.signal, options);
|
|
278
|
+
const info = SPEECH_FORMAT_INFO[format];
|
|
153
279
|
response.statusCode = 200;
|
|
154
|
-
response.setHeader("Content-Type",
|
|
155
|
-
response.setHeader("X-Phren-Audio",
|
|
280
|
+
response.setHeader("Content-Type", info.contentType);
|
|
281
|
+
response.setHeader("X-Phren-Audio", info.audio);
|
|
282
|
+
response.setHeader("X-Phren-Audio-Rate", String(info.sampleRate));
|
|
283
|
+
response.setHeader("X-Phren-Speech-Model", model);
|
|
156
284
|
try {
|
|
157
285
|
for await (const chunk of audio) {
|
|
158
286
|
if (!response.write(chunk))
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
import type { Json, Target } from "./protocol.js";
|
|
2
|
+
export interface Stall {
|
|
3
|
+
stalled: true;
|
|
4
|
+
stalledSince: string;
|
|
5
|
+
stallFor: number;
|
|
6
|
+
}
|
|
7
|
+
export interface StallReaders {
|
|
8
|
+
screen: (target: Target) => Promise<string>;
|
|
9
|
+
transcript: (target: Target) => Promise<string>;
|
|
10
|
+
now: () => number;
|
|
11
|
+
threshold: () => number;
|
|
12
|
+
}
|
|
13
|
+
export declare function stallThreshold(): number;
|
|
14
|
+
/** Requires successful observations of both sources. Identity, terminal, work
|
|
15
|
+
* status and either source changing all reset the clock. Missing data is unknown. */
|
|
16
|
+
export declare class StallDetector {
|
|
17
|
+
private readers;
|
|
18
|
+
private entries;
|
|
19
|
+
constructor(readers?: StallReaders);
|
|
20
|
+
observe(target: Target, pane: Json): Promise<Stall | undefined>;
|
|
21
|
+
}
|
|
22
|
+
export declare const sessionStalls: StallDetector;
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
import { createHash } from "node:crypto";
|
|
2
|
+
import { open } from "node:fs/promises";
|
|
3
|
+
import { targetTranscriptPath } from "./transcripts.js";
|
|
4
|
+
import { terminalProvider } from "./terminal.js";
|
|
5
|
+
export function stallThreshold() {
|
|
6
|
+
const value = Number(process.env.PHREN_STALL_MS ?? 300_000);
|
|
7
|
+
return Number.isFinite(value) && value >= 0 ? value : 300_000;
|
|
8
|
+
}
|
|
9
|
+
async function transcriptStamp(target) {
|
|
10
|
+
const file = await open(await targetTranscriptPath(target), "r");
|
|
11
|
+
try {
|
|
12
|
+
const info = await file.stat(), start = Math.max(0, info.size - 8192), buffer = Buffer.alloc(info.size - start);
|
|
13
|
+
await file.read(buffer, 0, buffer.length, start);
|
|
14
|
+
return `${info.size}:${createHash("sha256").update(buffer).digest("hex")}`;
|
|
15
|
+
}
|
|
16
|
+
finally {
|
|
17
|
+
await file.close();
|
|
18
|
+
}
|
|
19
|
+
}
|
|
20
|
+
/** Requires successful observations of both sources. Identity, terminal, work
|
|
21
|
+
* status and either source changing all reset the clock. Missing data is unknown. */
|
|
22
|
+
export class StallDetector {
|
|
23
|
+
readers;
|
|
24
|
+
entries = new Map();
|
|
25
|
+
constructor(readers = {
|
|
26
|
+
screen: target => terminalProvider().readScreen(target.server, target.pane, { scope: "pane", source: "visible", lines: 120, stripAnsi: true, timeoutMs: 1000 }),
|
|
27
|
+
transcript: transcriptStamp, now: Date.now, threshold: stallThreshold,
|
|
28
|
+
}) {
|
|
29
|
+
this.readers = readers;
|
|
30
|
+
}
|
|
31
|
+
async observe(target, pane) {
|
|
32
|
+
const key = JSON.stringify([target.server, target.pane]);
|
|
33
|
+
if (pane.agent_status !== "working" || this.readers.threshold() === 0) {
|
|
34
|
+
this.entries.delete(key);
|
|
35
|
+
return undefined;
|
|
36
|
+
}
|
|
37
|
+
let screen, transcript;
|
|
38
|
+
try {
|
|
39
|
+
[screen, transcript] = await Promise.all([this.readers.screen(target), this.readers.transcript(target)]);
|
|
40
|
+
}
|
|
41
|
+
catch {
|
|
42
|
+
this.entries.delete(key);
|
|
43
|
+
return undefined;
|
|
44
|
+
}
|
|
45
|
+
const signature = createHash("sha256").update(JSON.stringify([target, pane.terminal_id, screen, transcript])).digest("hex");
|
|
46
|
+
const now = this.readers.now(), prior = this.entries.get(key);
|
|
47
|
+
if (!prior || prior.signature !== signature) {
|
|
48
|
+
this.entries.delete(key);
|
|
49
|
+
this.entries.set(key, { signature, since: now });
|
|
50
|
+
}
|
|
51
|
+
while (this.entries.size > 1024)
|
|
52
|
+
this.entries.delete(this.entries.keys().next().value);
|
|
53
|
+
const since = this.entries.get(key).since;
|
|
54
|
+
return now - since >= this.readers.threshold() ? { stalled: true, stalledSince: new Date(since).toISOString(), stallFor: Math.floor((now - since) / 1000) } : undefined;
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
export const sessionStalls = new StallDetector();
|