@stage-labs/metro 0.1.0-beta.236 → 0.1.0-beta.238
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/claude.js +1 -1
- package/dist/cli.js +7 -0
- package/dist/voice.js +39 -0
- package/package.json +1 -1
- package/runtime/marketplace/plugin/.claude-plugin/plugin.json +1 -1
- package/runtime/node_modules/@metro-labs/daemon/package.json +2 -0
- package/runtime/node_modules/@metro-labs/daemon/src/boot/boot.ts +5 -0
- package/runtime/node_modules/@metro-labs/daemon/src/claude/session.ts +8 -4
- package/runtime/node_modules/@metro-labs/daemon/src/routes/session-apis.ts +2 -0
- package/runtime/node_modules/@metro-labs/daemon/src/voice/api.ts +14 -0
- package/runtime/node_modules/@metro-labs/daemon/src/voice/audio.ts +153 -0
- package/runtime/node_modules/@metro-labs/daemon/src/voice/brain.ts +95 -0
- package/runtime/node_modules/@metro-labs/daemon/src/voice/call.ts +156 -0
- package/runtime/node_modules/@metro-labs/daemon/src/voice/calls.ts +54 -0
- package/runtime/node_modules/@metro-labs/daemon/src/voice/peer.ts +84 -0
- package/runtime/node_modules/@metro-labs/daemon/src/voice/scribe.ts +102 -0
- package/runtime/node_modules/@metro-labs/daemon/src/voice/speakable.ts +54 -0
- package/runtime/node_modules/@metro-labs/daemon/src/voice/speech.ts +104 -0
- package/runtime/node_modules/@metro-labs/daemon/src/voice/store.ts +97 -0
- package/runtime/node_modules/@metro-labs/daemon/src/voice/talk.ts +275 -0
- package/runtime/node_modules/@metro-labs/xmtp/src/actions-conv.ts +2 -1
- package/runtime/node_modules/@metro-labs/xmtp/src/actions.ts +11 -1
- package/runtime/node_modules/@metro-labs/xmtp/src/codecs.ts +21 -3
- package/runtime/node_modules/@metro-labs/xmtp/src/emit-payloads.ts +2 -1
- package/runtime/runtime.json +1 -1
- package/runtime/stations.json +2 -0
package/dist/claude.js
CHANGED
|
@@ -9,7 +9,7 @@ import { localPort, localUrl } from './runtime.js';
|
|
|
9
9
|
import { keepInSession, takeBackConversation } from './background.js';
|
|
10
10
|
const CHANNEL_FLAGS = ['--dangerously-load-development-channels', 'server:metro'];
|
|
11
11
|
const FRESH_PROMPT_FLAGS = ['--system-prompt-snapshot', 'off'];
|
|
12
|
-
const PERMISSION_MODE_FLAG = { auto: 'auto', bypass: 'bypassPermissions' };
|
|
12
|
+
export const PERMISSION_MODE_FLAG = { auto: 'auto', bypass: 'bypassPermissions' };
|
|
13
13
|
const KEY_HEADER = 'x-metro-key';
|
|
14
14
|
const PROBE_MS = 3_000;
|
|
15
15
|
const AUTH_STATUS_MS = 20_000;
|
package/dist/cli.js
CHANGED
|
@@ -8,6 +8,7 @@ import { update } from './update.js';
|
|
|
8
8
|
import { serve } from './serve.js';
|
|
9
9
|
import { service, serviceStopHint } from './service.js';
|
|
10
10
|
import { currentVersion } from './version.js';
|
|
11
|
+
import { voiceSession } from './voice.js';
|
|
11
12
|
const USAGE = `metro: run your agent on this machine
|
|
12
13
|
|
|
13
14
|
metro serve [--port <n>] [--owner <organization id>]
|
|
@@ -37,6 +38,11 @@ const USAGE = `metro: run your agent on this machine
|
|
|
37
38
|
the last run, Claude Code reads the memory skill (or the file given) and
|
|
38
39
|
files them into its memory, with no metro tools and writes only inside
|
|
39
40
|
the memory folder
|
|
41
|
+
metro voice --model <model>
|
|
42
|
+
the agent's side of a voice call, which the daemon starts when a call
|
|
43
|
+
comes in: Claude Code reads the call's words as stream-json on stdin and
|
|
44
|
+
streams its replies on stdout, with the agent's memory, skills and tools
|
|
45
|
+
but no metro channel
|
|
40
46
|
metro update update to the newest published version (--check only reports)
|
|
41
47
|
metro version print this CLI's version
|
|
42
48
|
|
|
@@ -71,6 +77,7 @@ const COMMANDS = {
|
|
|
71
77
|
tail: () => tailEvents(process.argv.slice(3)),
|
|
72
78
|
claude: () => launchClaude(process.argv.slice(3)),
|
|
73
79
|
memory: () => memoryRoutine(memoryPaths(process.env, process.argv[3])),
|
|
80
|
+
voice: () => voiceSession(process.argv.slice(3)),
|
|
74
81
|
update: () => update(process.argv.slice(3)),
|
|
75
82
|
version: async () => {
|
|
76
83
|
process.stdout.write(`${currentVersion()}\n`);
|
package/dist/voice.js
ADDED
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
import { homedir } from 'node:os';
|
|
2
|
+
import { headlessEnv, PERMISSION_MODE_FLAG, runClaude } from './claude.js';
|
|
3
|
+
import { permissionMode, systemPrompt } from './route.js';
|
|
4
|
+
const VOICE_PROMPT = `You are on a live voice call through Metro. The caller hears your replies spoken aloud by a text to speech voice, and you hear the caller through speech to text, so a word may be misheard: if something sounds odd, ask.
|
|
5
|
+
- Speak like a person on the phone: short sentences, usually one to three. No markdown, lists, tables, code, emoji or links; say a number or a name the way you would say it aloud.
|
|
6
|
+
- Start with the answer. Do not repeat the question.
|
|
7
|
+
- Before you use a tool, first say one short sentence about what you are checking.
|
|
8
|
+
- Look things up yourself with Read, Grep or Glob when it is quick. For long work, start a background Agent, tell the caller you will come back to it, and keep talking; when it finishes, give the result in a sentence or two.
|
|
9
|
+
- Text in [square brackets] comes from the call system, not from the caller.
|
|
10
|
+
- You cannot send chat messages during the call. When the call ends, you will be asked for call notes, which are posted to the chat.`;
|
|
11
|
+
function voiceArgs(model, mode, prompt) {
|
|
12
|
+
return [
|
|
13
|
+
'-p',
|
|
14
|
+
'--input-format',
|
|
15
|
+
'stream-json',
|
|
16
|
+
'--output-format',
|
|
17
|
+
'stream-json',
|
|
18
|
+
'--verbose',
|
|
19
|
+
'--include-partial-messages',
|
|
20
|
+
'--model',
|
|
21
|
+
model,
|
|
22
|
+
'--permission-mode',
|
|
23
|
+
PERMISSION_MODE_FLAG[mode],
|
|
24
|
+
'--settings',
|
|
25
|
+
JSON.stringify({ enabledPlugins: { 'metro@metro': false } }),
|
|
26
|
+
'--strict-mcp-config',
|
|
27
|
+
'--no-session-persistence',
|
|
28
|
+
'--append-system-prompt',
|
|
29
|
+
prompt === null ? VOICE_PROMPT : `${prompt}\n\n${VOICE_PROMPT}`,
|
|
30
|
+
];
|
|
31
|
+
}
|
|
32
|
+
export async function voiceSession(argv) {
|
|
33
|
+
const at = argv.indexOf('--model');
|
|
34
|
+
const model = at < 0 ? '' : (argv[at + 1] ?? '');
|
|
35
|
+
if (model === '')
|
|
36
|
+
throw new Error('metro voice needs --model <model>');
|
|
37
|
+
process.chdir(homedir());
|
|
38
|
+
return runClaude(voiceArgs(model, permissionMode(), systemPrompt()), await headlessEnv());
|
|
39
|
+
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "metro",
|
|
3
|
-
"version": "0.1.0-beta.
|
|
3
|
+
"version": "0.1.0-beta.238",
|
|
4
4
|
"description": "Metro for Claude Code, installed by the metro daemon on this machine: every connector the agent holds as its own MCP server relayed through the daemon, the orchestrator-only guard on the main thread, and the standing rules loaded at each session start. No vendor credential ever sits in a config file.",
|
|
5
5
|
"author": {
|
|
6
6
|
"name": "Bonustrack",
|
|
@@ -52,6 +52,7 @@ import type { StationName } from '@metro-labs/core/station-names';
|
|
|
52
52
|
import { startUploadReaper } from '../files/upload-store.js';
|
|
53
53
|
import { startAttachReaper } from '../files/attach-reaper.js';
|
|
54
54
|
import { startCloudWatchPublisher } from '../server/cloudwatch.js';
|
|
55
|
+
import { isCallEvent, onCallEvent } from '../voice/calls.js';
|
|
55
56
|
|
|
56
57
|
installCrashGuard();
|
|
57
58
|
acquireLock(join(STATE_DIR, '.tail-lock'));
|
|
@@ -67,6 +68,10 @@ const supervisor = new TrainSupervisor();
|
|
|
67
68
|
const emit = makeEmit();
|
|
68
69
|
|
|
69
70
|
supervisor.onTrainEvent((env, train) => {
|
|
71
|
+
if (isCallEvent(env)) {
|
|
72
|
+
onCallEvent(env);
|
|
73
|
+
return;
|
|
74
|
+
}
|
|
70
75
|
const entry = trainEventToMetroEvent(env, train);
|
|
71
76
|
if (entry) emit(entry);
|
|
72
77
|
});
|
|
@@ -110,13 +110,17 @@ export function hasConversation(home: string, dir = claudeDir()): boolean {
|
|
|
110
110
|
);
|
|
111
111
|
}
|
|
112
112
|
|
|
113
|
-
function
|
|
113
|
+
export function metroCli(args: string[], fixed?: string[]): string[] {
|
|
114
114
|
const bin = process.env.METRO_CLI_BIN?.trim() ?? '';
|
|
115
115
|
const runtime = agentUser() === null ? process.execPath : 'node';
|
|
116
|
-
const base =
|
|
117
|
-
const continues = (deps.continues ?? hasConversation)(home);
|
|
116
|
+
const base = fixed ?? (bin === '' ? ['metro', ...args] : [runtime, bin, ...args]);
|
|
118
117
|
const env = Object.entries(sessionEnv()).map(([k, v]) => `${k}=${v}`);
|
|
119
|
-
|
|
118
|
+
return env.length === 0 || fixed !== undefined ? base : ['env', ...env, ...base];
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
function metroCommand(deps: SessionDeps, home: string): string[] {
|
|
122
|
+
const command = metroCli(['claude'], deps.metro);
|
|
123
|
+
const continues = (deps.continues ?? hasConversation)(home);
|
|
120
124
|
return continues ? [...command, '-c'] : command;
|
|
121
125
|
}
|
|
122
126
|
|
|
@@ -21,6 +21,7 @@ import { handleTerminalRequest, type TerminalApiDeps } from '../terminal/api.js'
|
|
|
21
21
|
import { handleSchedulesRequest } from '../agent-user/schedules-api.js';
|
|
22
22
|
import { handleFilesRequest } from '../agent-user/files-api.js';
|
|
23
23
|
import { handleVaultRequest } from '../vault/api.js';
|
|
24
|
+
import { handleVoiceRequest } from '../voice/api.js';
|
|
24
25
|
|
|
25
26
|
export interface SessionApis {
|
|
26
27
|
agentApi?: AgentApiDeps;
|
|
@@ -58,6 +59,7 @@ export function handleSessionApis(
|
|
|
58
59
|
() => handleSchedulesRequest(req, res),
|
|
59
60
|
() => handleFilesRequest(req, res),
|
|
60
61
|
() => handleVaultRequest(req, res),
|
|
62
|
+
() => handleVoiceRequest(req, res),
|
|
61
63
|
...when(apis.ownerApi, (d) => handleOwnerRequest(req, res, d)),
|
|
62
64
|
...when(apis.machineApi, (d) => handleMachineRequest(req, res, d)),
|
|
63
65
|
...when(apis.modelApi, (d) => handleModelRequest(req, res, d)),
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
import type { IncomingMessage, ServerResponse } from 'node:http';
|
|
2
|
+
import { readJsonBody, sessionRoute } from '@metro-labs/http/api-http';
|
|
3
|
+
import { patchVoice, publicVoice, readVoice, writeVoice } from './store.js';
|
|
4
|
+
|
|
5
|
+
const PATH = '/api/voice';
|
|
6
|
+
|
|
7
|
+
async function answer(req: IncomingMessage): Promise<unknown> {
|
|
8
|
+
if (req.method === 'PUT') writeVoice(patchVoice(readVoice(), await readJsonBody(req)));
|
|
9
|
+
return publicVoice(readVoice());
|
|
10
|
+
}
|
|
11
|
+
|
|
12
|
+
export function handleVoiceRequest(req: IncomingMessage, res: ServerResponse): boolean {
|
|
13
|
+
return sessionRoute(req, res, { methods: { [PATH]: ['GET', 'PUT'] }, admin: ['PUT'], label: 'voice-api' }, () => answer(req));
|
|
14
|
+
}
|
|
@@ -0,0 +1,153 @@
|
|
|
1
|
+
import OpusScript from 'opusscript';
|
|
2
|
+
import { errMsg, log } from '@metro-labs/core/log';
|
|
3
|
+
|
|
4
|
+
const RATE = 48_000;
|
|
5
|
+
const FRAME = 960;
|
|
6
|
+
const FRAME_MS = 20;
|
|
7
|
+
const MAX_LATE_FRAMES = 5;
|
|
8
|
+
|
|
9
|
+
interface Frame {
|
|
10
|
+
pcm: Int16Array;
|
|
11
|
+
chars: number;
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
export function downsample3(pcm: Int16Array): Int16Array {
|
|
15
|
+
const out = new Int16Array(Math.floor(pcm.length / 3));
|
|
16
|
+
for (let i = 0; i < out.length; i += 1)
|
|
17
|
+
out[i] = Math.round(((pcm[i * 3] ?? 0) + (pcm[i * 3 + 1] ?? 0) + (pcm[i * 3 + 2] ?? 0)) / 3);
|
|
18
|
+
return out;
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
export function upsample2(pcm: Int16Array, carry: number): Int16Array {
|
|
22
|
+
const out = new Int16Array(pcm.length * 2);
|
|
23
|
+
let prev = carry;
|
|
24
|
+
for (let i = 0; i < pcm.length; i += 1) {
|
|
25
|
+
const cur = pcm[i] ?? 0;
|
|
26
|
+
out[i * 2] = Math.round((prev + cur) / 2);
|
|
27
|
+
out[i * 2 + 1] = cur;
|
|
28
|
+
prev = cur;
|
|
29
|
+
}
|
|
30
|
+
return out;
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
export const pcmOf = (bytes: Buffer): Int16Array => {
|
|
34
|
+
const copy = Buffer.from(bytes);
|
|
35
|
+
return new Int16Array(copy.buffer, copy.byteOffset, Math.floor(copy.length / 2));
|
|
36
|
+
};
|
|
37
|
+
|
|
38
|
+
export const bytesOf = (pcm: Int16Array): Buffer => Buffer.from(pcm.buffer, pcm.byteOffset, pcm.byteLength);
|
|
39
|
+
|
|
40
|
+
export class OpusIn {
|
|
41
|
+
private readonly decoder = new OpusScript(RATE, 1, OpusScript.Application.VOIP);
|
|
42
|
+
|
|
43
|
+
decode(packet: Buffer): Int16Array | null {
|
|
44
|
+
try {
|
|
45
|
+
return pcmOf(this.decoder.decode(packet));
|
|
46
|
+
} catch (err) {
|
|
47
|
+
log.debug({ err: errMsg(err) }, 'voice: an audio packet could not be decoded');
|
|
48
|
+
return null;
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
close(): void {
|
|
53
|
+
this.decoder.delete();
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
export interface Playback {
|
|
58
|
+
send(opus: Buffer, timestamp: number, marker: boolean): void;
|
|
59
|
+
played(chars: number): void;
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
export class Player {
|
|
63
|
+
private readonly encoder = new OpusScript(RATE, 1, OpusScript.Application.VOIP);
|
|
64
|
+
private readonly queue: Frame[] = [];
|
|
65
|
+
private pending = new Int16Array(0);
|
|
66
|
+
private pendingChars = 0;
|
|
67
|
+
private timer: ReturnType<typeof setTimeout> | null = null;
|
|
68
|
+
private started = 0;
|
|
69
|
+
private sent = 0;
|
|
70
|
+
private speaking = false;
|
|
71
|
+
private onIdle: (() => void) | null = null;
|
|
72
|
+
private onFirst: (() => void) | null = null;
|
|
73
|
+
|
|
74
|
+
constructor(private readonly out: Playback) {}
|
|
75
|
+
|
|
76
|
+
get busy(): boolean {
|
|
77
|
+
return this.queue.length > 0 || this.pending.length > 0;
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
whenFirstPlayed(fn: () => void): void {
|
|
81
|
+
this.onFirst = fn;
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
whenIdle(fn: () => void): void {
|
|
85
|
+
this.onIdle = fn;
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
push(pcm: Int16Array, chars: number): void {
|
|
89
|
+
const joined = new Int16Array(this.pending.length + pcm.length);
|
|
90
|
+
joined.set(this.pending);
|
|
91
|
+
joined.set(pcm, this.pending.length);
|
|
92
|
+
let at = 0;
|
|
93
|
+
for (; at + FRAME <= joined.length; at += FRAME) this.queue.push({ pcm: joined.slice(at, at + FRAME), chars: 0 });
|
|
94
|
+
this.pending = joined.slice(at);
|
|
95
|
+
this.pendingChars += chars;
|
|
96
|
+
const last = this.queue.at(-1);
|
|
97
|
+
if (last !== undefined && this.pending.length === 0) {
|
|
98
|
+
last.chars += this.pendingChars;
|
|
99
|
+
this.pendingChars = 0;
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
finish(): void {
|
|
104
|
+
if (this.pending.length === 0) return;
|
|
105
|
+
const pcm = new Int16Array(FRAME);
|
|
106
|
+
pcm.set(this.pending);
|
|
107
|
+
this.queue.push({ pcm, chars: this.pendingChars });
|
|
108
|
+
this.pending = new Int16Array(0);
|
|
109
|
+
this.pendingChars = 0;
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
clear(): void {
|
|
113
|
+
this.queue.length = 0;
|
|
114
|
+
this.pending = new Int16Array(0);
|
|
115
|
+
this.pendingChars = 0;
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
start(): void {
|
|
119
|
+
if (this.timer !== null) return;
|
|
120
|
+
this.started = performance.now();
|
|
121
|
+
this.sent = 0;
|
|
122
|
+
this.tick();
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
stop(): void {
|
|
126
|
+
if (this.timer !== null) clearTimeout(this.timer);
|
|
127
|
+
this.timer = null;
|
|
128
|
+
this.encoder.delete();
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
private tick(): void {
|
|
132
|
+
const due = Math.floor((performance.now() - this.started) / FRAME_MS) + 1;
|
|
133
|
+
if (due - this.sent > MAX_LATE_FRAMES) this.sent = due - 1;
|
|
134
|
+
while (this.sent < due) this.sendOne();
|
|
135
|
+
const next = this.started + this.sent * FRAME_MS;
|
|
136
|
+
this.timer = setTimeout(() => {
|
|
137
|
+
this.tick();
|
|
138
|
+
}, Math.max(0, next - performance.now()));
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
private sendOne(): void {
|
|
142
|
+
const frame = this.queue.shift();
|
|
143
|
+
const marker = frame !== undefined && !this.speaking;
|
|
144
|
+
this.speaking = frame !== undefined;
|
|
145
|
+
const pcm = frame?.pcm ?? new Int16Array(FRAME);
|
|
146
|
+
this.out.send(this.encoder.encode(bytesOf(pcm), FRAME), (this.sent * FRAME) >>> 0, marker);
|
|
147
|
+
this.sent += 1;
|
|
148
|
+
if (frame === undefined) return;
|
|
149
|
+
if (marker) this.onFirst?.();
|
|
150
|
+
if (frame.chars > 0) this.out.played(frame.chars);
|
|
151
|
+
if (!this.busy) this.onIdle?.();
|
|
152
|
+
}
|
|
153
|
+
}
|
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
import { spawn, type ChildProcessWithoutNullStreams } from 'node:child_process';
|
|
2
|
+
import { errMsg, log } from '@metro-labs/core/log';
|
|
3
|
+
import { isRecord } from '@metro-labs/core/is-record';
|
|
4
|
+
import { drainLines } from '@metro-labs/core/trains/protocol';
|
|
5
|
+
import { asAgent } from '../agent-user/user.js';
|
|
6
|
+
import { metroCli } from '../claude/session.js';
|
|
7
|
+
|
|
8
|
+
const CLOSE_GRACE_MS = 10_000;
|
|
9
|
+
|
|
10
|
+
export interface Thinking {
|
|
11
|
+
turnStarted(): void;
|
|
12
|
+
text(delta: string): void;
|
|
13
|
+
tool(name: string): void;
|
|
14
|
+
turnEnded(text: string): void;
|
|
15
|
+
exited(reason: string): void;
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
const record = (value: unknown): Record<string, unknown> => (isRecord(value) ? value : {});
|
|
19
|
+
|
|
20
|
+
function streamEvent(msg: Record<string, unknown>, mind: Thinking): void {
|
|
21
|
+
if (msg.parent_tool_use_id !== null && msg.parent_tool_use_id !== undefined) return;
|
|
22
|
+
const event = record(msg.event);
|
|
23
|
+
const delta = record(event.delta);
|
|
24
|
+
if (event.type === 'content_block_delta' && delta.type === 'text_delta' && typeof delta.text === 'string') mind.text(delta.text);
|
|
25
|
+
const block = record(event.content_block);
|
|
26
|
+
if (event.type === 'content_block_start' && block.type === 'tool_use') mind.tool(typeof block.name === 'string' ? block.name : 'tool');
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
function readBrainLine(line: string, mind: Thinking): void {
|
|
30
|
+
let parsed: unknown;
|
|
31
|
+
try {
|
|
32
|
+
parsed = JSON.parse(line);
|
|
33
|
+
} catch {
|
|
34
|
+
return;
|
|
35
|
+
}
|
|
36
|
+
const msg = record(parsed);
|
|
37
|
+
if (msg.type === 'system' && msg.subtype === 'init') mind.turnStarted();
|
|
38
|
+
else if (msg.type === 'stream_event') streamEvent(msg, mind);
|
|
39
|
+
else if (msg.type === 'result') mind.turnEnded(typeof msg.result === 'string' ? msg.result : '');
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
export class Brain {
|
|
43
|
+
private readonly child: ChildProcessWithoutNullStreams;
|
|
44
|
+
private out = '';
|
|
45
|
+
private gone = false;
|
|
46
|
+
|
|
47
|
+
constructor(model: string, private readonly mind: Thinking) {
|
|
48
|
+
const [command = 'metro', ...args] = metroCli(['voice', '--model', model]);
|
|
49
|
+
const [file, argv] = asAgent(command, args);
|
|
50
|
+
this.child = spawn(file, argv, { stdio: ['pipe', 'pipe', 'pipe'], cwd: '/' });
|
|
51
|
+
this.child.stdout.setEncoding('utf8');
|
|
52
|
+
this.child.stdout.on('data', (chunk: string) => {
|
|
53
|
+
this.out = drainLines('voice-brain', this.out + chunk, (line) => {
|
|
54
|
+
readBrainLine(line, this.mind);
|
|
55
|
+
});
|
|
56
|
+
});
|
|
57
|
+
this.child.stderr.setEncoding('utf8');
|
|
58
|
+
this.child.stderr.on('data', (chunk: string) => {
|
|
59
|
+
log.debug({ text: chunk.slice(0, 500) }, 'voice: brain stderr');
|
|
60
|
+
});
|
|
61
|
+
this.child.stdin.on('error', (err) => {
|
|
62
|
+
log.debug({ err: errMsg(err) }, 'voice: brain stdin closed');
|
|
63
|
+
});
|
|
64
|
+
this.child.on('error', (err) => {
|
|
65
|
+
this.exit(`could not start: ${errMsg(err)}`);
|
|
66
|
+
});
|
|
67
|
+
this.child.on('exit', (code, signal) => {
|
|
68
|
+
this.exit(`exited (${String(code ?? signal)})`);
|
|
69
|
+
});
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
get alive(): boolean {
|
|
73
|
+
return !this.gone;
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
tell(text: string, now = false): void {
|
|
77
|
+
if (this.gone) return;
|
|
78
|
+
const message = { type: 'user', message: { role: 'user', content: text }, parent_tool_use_id: null, session_id: '' };
|
|
79
|
+
this.child.stdin.write(`${JSON.stringify(now ? { ...message, priority: 'now' } : message)}\n`);
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
close(): void {
|
|
83
|
+
if (this.gone) return;
|
|
84
|
+
this.child.stdin.end();
|
|
85
|
+
setTimeout(() => {
|
|
86
|
+
if (!this.gone) this.child.kill('SIGKILL');
|
|
87
|
+
}, CLOSE_GRACE_MS).unref();
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
private exit(reason: string): void {
|
|
91
|
+
if (this.gone) return;
|
|
92
|
+
this.gone = true;
|
|
93
|
+
this.mind.exited(reason);
|
|
94
|
+
}
|
|
95
|
+
}
|
|
@@ -0,0 +1,156 @@
|
|
|
1
|
+
import { randomUUID } from 'node:crypto';
|
|
2
|
+
import { errMsg, log } from '@metro-labs/core/log';
|
|
3
|
+
import { isRecord } from '@metro-labs/core/is-record';
|
|
4
|
+
import { forwardTrainCall } from '../stations/train-call.js';
|
|
5
|
+
import { Peer } from './peer.js';
|
|
6
|
+
import { Talk } from './talk.js';
|
|
7
|
+
import type { VoiceConfig } from './store.js';
|
|
8
|
+
|
|
9
|
+
const OFFER_WAIT_MS = 45_000;
|
|
10
|
+
const LONGEST_CALL_MS = 60 * 60_000;
|
|
11
|
+
const CONTEXT_MESSAGES = 12;
|
|
12
|
+
const CONTEXT_CHARS = 400;
|
|
13
|
+
|
|
14
|
+
interface CallStart {
|
|
15
|
+
line: string;
|
|
16
|
+
from: string;
|
|
17
|
+
callerName: string;
|
|
18
|
+
callId: string;
|
|
19
|
+
callerPeer: string;
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
const str = (value: unknown): string => (typeof value === 'string' ? value : '');
|
|
23
|
+
|
|
24
|
+
async function trainCall(action: string, args: Record<string, unknown>): Promise<unknown> {
|
|
25
|
+
const answer = await forwardTrainCall('xmtp', action, args);
|
|
26
|
+
if (answer.error !== undefined) throw new Error(answer.error);
|
|
27
|
+
return answer.result;
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
function contextLine(row: unknown, start: CallStart): string | null {
|
|
31
|
+
if (!isRecord(row) || typeof row.text !== 'string' || row.text.startsWith('[call')) return null;
|
|
32
|
+
const who = row.from === start.from ? start.callerName : 'you';
|
|
33
|
+
return `${str(row.ts).slice(11, 16)} ${who}: ${row.text.replace(/\s+/g, ' ').slice(0, CONTEXT_CHARS)}`;
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
async function recentChat(start: CallStart): Promise<string> {
|
|
37
|
+
try {
|
|
38
|
+
const result = await trainCall('read', { line: start.line, limit: CONTEXT_MESSAGES });
|
|
39
|
+
const rows = isRecord(result) && Array.isArray(result.messages) ? result.messages : [];
|
|
40
|
+
return rows.map((row) => contextLine(row, start)).filter((l): l is string => l !== null).join('\n');
|
|
41
|
+
} catch (err) {
|
|
42
|
+
log.warn({ err: errMsg(err) }, 'voice: could not read the chat before the call');
|
|
43
|
+
return '';
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
function opening(start: CallStart, chat: string): string {
|
|
48
|
+
const recent = chat === '' ? '' : ` Recent chat messages before the call, oldest first (times in UTC):\n${chat}\n`;
|
|
49
|
+
return `[A voice call with ${start.callerName} is starting on Stage, in your chat with them.${recent} Greet ${start.callerName} in a few words.]`;
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
export class Call {
|
|
53
|
+
private readonly selfPeer = randomUUID();
|
|
54
|
+
private readonly startedAt = Date.now();
|
|
55
|
+
private readonly talk: Talk;
|
|
56
|
+
private peer: Peer | null = null;
|
|
57
|
+
private timer: ReturnType<typeof setTimeout> | null = null;
|
|
58
|
+
private longest: ReturnType<typeof setTimeout>;
|
|
59
|
+
private over = false;
|
|
60
|
+
|
|
61
|
+
constructor(
|
|
62
|
+
readonly start: CallStart,
|
|
63
|
+
cfg: VoiceConfig,
|
|
64
|
+
private readonly onOver: () => void,
|
|
65
|
+
) {
|
|
66
|
+
this.talk = new Talk(
|
|
67
|
+
cfg,
|
|
68
|
+
(opus, timestamp, marker) => {
|
|
69
|
+
this.peer?.sendOpus(opus, timestamp, marker);
|
|
70
|
+
},
|
|
71
|
+
(reason) => {
|
|
72
|
+
this.hangUp(reason);
|
|
73
|
+
},
|
|
74
|
+
);
|
|
75
|
+
this.longest = setTimeout(() => {
|
|
76
|
+
this.hangUp('the call reached one hour');
|
|
77
|
+
}, LONGEST_CALL_MS);
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
async begin(): Promise<void> {
|
|
81
|
+
log.info({ line: this.start.line, callId: this.start.callId }, 'voice: answering a call');
|
|
82
|
+
this.talk.prime(opening(this.start, await recentChat(this.start)));
|
|
83
|
+
await this.signal({ kind: 'join', callId: this.start.callId, from: this.selfPeer });
|
|
84
|
+
this.timer = setTimeout(() => {
|
|
85
|
+
this.hangUp('no offer arrived');
|
|
86
|
+
}, OFFER_WAIT_MS);
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
signalled(raw: Record<string, unknown>): void {
|
|
90
|
+
if (raw.callId !== this.start.callId || this.over) return;
|
|
91
|
+
const fromCaller = raw.from === this.start.callerPeer;
|
|
92
|
+
if (raw.kind === 'offer' && fromCaller && raw.to === this.selfPeer && this.peer === null && typeof raw.sdp === 'string')
|
|
93
|
+
this.answer(this.start.callerPeer, raw.sdp).catch((err: unknown) => {
|
|
94
|
+
this.hangUp(`answering failed: ${errMsg(err)}`);
|
|
95
|
+
});
|
|
96
|
+
else if (raw.kind === 'leave' && fromCaller) this.end('the caller hung up', false);
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
hangUp(reason: string): void {
|
|
100
|
+
this.end(reason, true);
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
private async answer(to: string, offer: string): Promise<void> {
|
|
104
|
+
if (this.timer !== null) clearTimeout(this.timer);
|
|
105
|
+
this.peer = new Peer({
|
|
106
|
+
audio: (opus) => {
|
|
107
|
+
this.talk.hear(opus);
|
|
108
|
+
},
|
|
109
|
+
state: (state) => {
|
|
110
|
+
this.stateChanged(state);
|
|
111
|
+
},
|
|
112
|
+
});
|
|
113
|
+
const sdp = await this.peer.answer(offer);
|
|
114
|
+
if (this.over) return;
|
|
115
|
+
await this.signal({ kind: 'answer', callId: this.start.callId, from: this.selfPeer, to, sdp });
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
private stateChanged(state: string): void {
|
|
119
|
+
log.info({ state, afterMs: Date.now() - this.startedAt }, 'voice: call connection');
|
|
120
|
+
if (state === 'connected') this.talk.connect();
|
|
121
|
+
else if (state === 'failed' || state === 'closed') this.hangUp(`the connection ${state}`);
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
private signal(signal: Record<string, unknown>): Promise<unknown> {
|
|
125
|
+
return trainCall('callSignal', { line: this.start.line, signal });
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
private end(reason: string, sendLeave: boolean): void {
|
|
129
|
+
if (this.over) return;
|
|
130
|
+
this.over = true;
|
|
131
|
+
if (this.timer !== null) clearTimeout(this.timer);
|
|
132
|
+
clearTimeout(this.longest);
|
|
133
|
+
log.info({ reason, seconds: Math.round((Date.now() - this.startedAt) / 1000) }, 'voice: call over');
|
|
134
|
+
this.wrapUp(reason, sendLeave)
|
|
135
|
+
.catch((err: unknown) => {
|
|
136
|
+
log.warn({ err: errMsg(err) }, 'voice: wrapping up the call failed');
|
|
137
|
+
})
|
|
138
|
+
.finally(() => {
|
|
139
|
+
this.onOver();
|
|
140
|
+
});
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
private async wrapUp(reason: string, sendLeave: boolean): Promise<void> {
|
|
144
|
+
if (sendLeave)
|
|
145
|
+
await this.signal({ kind: 'leave', callId: this.start.callId, from: this.selfPeer }).catch((err: unknown) => {
|
|
146
|
+
log.warn({ err: errMsg(err) }, 'voice: could not send the hang-up');
|
|
147
|
+
});
|
|
148
|
+
this.peer?.close();
|
|
149
|
+
const notes = await this.talk.finish();
|
|
150
|
+
const minutes = Math.max(1, Math.round((Date.now() - this.startedAt) / 60_000));
|
|
151
|
+
const timing = this.talk.replyTime === null ? '' : `\n${this.talk.replyTime}`;
|
|
152
|
+
const failure = reason.startsWith('the caller') || reason === 'no offer arrived' ? null : `The voice call ended: ${reason}.`;
|
|
153
|
+
const text = [failure, notes === null ? null : `Call notes (${String(minutes)} min):\n${notes}${timing}`].filter((part) => part !== null).join('\n\n');
|
|
154
|
+
if (text !== '') await trainCall('send', { line: this.start.line, text });
|
|
155
|
+
}
|
|
156
|
+
}
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
import { errMsg, log } from '@metro-labs/core/log';
|
|
2
|
+
import { isRecord } from '@metro-labs/core/is-record';
|
|
3
|
+
import type { TrainEvent } from '@metro-labs/core/trains/protocol';
|
|
4
|
+
import { lineReceives, mayApprove } from '../agents/map.js';
|
|
5
|
+
import { Call } from './call.js';
|
|
6
|
+
import { readVoice, voiceReady } from './store.js';
|
|
7
|
+
|
|
8
|
+
const INVITE_FRESH_MS = 60_000;
|
|
9
|
+
const CALL_TYPES = new Set(['callInvite', 'callSignal']);
|
|
10
|
+
|
|
11
|
+
let current: Call | null = null;
|
|
12
|
+
|
|
13
|
+
const record = (value: unknown): Record<string, unknown> => (isRecord(value) ? value : {});
|
|
14
|
+
const str = (value: unknown): string => (typeof value === 'string' ? value : '');
|
|
15
|
+
|
|
16
|
+
const contentTypeOf = (env: TrainEvent): string => str(record(env.payload).contentType);
|
|
17
|
+
|
|
18
|
+
export const isCallEvent = (env: TrainEvent): boolean => CALL_TYPES.has(contentTypeOf(env));
|
|
19
|
+
|
|
20
|
+
function refusal(env: TrainEvent, line: string, from: string): string | null {
|
|
21
|
+
if (current !== null) return 'already in a call';
|
|
22
|
+
const sent = Date.parse(str(env.ts));
|
|
23
|
+
if (Number.isNaN(sent) || Date.now() - sent > INVITE_FRESH_MS) return 'the invite is too old';
|
|
24
|
+
if (!voiceReady(readVoice())) return 'voice is off or has no API key on the Voice page';
|
|
25
|
+
if (!lineReceives(line) || !mayApprove(line, from)) return 'the caller is not someone who can approve for this channel';
|
|
26
|
+
return null;
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
function invite(env: TrainEvent, body: Record<string, unknown>): void {
|
|
30
|
+
const line = str(env.line);
|
|
31
|
+
const from = str(env.from);
|
|
32
|
+
const callId = str(body.callId);
|
|
33
|
+
const callerPeer = str(body.from);
|
|
34
|
+
if (callId === '' || callerPeer === '') return;
|
|
35
|
+
const refused = refusal(env, line, from);
|
|
36
|
+
if (refused !== null) {
|
|
37
|
+
log.info({ line, reason: refused }, 'voice: a call came in and is not answered');
|
|
38
|
+
return;
|
|
39
|
+
}
|
|
40
|
+
const callerName = str(env.from_display_name) || str(env.from_name) || 'the caller';
|
|
41
|
+
const call = new Call({ line, from, callerName, callId, callerPeer }, readVoice(), () => {
|
|
42
|
+
if (current === call) current = null;
|
|
43
|
+
});
|
|
44
|
+
current = call;
|
|
45
|
+
call.begin().catch((err: unknown) => {
|
|
46
|
+
call.hangUp(`starting failed: ${errMsg(err)}`);
|
|
47
|
+
});
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
export function onCallEvent(env: TrainEvent): void {
|
|
51
|
+
const body = record(record(env.payload).call);
|
|
52
|
+
if (contentTypeOf(env) === 'callInvite') invite(env, body);
|
|
53
|
+
else if (current !== null && env.line === current.start.line) current.signalled(body);
|
|
54
|
+
}
|