@stage-labs/metro 0.1.0-beta.235 → 0.1.0-beta.237
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/claude.js +1 -1
- package/dist/cli.js +14 -5
- package/dist/memory.js +2 -2
- package/dist/voice.js +39 -0
- package/package.json +1 -1
- package/runtime/marketplace/plugin/.claude-plugin/plugin.json +1 -1
- package/runtime/marketplace/plugin/MEMORY.md +2 -2
- package/runtime/node_modules/@metro-labs/daemon/package.json +2 -0
- package/runtime/node_modules/@metro-labs/daemon/src/agent-user/schedule-detail.ts +2 -2
- package/runtime/node_modules/@metro-labs/daemon/src/boot/boot.ts +5 -0
- package/runtime/node_modules/@metro-labs/daemon/src/claude/memory-routine.ts +25 -16
- package/runtime/node_modules/@metro-labs/daemon/src/claude/session.ts +8 -4
- package/runtime/node_modules/@metro-labs/daemon/src/claude/setup.ts +1 -1
- package/runtime/node_modules/@metro-labs/daemon/src/routes/session-apis.ts +2 -0
- package/runtime/node_modules/@metro-labs/daemon/src/voice/api.ts +14 -0
- package/runtime/node_modules/@metro-labs/daemon/src/voice/audio.ts +153 -0
- package/runtime/node_modules/@metro-labs/daemon/src/voice/brain.ts +95 -0
- package/runtime/node_modules/@metro-labs/daemon/src/voice/call.ts +154 -0
- package/runtime/node_modules/@metro-labs/daemon/src/voice/calls.ts +54 -0
- package/runtime/node_modules/@metro-labs/daemon/src/voice/peer.ts +84 -0
- package/runtime/node_modules/@metro-labs/daemon/src/voice/scribe.ts +102 -0
- package/runtime/node_modules/@metro-labs/daemon/src/voice/speakable.ts +54 -0
- package/runtime/node_modules/@metro-labs/daemon/src/voice/speech.ts +91 -0
- package/runtime/node_modules/@metro-labs/daemon/src/voice/store.ts +97 -0
- package/runtime/node_modules/@metro-labs/daemon/src/voice/talk.ts +234 -0
- package/runtime/node_modules/@metro-labs/xmtp/src/actions-conv.ts +2 -1
- package/runtime/node_modules/@metro-labs/xmtp/src/actions.ts +11 -1
- package/runtime/node_modules/@metro-labs/xmtp/src/codecs.ts +21 -3
- package/runtime/node_modules/@metro-labs/xmtp/src/emit-payloads.ts +2 -1
- package/runtime/runtime.json +1 -1
- package/runtime/stations.json +2 -0
|
@@ -0,0 +1,154 @@
|
|
|
1
|
+
import { randomUUID } from 'node:crypto';
|
|
2
|
+
import { errMsg, log } from '@metro-labs/core/log';
|
|
3
|
+
import { isRecord } from '@metro-labs/core/is-record';
|
|
4
|
+
import { forwardTrainCall } from '../stations/train-call.js';
|
|
5
|
+
import { Peer } from './peer.js';
|
|
6
|
+
import { Talk } from './talk.js';
|
|
7
|
+
import type { VoiceConfig } from './store.js';
|
|
8
|
+
|
|
9
|
+
const OFFER_WAIT_MS = 45_000;
|
|
10
|
+
const LONGEST_CALL_MS = 60 * 60_000;
|
|
11
|
+
const CONTEXT_MESSAGES = 12;
|
|
12
|
+
const CONTEXT_CHARS = 400;
|
|
13
|
+
|
|
14
|
+
interface CallStart {
|
|
15
|
+
line: string;
|
|
16
|
+
from: string;
|
|
17
|
+
callerName: string;
|
|
18
|
+
callId: string;
|
|
19
|
+
callerPeer: string;
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
const str = (value: unknown): string => (typeof value === 'string' ? value : '');
|
|
23
|
+
|
|
24
|
+
async function trainCall(action: string, args: Record<string, unknown>): Promise<unknown> {
|
|
25
|
+
const answer = await forwardTrainCall('xmtp', action, args);
|
|
26
|
+
if (answer.error !== undefined) throw new Error(answer.error);
|
|
27
|
+
return answer.result;
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
function contextLine(row: unknown, start: CallStart): string | null {
|
|
31
|
+
if (!isRecord(row) || typeof row.text !== 'string' || row.text.startsWith('[call')) return null;
|
|
32
|
+
const who = row.from === start.from ? start.callerName : 'you';
|
|
33
|
+
return `${str(row.ts).slice(11, 16)} ${who}: ${row.text.replace(/\s+/g, ' ').slice(0, CONTEXT_CHARS)}`;
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
async function recentChat(start: CallStart): Promise<string> {
|
|
37
|
+
try {
|
|
38
|
+
const result = await trainCall('read', { line: start.line, limit: CONTEXT_MESSAGES });
|
|
39
|
+
const rows = isRecord(result) && Array.isArray(result.messages) ? result.messages : [];
|
|
40
|
+
return rows.map((row) => contextLine(row, start)).filter((l): l is string => l !== null).join('\n');
|
|
41
|
+
} catch (err) {
|
|
42
|
+
log.warn({ err: errMsg(err) }, 'voice: could not read the chat before the call');
|
|
43
|
+
return '';
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
function opening(start: CallStart, chat: string): string {
|
|
48
|
+
const recent = chat === '' ? '' : ` Recent chat messages before the call, oldest first (times in UTC):\n${chat}\n`;
|
|
49
|
+
return `[A voice call with ${start.callerName} is starting on Stage, in your chat with them.${recent} Greet ${start.callerName} in a few words.]`;
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
export class Call {
|
|
53
|
+
private readonly selfPeer = randomUUID();
|
|
54
|
+
private readonly startedAt = Date.now();
|
|
55
|
+
private readonly talk: Talk;
|
|
56
|
+
private peer: Peer | null = null;
|
|
57
|
+
private timer: ReturnType<typeof setTimeout> | null = null;
|
|
58
|
+
private longest: ReturnType<typeof setTimeout>;
|
|
59
|
+
private over = false;
|
|
60
|
+
|
|
61
|
+
constructor(
|
|
62
|
+
readonly start: CallStart,
|
|
63
|
+
cfg: VoiceConfig,
|
|
64
|
+
private readonly onOver: () => void,
|
|
65
|
+
) {
|
|
66
|
+
this.talk = new Talk(
|
|
67
|
+
cfg,
|
|
68
|
+
(opus, timestamp, marker) => {
|
|
69
|
+
this.peer?.sendOpus(opus, timestamp, marker);
|
|
70
|
+
},
|
|
71
|
+
(reason) => {
|
|
72
|
+
this.hangUp(reason);
|
|
73
|
+
},
|
|
74
|
+
);
|
|
75
|
+
this.longest = setTimeout(() => {
|
|
76
|
+
this.hangUp('the call reached one hour');
|
|
77
|
+
}, LONGEST_CALL_MS);
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
async begin(): Promise<void> {
|
|
81
|
+
log.info({ line: this.start.line, callId: this.start.callId }, 'voice: answering a call');
|
|
82
|
+
this.talk.prime(opening(this.start, await recentChat(this.start)));
|
|
83
|
+
await this.signal({ kind: 'join', callId: this.start.callId, from: this.selfPeer });
|
|
84
|
+
this.timer = setTimeout(() => {
|
|
85
|
+
this.hangUp('no offer arrived');
|
|
86
|
+
}, OFFER_WAIT_MS);
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
signalled(raw: Record<string, unknown>): void {
|
|
90
|
+
if (raw.callId !== this.start.callId || this.over) return;
|
|
91
|
+
const fromCaller = raw.from === this.start.callerPeer;
|
|
92
|
+
if (raw.kind === 'offer' && fromCaller && raw.to === this.selfPeer && this.peer === null && typeof raw.sdp === 'string')
|
|
93
|
+
this.answer(this.start.callerPeer, raw.sdp).catch((err: unknown) => {
|
|
94
|
+
this.hangUp(`answering failed: ${errMsg(err)}`);
|
|
95
|
+
});
|
|
96
|
+
else if (raw.kind === 'leave' && fromCaller) this.end('the caller hung up', false);
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
hangUp(reason: string): void {
|
|
100
|
+
this.end(reason, true);
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
private async answer(to: string, offer: string): Promise<void> {
|
|
104
|
+
if (this.timer !== null) clearTimeout(this.timer);
|
|
105
|
+
this.peer = new Peer({
|
|
106
|
+
audio: (opus) => {
|
|
107
|
+
this.talk.hear(opus);
|
|
108
|
+
},
|
|
109
|
+
state: (state) => {
|
|
110
|
+
this.stateChanged(state);
|
|
111
|
+
},
|
|
112
|
+
});
|
|
113
|
+
const sdp = await this.peer.answer(offer);
|
|
114
|
+
await this.signal({ kind: 'answer', callId: this.start.callId, from: this.selfPeer, to, sdp });
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
private stateChanged(state: string): void {
|
|
118
|
+
log.info({ state, afterMs: Date.now() - this.startedAt }, 'voice: call connection');
|
|
119
|
+
if (state === 'connected') this.talk.connect();
|
|
120
|
+
else if (state === 'failed' || state === 'closed') this.hangUp(`the connection ${state}`);
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
private signal(signal: Record<string, unknown>): Promise<unknown> {
|
|
124
|
+
return trainCall('callSignal', { line: this.start.line, signal });
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
private end(reason: string, sendLeave: boolean): void {
|
|
128
|
+
if (this.over) return;
|
|
129
|
+
this.over = true;
|
|
130
|
+
if (this.timer !== null) clearTimeout(this.timer);
|
|
131
|
+
clearTimeout(this.longest);
|
|
132
|
+
log.info({ reason, seconds: Math.round((Date.now() - this.startedAt) / 1000) }, 'voice: call over');
|
|
133
|
+
this.wrapUp(reason, sendLeave)
|
|
134
|
+
.catch((err: unknown) => {
|
|
135
|
+
log.warn({ err: errMsg(err) }, 'voice: wrapping up the call failed');
|
|
136
|
+
})
|
|
137
|
+
.finally(() => {
|
|
138
|
+
this.onOver();
|
|
139
|
+
});
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
private async wrapUp(reason: string, sendLeave: boolean): Promise<void> {
|
|
143
|
+
if (sendLeave)
|
|
144
|
+
await this.signal({ kind: 'leave', callId: this.start.callId, from: this.selfPeer }).catch((err: unknown) => {
|
|
145
|
+
log.warn({ err: errMsg(err) }, 'voice: could not send the hang-up');
|
|
146
|
+
});
|
|
147
|
+
this.peer?.close();
|
|
148
|
+
const notes = await this.talk.finish();
|
|
149
|
+
const minutes = Math.max(1, Math.round((Date.now() - this.startedAt) / 60_000));
|
|
150
|
+
if (notes !== null) await trainCall('send', { line: this.start.line, text: `Call notes (${String(minutes)} min):\n${notes}` });
|
|
151
|
+
else if (!reason.startsWith('the caller') && reason !== 'no offer arrived')
|
|
152
|
+
await trainCall('send', { line: this.start.line, text: `The voice call ended: ${reason}.` });
|
|
153
|
+
}
|
|
154
|
+
}
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
import { errMsg, log } from '@metro-labs/core/log';
|
|
2
|
+
import { isRecord } from '@metro-labs/core/is-record';
|
|
3
|
+
import type { TrainEvent } from '@metro-labs/core/trains/protocol';
|
|
4
|
+
import { lineReceives, mayApprove } from '../agents/map.js';
|
|
5
|
+
import { Call } from './call.js';
|
|
6
|
+
import { readVoice, voiceReady } from './store.js';
|
|
7
|
+
|
|
8
|
+
const INVITE_FRESH_MS = 60_000;
|
|
9
|
+
const CALL_TYPES = new Set(['callInvite', 'callSignal']);
|
|
10
|
+
|
|
11
|
+
let current: Call | null = null;
|
|
12
|
+
|
|
13
|
+
const record = (value: unknown): Record<string, unknown> => (isRecord(value) ? value : {});
|
|
14
|
+
const str = (value: unknown): string => (typeof value === 'string' ? value : '');
|
|
15
|
+
|
|
16
|
+
const contentTypeOf = (env: TrainEvent): string => str(record(env.payload).contentType);
|
|
17
|
+
|
|
18
|
+
export const isCallEvent = (env: TrainEvent): boolean => CALL_TYPES.has(contentTypeOf(env));
|
|
19
|
+
|
|
20
|
+
function refusal(env: TrainEvent, line: string, from: string): string | null {
|
|
21
|
+
if (current !== null) return 'already in a call';
|
|
22
|
+
const sent = Date.parse(str(env.ts));
|
|
23
|
+
if (Number.isNaN(sent) || Date.now() - sent > INVITE_FRESH_MS) return 'the invite is too old';
|
|
24
|
+
if (!voiceReady(readVoice())) return 'voice is off or has no API key on the Voice page';
|
|
25
|
+
if (!lineReceives(line) || !mayApprove(line, from)) return 'the caller is not someone who can approve for this channel';
|
|
26
|
+
return null;
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
function invite(env: TrainEvent, body: Record<string, unknown>): void {
|
|
30
|
+
const line = str(env.line);
|
|
31
|
+
const from = str(env.from);
|
|
32
|
+
const callId = str(body.callId);
|
|
33
|
+
const callerPeer = str(body.from);
|
|
34
|
+
if (callId === '' || callerPeer === '') return;
|
|
35
|
+
const refused = refusal(env, line, from);
|
|
36
|
+
if (refused !== null) {
|
|
37
|
+
log.info({ line, reason: refused }, 'voice: a call came in and is not answered');
|
|
38
|
+
return;
|
|
39
|
+
}
|
|
40
|
+
const callerName = str(env.from_display_name) || str(env.from_name) || 'the caller';
|
|
41
|
+
const call = new Call({ line, from, callerName, callId, callerPeer }, readVoice(), () => {
|
|
42
|
+
if (current === call) current = null;
|
|
43
|
+
});
|
|
44
|
+
current = call;
|
|
45
|
+
call.begin().catch((err: unknown) => {
|
|
46
|
+
call.hangUp(`starting failed: ${errMsg(err)}`);
|
|
47
|
+
});
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
export function onCallEvent(env: TrainEvent): void {
|
|
51
|
+
const body = record(record(env.payload).call);
|
|
52
|
+
if (contentTypeOf(env) === 'callInvite') invite(env, body);
|
|
53
|
+
else if (current !== null && env.line === current.start.line) current.signalled(body);
|
|
54
|
+
}
|
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
import { RTCPeerConnection, RtpHeader, RtpPacket, type RTCDataChannel, type RTCRtpSender } from 'werift';
|
|
2
|
+
import { errMsg, log } from '@metro-labs/core/log';
|
|
3
|
+
|
|
4
|
+
const ICE_SERVERS = [{ urls: 'stun:stun.cloudflare.com:3478' }, { urls: 'stun:stun.l.google.com:19302' }];
|
|
5
|
+
const UDP_PORTS: [number, number] = [40_000, 40_100];
|
|
6
|
+
const GATHER_MS = 2_500;
|
|
7
|
+
const OPUS_PT = 111;
|
|
8
|
+
const MEDIA = JSON.stringify({ audio: true, video: false, screen: false });
|
|
9
|
+
|
|
10
|
+
interface PeerEvents {
|
|
11
|
+
audio(opus: Buffer): void;
|
|
12
|
+
state(state: string): void;
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
function gathered(pc: RTCPeerConnection): Promise<void> {
|
|
16
|
+
if (pc.iceGatheringState === 'complete') return Promise.resolve();
|
|
17
|
+
return new Promise((resolve) => {
|
|
18
|
+
const timer = setTimeout(resolve, GATHER_MS);
|
|
19
|
+
pc.iceGatheringStateChange.subscribe((state) => {
|
|
20
|
+
if (state !== 'complete') return;
|
|
21
|
+
clearTimeout(timer);
|
|
22
|
+
resolve();
|
|
23
|
+
});
|
|
24
|
+
});
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
export class Peer {
|
|
28
|
+
private readonly pc = new RTCPeerConnection({
|
|
29
|
+
iceServers: ICE_SERVERS,
|
|
30
|
+
bundlePolicy: 'max-bundle',
|
|
31
|
+
icePortRange: UDP_PORTS,
|
|
32
|
+
iceUseIpv6: false,
|
|
33
|
+
});
|
|
34
|
+
private readonly channel: RTCDataChannel;
|
|
35
|
+
private sender: RTCRtpSender | null = null;
|
|
36
|
+
private sequence = Math.floor(Math.random() * 0xffff);
|
|
37
|
+
|
|
38
|
+
constructor(private readonly events: PeerEvents) {
|
|
39
|
+
this.channel = this.pc.createDataChannel('media', { negotiated: true, id: 0 });
|
|
40
|
+
this.channel.stateChanged.subscribe((state) => {
|
|
41
|
+
if (state === 'open') this.channel.send(MEDIA);
|
|
42
|
+
});
|
|
43
|
+
this.pc.onTrack.subscribe((track) => {
|
|
44
|
+
if (track.kind === 'audio')
|
|
45
|
+
track.onReceiveRtp.subscribe((rtp) => {
|
|
46
|
+
this.events.audio(rtp.payload);
|
|
47
|
+
});
|
|
48
|
+
});
|
|
49
|
+
this.pc.connectionStateChange.subscribe((state) => {
|
|
50
|
+
this.events.state(state);
|
|
51
|
+
});
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
async answer(offer: string): Promise<string> {
|
|
55
|
+
await this.pc.setRemoteDescription({ type: 'offer', sdp: offer });
|
|
56
|
+
for (const transceiver of this.pc.getTransceivers()) {
|
|
57
|
+
if (transceiver.kind === 'audio') {
|
|
58
|
+
transceiver.setDirection('sendrecv');
|
|
59
|
+
this.sender = transceiver.sender;
|
|
60
|
+
} else transceiver.setDirection('inactive');
|
|
61
|
+
}
|
|
62
|
+
await this.pc.setLocalDescription(await this.pc.createAnswer());
|
|
63
|
+
await gathered(this.pc);
|
|
64
|
+
const sdp = this.pc.localDescription?.sdp;
|
|
65
|
+
if (sdp === undefined) throw new Error('no local description after the answer');
|
|
66
|
+
return sdp;
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
sendOpus(payload: Buffer, timestamp: number, marker: boolean): void {
|
|
70
|
+
const sender = this.sender;
|
|
71
|
+
if (sender === null) return;
|
|
72
|
+
this.sequence = (this.sequence + 1) & 0xffff;
|
|
73
|
+
const header = new RtpHeader({ payloadType: OPUS_PT, sequenceNumber: this.sequence, timestamp, marker });
|
|
74
|
+
sender.sendRtp(new RtpPacket(header, payload)).catch((err: unknown) => {
|
|
75
|
+
log.debug({ err: errMsg(err) }, 'voice: an audio packet was not sent');
|
|
76
|
+
});
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
close(): void {
|
|
80
|
+
this.pc.close().catch((err: unknown) => {
|
|
81
|
+
log.debug({ err: errMsg(err) }, 'voice: closing the call connection failed');
|
|
82
|
+
});
|
|
83
|
+
}
|
|
84
|
+
}
|
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
import WebSocket from 'ws';
|
|
2
|
+
import { errMsg, log } from '@metro-labs/core/log';
|
|
3
|
+
import { isRecord } from '@metro-labs/core/is-record';
|
|
4
|
+
import { bytesOf } from './audio.js';
|
|
5
|
+
|
|
6
|
+
export const elevenLabs = (): string => process.env.METRO_ELEVENLABS_URL ?? 'wss://api.elevenlabs.io';
|
|
7
|
+
const STT_RATE = 16_000;
|
|
8
|
+
const CHUNK_SAMPLES = STT_RATE / 10;
|
|
9
|
+
const HELD_SAMPLES = STT_RATE * 2;
|
|
10
|
+
export const VAD_SILENCE_SECS = 0.5;
|
|
11
|
+
|
|
12
|
+
interface Heard {
|
|
13
|
+
partial(text: string): void;
|
|
14
|
+
committed(text: string): void;
|
|
15
|
+
failed(reason: string): void;
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
const FATAL = new Set([
|
|
19
|
+
'auth_error',
|
|
20
|
+
'quota_exceeded',
|
|
21
|
+
'unaccepted_terms',
|
|
22
|
+
'rate_limited',
|
|
23
|
+
'resource_exhausted',
|
|
24
|
+
'session_time_limit_exceeded',
|
|
25
|
+
'error',
|
|
26
|
+
]);
|
|
27
|
+
|
|
28
|
+
export function wsText(data: WebSocket.RawData): string {
|
|
29
|
+
if (Array.isArray(data)) return Buffer.concat(data).toString('utf8');
|
|
30
|
+
return (Buffer.isBuffer(data) ? data : Buffer.from(data)).toString('utf8');
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
export function parsed(raw: string): Record<string, unknown> {
|
|
34
|
+
try {
|
|
35
|
+
const msg: unknown = JSON.parse(raw);
|
|
36
|
+
return isRecord(msg) ? msg : {};
|
|
37
|
+
} catch {
|
|
38
|
+
return {};
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
function sttUrl(): string {
|
|
43
|
+
const query = new URLSearchParams({
|
|
44
|
+
model_id: 'scribe_v2_realtime',
|
|
45
|
+
audio_format: 'pcm_16000',
|
|
46
|
+
commit_strategy: 'vad',
|
|
47
|
+
vad_silence_threshold_secs: String(VAD_SILENCE_SECS),
|
|
48
|
+
});
|
|
49
|
+
return `${elevenLabs()}/v1/speech-to-text/realtime?${query.toString()}`;
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
export class Scribe {
|
|
53
|
+
private readonly ws: WebSocket;
|
|
54
|
+
private buffer: Int16Array[] = [];
|
|
55
|
+
private buffered = 0;
|
|
56
|
+
|
|
57
|
+
constructor(apiKey: string, private readonly heard: Heard) {
|
|
58
|
+
this.ws = new WebSocket(sttUrl(), { headers: { 'xi-api-key': apiKey } });
|
|
59
|
+
this.ws.on('message', (data) => {
|
|
60
|
+
this.onMessage(wsText(data));
|
|
61
|
+
});
|
|
62
|
+
this.ws.on('error', (err) => {
|
|
63
|
+
log.warn({ err: errMsg(err) }, 'voice: speech to text connection error');
|
|
64
|
+
});
|
|
65
|
+
this.ws.on('close', (code) => {
|
|
66
|
+
if (code !== 1000) this.heard.failed(`speech to text closed (${String(code)})`);
|
|
67
|
+
});
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
send(pcm16k: Int16Array): void {
|
|
71
|
+
this.buffer.push(pcm16k);
|
|
72
|
+
this.buffered += pcm16k.length;
|
|
73
|
+
while (this.buffered > HELD_SAMPLES) this.buffered -= this.buffer.shift()?.length ?? this.buffered;
|
|
74
|
+
if (this.buffered < CHUNK_SAMPLES || this.ws.readyState !== WebSocket.OPEN) return;
|
|
75
|
+
const chunk = new Int16Array(this.buffered);
|
|
76
|
+
let at = 0;
|
|
77
|
+
for (const part of this.buffer) {
|
|
78
|
+
chunk.set(part, at);
|
|
79
|
+
at += part.length;
|
|
80
|
+
}
|
|
81
|
+
this.buffer = [];
|
|
82
|
+
this.buffered = 0;
|
|
83
|
+
this.ws.send(
|
|
84
|
+
JSON.stringify({ message_type: 'input_audio_chunk', audio_base_64: bytesOf(chunk).toString('base64'), commit: false, sample_rate: STT_RATE }),
|
|
85
|
+
);
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
close(): void {
|
|
89
|
+
this.ws.close(1000);
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
private onMessage(raw: string): void {
|
|
93
|
+
const msg = parsed(raw);
|
|
94
|
+
const type = typeof msg.message_type === 'string' ? msg.message_type : '';
|
|
95
|
+
const text = typeof msg.text === 'string' ? msg.text.trim() : '';
|
|
96
|
+
if (type === 'partial_transcript') this.heard.partial(text);
|
|
97
|
+
else if (type === 'committed_transcript') {
|
|
98
|
+
if (text !== '') this.heard.committed(text);
|
|
99
|
+
} else if (FATAL.has(type)) this.heard.failed(`speech to text: ${typeof msg.error === 'string' ? msg.error : type}`);
|
|
100
|
+
else log.debug({ type }, 'voice: speech to text said');
|
|
101
|
+
}
|
|
102
|
+
}
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
const FIRST_CLAUSE_MIN = 24;
|
|
2
|
+
const LONGEST = 220;
|
|
3
|
+
const SENTENCE_END = /[.!?…]+["')\]]*\s+|\n+/g;
|
|
4
|
+
const CLAUSE_END = /[,;:]\s+/g;
|
|
5
|
+
|
|
6
|
+
function speakable(text: string): string {
|
|
7
|
+
return text
|
|
8
|
+
.replace(/https?:\/\/\S+/g, 'the link')
|
|
9
|
+
.replace(/`{1,3}[^`]*`{1,3}/g, (code) => code.replace(/`/g, ''))
|
|
10
|
+
.replace(/^\s*(?:#+|[-*•]|\d+\.)\s+/gm, '')
|
|
11
|
+
.replace(/[*_~#|<>[\]{}]/g, '')
|
|
12
|
+
.replace(/\s+/g, ' ')
|
|
13
|
+
.trim();
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
function lastEnd(text: string, pattern: RegExp): number {
|
|
17
|
+
let end = -1;
|
|
18
|
+
for (const match of text.matchAll(pattern)) end = match.index + match[0].length;
|
|
19
|
+
return end;
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
function cutAt(text: string, first: boolean): number {
|
|
23
|
+
const sentence = lastEnd(text, SENTENCE_END);
|
|
24
|
+
if (sentence > 0) return sentence;
|
|
25
|
+
if (first && text.length >= FIRST_CLAUSE_MIN) {
|
|
26
|
+
const clause = lastEnd(text, CLAUSE_END);
|
|
27
|
+
if (clause > 0) return clause;
|
|
28
|
+
}
|
|
29
|
+
if (text.length < LONGEST) return -1;
|
|
30
|
+
const space = text.lastIndexOf(' ');
|
|
31
|
+
return space > 0 ? space + 1 : text.length;
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
export class Chunker {
|
|
35
|
+
private buffer = '';
|
|
36
|
+
private first = true;
|
|
37
|
+
|
|
38
|
+
add(delta: string): string[] {
|
|
39
|
+
this.buffer += delta;
|
|
40
|
+
const at = cutAt(this.buffer, this.first);
|
|
41
|
+
if (at < 0) return [];
|
|
42
|
+
const ready = speakable(this.buffer.slice(0, at));
|
|
43
|
+
this.buffer = this.buffer.slice(at);
|
|
44
|
+
if (ready === '') return [];
|
|
45
|
+
this.first = false;
|
|
46
|
+
return [ready];
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
flush(): string[] {
|
|
50
|
+
const rest = speakable(this.buffer);
|
|
51
|
+
this.buffer = '';
|
|
52
|
+
return rest === '' ? [] : [rest];
|
|
53
|
+
}
|
|
54
|
+
}
|
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
import WebSocket from 'ws';
|
|
2
|
+
import { errMsg, log } from '@metro-labs/core/log';
|
|
3
|
+
import { isRecord } from '@metro-labs/core/is-record';
|
|
4
|
+
import { pcmOf, upsample2 } from './audio.js';
|
|
5
|
+
import { elevenLabs, parsed, wsText } from './scribe.js';
|
|
6
|
+
|
|
7
|
+
const TTS_MODEL = 'eleven_flash_v2_5';
|
|
8
|
+
|
|
9
|
+
interface Voiced {
|
|
10
|
+
audio(pcm48k: Int16Array, chars: number): void;
|
|
11
|
+
done(): void;
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
function ttsUrl(voiceId: string): string {
|
|
15
|
+
const query = new URLSearchParams({
|
|
16
|
+
model_id: TTS_MODEL,
|
|
17
|
+
output_format: 'pcm_24000',
|
|
18
|
+
auto_mode: 'true',
|
|
19
|
+
inactivity_timeout: '180',
|
|
20
|
+
});
|
|
21
|
+
return `${elevenLabs()}/v1/text-to-speech/${encodeURIComponent(voiceId)}/stream-input?${query.toString()}`;
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
const charsOf = (msg: Record<string, unknown>): number => {
|
|
25
|
+
const alignment = msg.alignment;
|
|
26
|
+
return isRecord(alignment) && Array.isArray(alignment.chars) ? alignment.chars.length : 0;
|
|
27
|
+
};
|
|
28
|
+
|
|
29
|
+
export class Utterance {
|
|
30
|
+
private readonly ws: WebSocket;
|
|
31
|
+
private readonly outbox: string[] = [];
|
|
32
|
+
private listener: Voiced | null = null;
|
|
33
|
+
private carry = 0;
|
|
34
|
+
private closed = false;
|
|
35
|
+
|
|
36
|
+
constructor(apiKey: string, voiceId: string) {
|
|
37
|
+
this.ws = new WebSocket(ttsUrl(voiceId), { headers: { 'xi-api-key': apiKey } });
|
|
38
|
+
this.ws.on('open', () => {
|
|
39
|
+
this.ws.send(JSON.stringify({ text: ' ' }));
|
|
40
|
+
for (const line of this.outbox.splice(0)) this.ws.send(line);
|
|
41
|
+
});
|
|
42
|
+
this.ws.on('message', (data) => {
|
|
43
|
+
this.onMessage(wsText(data));
|
|
44
|
+
});
|
|
45
|
+
this.ws.on('error', (err) => {
|
|
46
|
+
log.warn({ err: errMsg(err) }, 'voice: text to speech connection error');
|
|
47
|
+
});
|
|
48
|
+
this.ws.on('close', () => {
|
|
49
|
+
this.closed = true;
|
|
50
|
+
this.listener?.done();
|
|
51
|
+
this.listener = null;
|
|
52
|
+
});
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
get usable(): boolean {
|
|
56
|
+
return !this.closed && this.listener === null && this.ws.readyState <= WebSocket.OPEN;
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
attach(listener: Voiced): void {
|
|
60
|
+
this.listener = listener;
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
say(text: string): void {
|
|
64
|
+
this.post({ text: `${text} ` });
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
end(): void {
|
|
68
|
+
this.post({ text: '' });
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
abort(): void {
|
|
72
|
+
this.listener = null;
|
|
73
|
+
this.ws.terminate();
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
private post(msg: Record<string, unknown>): void {
|
|
77
|
+
const line = JSON.stringify(msg);
|
|
78
|
+
if (this.ws.readyState === WebSocket.OPEN) this.ws.send(line);
|
|
79
|
+
else this.outbox.push(line);
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
private onMessage(raw: string): void {
|
|
83
|
+
const msg = parsed(raw);
|
|
84
|
+
if (typeof msg.audio === 'string' && msg.audio !== '') {
|
|
85
|
+
const pcm = pcmOf(Buffer.from(msg.audio, 'base64'));
|
|
86
|
+
const up = upsample2(pcm, this.carry);
|
|
87
|
+
this.carry = pcm.at(-1) ?? this.carry;
|
|
88
|
+
this.listener?.audio(up, charsOf(msg));
|
|
89
|
+
} else if (msg.isFinal !== true) log.warn({ msg: raw.slice(0, 300) }, 'voice: text to speech said');
|
|
90
|
+
}
|
|
91
|
+
}
|
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
import { join } from 'node:path';
|
|
2
|
+
import { readJson, writeSecure } from '@metro-labs/core/secure-fs';
|
|
3
|
+
import { isRecord } from '@metro-labs/core/is-record';
|
|
4
|
+
import { ApiError } from '@metro-labs/http/api-error';
|
|
5
|
+
import { agentsDir } from '../agents/files.js';
|
|
6
|
+
import { readModelConfig, type ModelConfig } from '../gateway/model-config.js';
|
|
7
|
+
|
|
8
|
+
const VOICE_PROVIDERS = ['elevenlabs'] as const;
|
|
9
|
+
type VoiceProvider = (typeof VOICE_PROVIDERS)[number];
|
|
10
|
+
|
|
11
|
+
export interface VoiceConfig {
|
|
12
|
+
provider: VoiceProvider;
|
|
13
|
+
apiKey: string;
|
|
14
|
+
voiceId: string;
|
|
15
|
+
model: string;
|
|
16
|
+
enabled: boolean;
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
const DEFAULT_VOICE_ID = 'EXAVITQu4vr4xnSDxMaL';
|
|
20
|
+
const DEFAULT_VOICE_MODEL = 'anthropic:claude-sonnet-5-5';
|
|
21
|
+
|
|
22
|
+
const VOICE_FILE = 'voice.json';
|
|
23
|
+
const MAX_FIELD = 512;
|
|
24
|
+
|
|
25
|
+
const text = (value: unknown): string => (typeof value === 'string' ? value.trim() : '');
|
|
26
|
+
|
|
27
|
+
const isProvider = (value: unknown): value is VoiceProvider =>
|
|
28
|
+
typeof value === 'string' && (VOICE_PROVIDERS as readonly string[]).includes(value);
|
|
29
|
+
|
|
30
|
+
export function parseVoice(raw: unknown): VoiceConfig {
|
|
31
|
+
const r = isRecord(raw) ? raw : {};
|
|
32
|
+
return {
|
|
33
|
+
provider: isProvider(r.provider) ? r.provider : 'elevenlabs',
|
|
34
|
+
apiKey: text(r.apiKey),
|
|
35
|
+
voiceId: text(r.voiceId),
|
|
36
|
+
model: text(r.model),
|
|
37
|
+
enabled: r.enabled !== false,
|
|
38
|
+
};
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
export const readVoice = (dir = agentsDir()): VoiceConfig =>
|
|
42
|
+
parseVoice(readJson<unknown>(join(dir, VOICE_FILE), null, { warn: 'voice: voice.json is not readable' }));
|
|
43
|
+
|
|
44
|
+
export function writeVoice(cfg: VoiceConfig, dir = agentsDir()): void {
|
|
45
|
+
writeSecure(join(dir, VOICE_FILE), JSON.stringify(cfg, null, 2));
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
export const voiceIdOf = (cfg: VoiceConfig): string => (cfg.voiceId === '' ? DEFAULT_VOICE_ID : cfg.voiceId);
|
|
49
|
+
const voiceModelOf = (cfg: VoiceConfig): string => (cfg.model === '' ? DEFAULT_VOICE_MODEL : cfg.model);
|
|
50
|
+
export function brainModel(cfg: VoiceConfig, models: ModelConfig = readModelConfig()): string {
|
|
51
|
+
const model = voiceModelOf(cfg);
|
|
52
|
+
const anthropic = /^anthropic:(.+)$/.exec(model)?.[1];
|
|
53
|
+
if (anthropic === undefined) return model;
|
|
54
|
+
return models.connections.some((c) => c.provider === 'anthropic') ? model : anthropic;
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
export const voiceReady = (cfg: VoiceConfig): boolean => cfg.enabled && cfg.apiKey !== '';
|
|
58
|
+
|
|
59
|
+
export function publicVoice(cfg: VoiceConfig): Record<string, unknown> {
|
|
60
|
+
return {
|
|
61
|
+
provider: cfg.provider,
|
|
62
|
+
providers: VOICE_PROVIDERS,
|
|
63
|
+
hasKey: cfg.apiKey !== '',
|
|
64
|
+
voiceId: cfg.voiceId,
|
|
65
|
+
model: cfg.model,
|
|
66
|
+
enabled: cfg.enabled,
|
|
67
|
+
defaults: { voiceId: DEFAULT_VOICE_ID, model: DEFAULT_VOICE_MODEL },
|
|
68
|
+
};
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
function field(body: Record<string, unknown>, key: string, current: string): string {
|
|
72
|
+
const value = body[key];
|
|
73
|
+
if (value === undefined) return current;
|
|
74
|
+
if (typeof value !== 'string') throw new ApiError(`${key} must be a string`, 400);
|
|
75
|
+
if (value.length > MAX_FIELD) throw new ApiError(`${key} is too long`, 400);
|
|
76
|
+
return value.trim();
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
function keyOf(body: Record<string, unknown>, current: string): string {
|
|
80
|
+
if (body.apiKey === null) return '';
|
|
81
|
+
const next = field(body, 'apiKey', current);
|
|
82
|
+
return next === '' ? current : next;
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
export function patchVoice(cfg: VoiceConfig, body: unknown): VoiceConfig {
|
|
86
|
+
if (!isRecord(body)) throw new ApiError('the body must be a JSON object', 400);
|
|
87
|
+
if (body.provider !== undefined && !isProvider(body.provider))
|
|
88
|
+
throw new ApiError(`provider must be one of ${VOICE_PROVIDERS.join(', ')}`, 400);
|
|
89
|
+
if (body.enabled !== undefined && typeof body.enabled !== 'boolean') throw new ApiError('enabled must be true or false', 400);
|
|
90
|
+
return {
|
|
91
|
+
provider: isProvider(body.provider) ? body.provider : cfg.provider,
|
|
92
|
+
apiKey: keyOf(body, cfg.apiKey),
|
|
93
|
+
voiceId: field(body, 'voiceId', cfg.voiceId),
|
|
94
|
+
model: field(body, 'model', cfg.model),
|
|
95
|
+
enabled: typeof body.enabled === 'boolean' ? body.enabled : cfg.enabled,
|
|
96
|
+
};
|
|
97
|
+
}
|