@stage-labs/metro 0.1.0-beta.263 → 0.1.0-beta.265
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli.js +2 -2
- package/dist/runtime-install.js +22 -5
- package/package.json +1 -1
- package/runtime/marketplace/plugin/.claude-plugin/plugin.json +1 -1
- package/runtime/marketplace/plugin/bin/policy.mjs +1 -1
- package/runtime/node_modules/@metro-labs/core/package.json +1 -0
- package/runtime/node_modules/@metro-labs/core/src/call.ts +81 -0
- package/runtime/node_modules/@metro-labs/core/src/runner-activity.ts +37 -1
- package/runtime/node_modules/@metro-labs/core/src/stations/train-boot.ts +22 -1
- package/runtime/node_modules/@metro-labs/daemon/src/approvals/pending.ts +35 -2
- package/runtime/node_modules/@metro-labs/daemon/src/boot/boot.ts +1 -1
- package/runtime/node_modules/@metro-labs/daemon/src/channels/inbound.ts +14 -11
- package/runtime/node_modules/@metro-labs/daemon/src/channels/pending.ts +3 -0
- package/runtime/node_modules/@metro-labs/daemon/src/mcp/call-bridge.ts +45 -0
- package/runtime/node_modules/@metro-labs/daemon/src/mcp/call-permission.ts +25 -0
- package/runtime/node_modules/@metro-labs/daemon/src/mcp/call-tools.ts +2 -0
- package/runtime/node_modules/@metro-labs/daemon/src/mcp/event-store.ts +9 -2
- package/runtime/node_modules/@metro-labs/daemon/src/mcp/index.ts +2 -0
- package/runtime/node_modules/@metro-labs/daemon/src/mcp/permission-prompt.ts +2 -1
- package/runtime/node_modules/@metro-labs/daemon/src/mcp/permission-relay.ts +15 -4
- package/runtime/node_modules/@metro-labs/daemon/src/mcp/policy-gate.ts +3 -3
- package/runtime/node_modules/@metro-labs/daemon/src/mcp/session.ts +20 -1
- package/runtime/node_modules/@metro-labs/daemon/src/mcp/speech-schema.ts +11 -0
- package/runtime/node_modules/@metro-labs/daemon/src/mcp/speech-send.ts +15 -0
- package/runtime/node_modules/@metro-labs/daemon/src/mcp/tool-schemas.ts +3 -1
- package/runtime/node_modules/@metro-labs/daemon/src/stations/runtime-deps.ts +23 -7
- package/runtime/node_modules/@metro-labs/daemon/src/voice/audio.ts +50 -10
- package/runtime/node_modules/@metro-labs/daemon/src/voice/brain.ts +20 -9
- package/runtime/node_modules/@metro-labs/daemon/src/voice/call.ts +125 -51
- package/runtime/node_modules/@metro-labs/daemon/src/voice/calls.ts +41 -15
- package/runtime/node_modules/@metro-labs/daemon/src/voice/peer.ts +9 -8
- package/runtime/node_modules/@metro-labs/daemon/src/voice/scribe.ts +15 -3
- package/runtime/node_modules/@metro-labs/daemon/src/voice/shared-talk.ts +85 -0
- package/runtime/node_modules/@metro-labs/daemon/src/voice/shared.ts +199 -0
- package/runtime/node_modules/@metro-labs/daemon/src/voice/speech-queue.ts +173 -0
- package/runtime/node_modules/@metro-labs/daemon/src/voice/speech.ts +40 -15
- package/runtime/node_modules/@metro-labs/daemon/src/voice/talk.ts +15 -7
- package/runtime/node_modules/@metro-labs/whatsapp/src/actions.ts +32 -1
- package/runtime/node_modules/@metro-labs/whatsapp/src/app-state.ts +49 -0
- package/runtime/node_modules/@metro-labs/whatsapp/src/client-history.ts +31 -0
- package/runtime/node_modules/@metro-labs/whatsapp/src/client-types.ts +46 -0
- package/runtime/node_modules/@metro-labs/whatsapp/src/client.ts +75 -77
- package/runtime/node_modules/@metro-labs/whatsapp/src/history-events.ts +127 -0
- package/runtime/node_modules/@metro-labs/whatsapp/src/history-files.ts +39 -0
- package/runtime/node_modules/@metro-labs/whatsapp/src/history-identity.ts +58 -0
- package/runtime/node_modules/@metro-labs/whatsapp/src/history-message.ts +161 -0
- package/runtime/node_modules/@metro-labs/whatsapp/src/history-state.ts +344 -0
- package/runtime/node_modules/@metro-labs/whatsapp/src/history-types.ts +91 -0
- package/runtime/node_modules/@metro-labs/whatsapp/src/history.ts +195 -0
- package/runtime/node_modules/@metro-labs/whatsapp/src/index.ts +1 -1
- package/runtime/node_modules/@metro-labs/whatsapp/src/members.ts +94 -0
- package/runtime/node_modules/@metro-labs/whatsapp/src/messages.ts +70 -0
- package/runtime/node_modules/@metro-labs/whatsapp/src/send-expiry.ts +65 -0
- package/runtime/node_modules/@metro-labs/whatsapp/src/station.ts +4 -0
- package/runtime/patches/5bca2c07d5852d3f58453cff0448dd85da13d3665b61e3d758905b298d432d7c.patch +31 -0
- package/runtime/runtime.json +1 -1
- package/runtime/sdk-runner/node_modules/@metro-labs/core/package.json +1 -0
- package/runtime/sdk-runner/node_modules/@metro-labs/core/src/call.ts +81 -0
- package/runtime/sdk-runner/node_modules/@metro-labs/core/src/runner-activity.ts +37 -1
- package/runtime/sdk-runner/node_modules/@metro-labs/core/src/stations/train-boot.ts +22 -1
- package/runtime/sdk-runner/runner.json +1 -1
- package/runtime/sdk-runner/src/activity.ts +14 -2
- package/runtime/sdk-runner/src/app.ts +5 -4
- package/runtime/sdk-runner/src/approvals.ts +14 -5
- package/runtime/sdk-runner/src/calls.ts +133 -0
- package/runtime/sdk-runner/src/inbox.ts +58 -26
- package/runtime/sdk-runner/src/input-queue.ts +70 -0
- package/runtime/sdk-runner/src/input-timing.ts +68 -0
- package/runtime/sdk-runner/src/link.ts +27 -4
- package/runtime/sdk-runner/src/main.ts +0 -4
- package/runtime/sdk-runner/src/rules.ts +19 -5
- package/runtime/sdk-runner/src/runner.ts +64 -88
- package/runtime/sdk-runner/src/session-store.ts +14 -1
- package/runtime/sdk-runner/src/session-watch.ts +27 -20
- package/runtime/stations.json +3 -0
- package/runtime/sdk-runner/src/speech.ts +0 -79
|
@@ -1,9 +1,13 @@
|
|
|
1
1
|
import { randomUUID } from 'node:crypto';
|
|
2
|
+
import type { CallRoute } from '@metro-labs/core/call';
|
|
2
3
|
import { errMsg, log } from '@metro-labs/core/log';
|
|
3
4
|
import { isRecord } from '@metro-labs/core/is-record';
|
|
4
5
|
import { forwardTrainCall } from '../stations/train-call.js';
|
|
5
6
|
import { CallApprovals } from './approvals.js';
|
|
6
|
-
import {
|
|
7
|
+
import type { Playback } from './audio.js';
|
|
8
|
+
import { Peer, type PeerEvents } from './peer.js';
|
|
9
|
+
import { SharedTalk } from './shared-talk.js';
|
|
10
|
+
import { endSharedCall, hearSharedCall, openSharedCall, sharedCallsSelected, type SharedCallTransport } from './shared.js';
|
|
7
11
|
import { Talk } from './talk.js';
|
|
8
12
|
import type { VoiceConfig } from './store.js';
|
|
9
13
|
|
|
@@ -12,8 +16,11 @@ const LONGEST_CALL_MS = 60 * 60_000;
|
|
|
12
16
|
const CONTEXT_MESSAGES = 12;
|
|
13
17
|
const CONTEXT_CHARS = 400;
|
|
14
18
|
const LEAVE_RETRY_MS = 3_000;
|
|
19
|
+
const SIGNAL_WAIT_MS = 2_000;
|
|
15
20
|
|
|
16
|
-
interface CallStart {
|
|
21
|
+
export interface CallStart {
|
|
22
|
+
agentId: string;
|
|
23
|
+
sourceId: string;
|
|
17
24
|
line: string;
|
|
18
25
|
lineName: string;
|
|
19
26
|
direct: boolean;
|
|
@@ -23,6 +30,23 @@ interface CallStart {
|
|
|
23
30
|
callerPeer: string;
|
|
24
31
|
}
|
|
25
32
|
|
|
33
|
+
type CallAudio = Pick<Talk, 'connect' | 'hear' | 'finish'>;
|
|
34
|
+
type CallPeer = Pick<Peer, 'answer' | 'sendOpus' | 'close'>;
|
|
35
|
+
type ContextStart = Omit<CallStart, 'agentId' | 'sourceId'>;
|
|
36
|
+
|
|
37
|
+
export interface CallDependencies {
|
|
38
|
+
train: typeof trainCall;
|
|
39
|
+
peer(events: PeerEvents): CallPeer;
|
|
40
|
+
cli(cfg: VoiceConfig, out: Playback['send'], ended: (reason: string) => void, approvals: CallApprovals): CallAudio & Pick<Talk, 'prime'>;
|
|
41
|
+
shared(cfg: VoiceConfig, out: Playback['send'], ended: (reason: string) => void, heard: (text: string, sourceId: string) => void): CallAudio & SharedCallTransport;
|
|
42
|
+
selected: typeof sharedCallsSelected;
|
|
43
|
+
open: typeof openSharedCall;
|
|
44
|
+
heard: typeof hearSharedCall;
|
|
45
|
+
ended: typeof endSharedCall;
|
|
46
|
+
signalWaitMs: number;
|
|
47
|
+
leaveRetryMs: number;
|
|
48
|
+
}
|
|
49
|
+
|
|
26
50
|
const str = (value: unknown): string => (typeof value === 'string' ? value : '');
|
|
27
51
|
|
|
28
52
|
export async function trainCall(action: string, args: Record<string, unknown>): Promise<unknown> {
|
|
@@ -31,20 +55,33 @@ export async function trainCall(action: string, args: Record<string, unknown>):
|
|
|
31
55
|
return answer.result;
|
|
32
56
|
}
|
|
33
57
|
|
|
34
|
-
|
|
58
|
+
const dependencies: CallDependencies = {
|
|
59
|
+
train: trainCall,
|
|
60
|
+
peer: (events) => new Peer(events),
|
|
61
|
+
cli: (cfg, out, ended, approvals) => new Talk(cfg, out, ended, approvals),
|
|
62
|
+
shared: (cfg, out, ended, heard) => new SharedTalk(cfg, out, ended, heard),
|
|
63
|
+
selected: sharedCallsSelected,
|
|
64
|
+
open: openSharedCall,
|
|
65
|
+
heard: hearSharedCall,
|
|
66
|
+
ended: endSharedCall,
|
|
67
|
+
signalWaitMs: SIGNAL_WAIT_MS,
|
|
68
|
+
leaveRetryMs: LEAVE_RETRY_MS,
|
|
69
|
+
};
|
|
70
|
+
|
|
71
|
+
function speaker(row: Record<string, unknown>, start: ContextStart): string {
|
|
35
72
|
if (row.self === true) return 'you';
|
|
36
73
|
return row.from === start.from ? start.callerName : `another member (${str(row.from)})`;
|
|
37
74
|
}
|
|
38
75
|
|
|
39
|
-
function contextLine(row: unknown, start:
|
|
76
|
+
function contextLine(row: unknown, start: ContextStart): string | null {
|
|
40
77
|
if (!isRecord(row) || typeof row.text !== 'string' || row.text.startsWith('[call')) return null;
|
|
41
78
|
const who = speaker(row, start);
|
|
42
79
|
return `${str(row.ts).slice(11, 16)} ${who}: ${row.text.replace(/\s+/g, ' ').slice(0, CONTEXT_CHARS)}`;
|
|
43
80
|
}
|
|
44
81
|
|
|
45
|
-
async function recentChat(start: CallStart): Promise<string> {
|
|
82
|
+
async function recentChat(start: CallStart, train: typeof trainCall): Promise<string> {
|
|
46
83
|
try {
|
|
47
|
-
const result = await
|
|
84
|
+
const result = await train('read', { line: start.line, limit: CONTEXT_MESSAGES });
|
|
48
85
|
const rows = isRecord(result) && Array.isArray(result.messages) ? result.messages : [];
|
|
49
86
|
return rows.map((row) => contextLine(row, start)).filter((l): l is string => l !== null).join('\n');
|
|
50
87
|
} catch (err) {
|
|
@@ -53,12 +90,12 @@ async function recentChat(start: CallStart): Promise<string> {
|
|
|
53
90
|
}
|
|
54
91
|
}
|
|
55
92
|
|
|
56
|
-
function where(start:
|
|
93
|
+
function where(start: ContextStart): string {
|
|
57
94
|
if (start.lineName !== '') return `the group chat "${start.lineName}"`;
|
|
58
95
|
return start.direct ? `your direct chat with ${start.callerName}` : 'a group chat with no name';
|
|
59
96
|
}
|
|
60
97
|
|
|
61
|
-
export function opening(start:
|
|
98
|
+
export function opening(start: ContextStart, chat: string): string {
|
|
62
99
|
const recent = chat === '' ? 'No earlier messages were found there.' : `Recent messages there before the call, oldest first (times in UTC):\n${chat}`;
|
|
63
100
|
return [
|
|
64
101
|
`[A voice call with ${start.callerName} is starting on Stage, in ${where(start)}.`,
|
|
@@ -70,49 +107,77 @@ export function opening(start: CallStart, chat: string): string {
|
|
|
70
107
|
}
|
|
71
108
|
|
|
72
109
|
export class Call {
|
|
110
|
+
readonly route: CallRoute;
|
|
73
111
|
private readonly selfPeer = randomUUID();
|
|
74
112
|
private readonly startedAt = Date.now();
|
|
75
|
-
private readonly talk:
|
|
76
|
-
private
|
|
113
|
+
private readonly talk: CallAudio;
|
|
114
|
+
private readonly shared: SharedCallTransport | null;
|
|
115
|
+
private readonly prime: ((context: string) => void) | null;
|
|
116
|
+
private readonly deps: CallDependencies;
|
|
117
|
+
private peer: CallPeer | null = null;
|
|
77
118
|
private timer: ReturnType<typeof setTimeout> | null = null;
|
|
78
|
-
private longest: ReturnType<typeof setTimeout>;
|
|
119
|
+
private readonly longest: ReturnType<typeof setTimeout>;
|
|
79
120
|
private over = false;
|
|
121
|
+
private begun = false;
|
|
80
122
|
private left: Promise<void> = Promise.resolve();
|
|
81
123
|
|
|
82
124
|
constructor(
|
|
83
125
|
readonly start: CallStart,
|
|
84
126
|
cfg: VoiceConfig,
|
|
85
127
|
private readonly onOver: () => void,
|
|
128
|
+
deps: Partial<CallDependencies> = {},
|
|
86
129
|
) {
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
130
|
+
this.deps = { ...dependencies, ...deps };
|
|
131
|
+
this.route = Object.freeze({ agentId: start.agentId, line: start.line, from: start.from, callId: start.callId, generation: randomUUID() });
|
|
132
|
+
const out: Playback['send'] = (opus, timestamp, marker) => {
|
|
133
|
+
if (this.over || this.peer === null) return Promise.reject(new Error('call audio is not connected'));
|
|
134
|
+
return this.peer.sendOpus(opus, timestamp, marker);
|
|
135
|
+
};
|
|
136
|
+
const ended = (reason: string): void => {
|
|
137
|
+
this.hangUp(reason);
|
|
138
|
+
};
|
|
139
|
+
if (this.deps.selected()) {
|
|
140
|
+
const shared = this.deps.shared(cfg, out, ended, (text, sourceId) => {
|
|
141
|
+
if (!this.over) this.deps.heard(this.route, text, sourceId);
|
|
142
|
+
});
|
|
143
|
+
this.talk = shared;
|
|
144
|
+
this.shared = shared;
|
|
145
|
+
this.prime = null;
|
|
146
|
+
} else {
|
|
147
|
+
const approvals = new CallApprovals(start.line, async (text) => {
|
|
148
|
+
await this.deps.train('send', { line: start.line, text });
|
|
149
|
+
});
|
|
150
|
+
const cli = this.deps.cli(cfg, out, ended, approvals);
|
|
151
|
+
this.talk = cli;
|
|
152
|
+
this.shared = null;
|
|
153
|
+
this.prime = (context) => {
|
|
154
|
+
cli.prime(context);
|
|
155
|
+
};
|
|
156
|
+
}
|
|
100
157
|
this.longest = setTimeout(() => {
|
|
101
158
|
this.hangUp('the call reached one hour');
|
|
102
159
|
}, LONGEST_CALL_MS);
|
|
103
160
|
}
|
|
104
161
|
|
|
105
162
|
async begin(): Promise<void> {
|
|
163
|
+
if (this.over || this.begun) return;
|
|
164
|
+
this.begun = true;
|
|
106
165
|
log.info({ line: this.start.line, callId: this.start.callId }, 'voice: answering a call');
|
|
107
|
-
|
|
108
|
-
.
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
166
|
+
if (this.shared !== null) {
|
|
167
|
+
if (!this.deps.open(this.route, this.start.sourceId, this.shared)) throw new Error('the shared conversation is unavailable');
|
|
168
|
+
} else {
|
|
169
|
+
recentChat(this.start, this.deps.train).then((chat) => {
|
|
170
|
+
if (!this.over) this.prime?.(opening(this.start, chat));
|
|
171
|
+
}).catch((err: unknown) => {
|
|
112
172
|
log.warn({ err: errMsg(err) }, 'voice: could not start the agent on the call');
|
|
113
173
|
});
|
|
174
|
+
}
|
|
114
175
|
await this.signal({ kind: 'join', callId: this.start.callId, from: this.selfPeer });
|
|
115
|
-
this.
|
|
176
|
+
if (this.over) {
|
|
177
|
+
await this.sendLeave();
|
|
178
|
+
return;
|
|
179
|
+
}
|
|
180
|
+
if (this.peer === null) this.timer = setTimeout(() => {
|
|
116
181
|
this.hangUp('no offer arrived');
|
|
117
182
|
}, OFFER_WAIT_MS);
|
|
118
183
|
}
|
|
@@ -138,9 +203,9 @@ export class Call {
|
|
|
138
203
|
|
|
139
204
|
private async answer(to: string, offer: string): Promise<void> {
|
|
140
205
|
if (this.timer !== null) clearTimeout(this.timer);
|
|
141
|
-
this.peer =
|
|
206
|
+
this.peer = this.deps.peer({
|
|
142
207
|
audio: (opus) => {
|
|
143
|
-
this.talk.hear(opus);
|
|
208
|
+
if (!this.over) this.talk.hear(opus);
|
|
144
209
|
},
|
|
145
210
|
state: (state) => {
|
|
146
211
|
this.stateChanged(state);
|
|
@@ -152,13 +217,14 @@ export class Call {
|
|
|
152
217
|
}
|
|
153
218
|
|
|
154
219
|
private stateChanged(state: string): void {
|
|
220
|
+
if (this.over) return;
|
|
155
221
|
log.info({ state, afterMs: Date.now() - this.startedAt }, 'voice: call connection');
|
|
156
222
|
if (state === 'connected') this.talk.connect();
|
|
157
223
|
else if (state === 'failed' || state === 'closed') this.hangUp(`the connection ${state}`);
|
|
158
224
|
}
|
|
159
225
|
|
|
160
226
|
private signal(signal: Record<string, unknown>): Promise<unknown> {
|
|
161
|
-
return
|
|
227
|
+
return this.deps.train('callSignal', { line: this.start.line, signal });
|
|
162
228
|
}
|
|
163
229
|
|
|
164
230
|
private end(reason: string): void {
|
|
@@ -167,34 +233,42 @@ export class Call {
|
|
|
167
233
|
if (this.timer !== null) clearTimeout(this.timer);
|
|
168
234
|
clearTimeout(this.longest);
|
|
169
235
|
log.info({ reason, seconds: Math.round((Date.now() - this.startedAt) / 1000) }, 'voice: call over');
|
|
170
|
-
this.
|
|
171
|
-
this.
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
.
|
|
177
|
-
|
|
178
|
-
});
|
|
236
|
+
this.talk.finish();
|
|
237
|
+
this.peer?.close();
|
|
238
|
+
if (this.shared !== null) this.deps.ended(this.route);
|
|
239
|
+
this.onOver();
|
|
240
|
+
this.left = this.sendLeave().then(async () => {
|
|
241
|
+
if (this.shared !== null || reason.startsWith('the caller') || reason === 'no offer arrived') return;
|
|
242
|
+
await this.deps.train('send', { line: this.start.line, text: `The voice call ended: ${reason}.` });
|
|
243
|
+
}).catch((err: unknown) => {
|
|
244
|
+
log.warn({ err: errMsg(err) }, 'voice: wrapping up the call failed');
|
|
245
|
+
});
|
|
179
246
|
}
|
|
180
247
|
|
|
181
248
|
private async sendLeave(): Promise<void> {
|
|
182
|
-
const leave = { kind: 'leave', callId: this.start.callId, from: this.selfPeer };
|
|
183
249
|
try {
|
|
184
|
-
await this.
|
|
250
|
+
await this.leaveOnce();
|
|
185
251
|
} catch (err) {
|
|
186
252
|
log.warn({ err: errMsg(err) }, 'voice: could not send the hang-up, trying once more');
|
|
187
|
-
await new Promise((r) => setTimeout(r,
|
|
188
|
-
await this.
|
|
253
|
+
await new Promise((r) => setTimeout(r, this.deps.leaveRetryMs));
|
|
254
|
+
await this.leaveOnce().catch((again: unknown) => {
|
|
189
255
|
log.warn({ err: errMsg(again) }, 'voice: could not send the hang-up; the next start sends it');
|
|
190
256
|
});
|
|
191
257
|
}
|
|
192
258
|
}
|
|
193
259
|
|
|
194
|
-
private async
|
|
195
|
-
this.
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
260
|
+
private async leaveOnce(): Promise<void> {
|
|
261
|
+
const leave = { kind: 'leave', callId: this.start.callId, from: this.selfPeer };
|
|
262
|
+
let timer: ReturnType<typeof setTimeout> | undefined;
|
|
263
|
+
const capped = new Promise<never>((_resolve, reject) => {
|
|
264
|
+
timer = setTimeout(() => {
|
|
265
|
+
reject(new Error('hang-up signal timed out'));
|
|
266
|
+
}, this.deps.signalWaitMs);
|
|
267
|
+
});
|
|
268
|
+
try {
|
|
269
|
+
await Promise.race([this.signal(leave), capped]);
|
|
270
|
+
} finally {
|
|
271
|
+
clearTimeout(timer);
|
|
272
|
+
}
|
|
199
273
|
}
|
|
200
274
|
}
|
|
@@ -1,9 +1,10 @@
|
|
|
1
|
+
import { randomUUID } from 'node:crypto';
|
|
1
2
|
import { errMsg, log } from '@metro-labs/core/log';
|
|
2
3
|
import { isRecord } from '@metro-labs/core/is-record';
|
|
3
4
|
import type { TrainEvent } from '@metro-labs/core/trains/protocol';
|
|
4
|
-
import { lineReceives, mayApprove } from '../agents/map.js';
|
|
5
|
-
import { Call, trainCall } from './call.js';
|
|
6
|
-
import { readVoice, voiceReady } from './store.js';
|
|
5
|
+
import { agentIdForLine, allowlistForLine, lineReceives, mayApprove, senderPermitted } from '../agents/map.js';
|
|
6
|
+
import { Call, trainCall, type CallStart } from './call.js';
|
|
7
|
+
import { readVoice, voiceReady, type VoiceConfig } from './store.js';
|
|
7
8
|
|
|
8
9
|
const RING_MS = 40_000;
|
|
9
10
|
const CALL_TYPES = new Set(['callInvite', 'callSignal']);
|
|
@@ -12,21 +13,30 @@ const SWEEP_FIRST_MS = 5_000;
|
|
|
12
13
|
const SWEEP_EVERY_MS = 10_000;
|
|
13
14
|
const SWEEP_TRIES = 12;
|
|
14
15
|
|
|
15
|
-
|
|
16
|
+
type CallHandle = Pick<Call, 'start' | 'begin' | 'hangUp' | 'leave' | 'signalled'>;
|
|
17
|
+
type MakeCall = (start: CallStart, cfg: VoiceConfig, onOver: () => void) => CallHandle;
|
|
18
|
+
|
|
19
|
+
let current: CallHandle | null = null;
|
|
20
|
+
const invites = new Set<string>();
|
|
21
|
+
const INVITES_MAX = 256;
|
|
22
|
+
const makeCall: MakeCall = (start, cfg, onOver) => new Call(start, cfg, onOver);
|
|
16
23
|
|
|
17
24
|
const record = (value: unknown): Record<string, unknown> => (isRecord(value) ? value : {});
|
|
18
25
|
const str = (value: unknown): string => (typeof value === 'string' ? value : '');
|
|
19
26
|
|
|
20
27
|
const contentTypeOf = (env: TrainEvent): string => str(record(env.payload).contentType);
|
|
28
|
+
const sourceIdOf = (env: TrainEvent): string => str(env.message_id) || str(env.id) || randomUUID();
|
|
29
|
+
const callerNameOf = (env: TrainEvent): string => str(env.from_display_name) || str(env.from_name) || 'the caller';
|
|
21
30
|
|
|
22
31
|
export const isCallEvent = (env: TrainEvent): boolean => CALL_TYPES.has(contentTypeOf(env));
|
|
23
32
|
|
|
24
|
-
const sameCaller = (call:
|
|
33
|
+
const sameCaller = (call: CallHandle, line: string, from: string): boolean => call.start.line === line && call.start.from === from;
|
|
25
34
|
|
|
26
35
|
function refusal(line: string, from: string): string | null {
|
|
27
36
|
if (current !== null && !sameCaller(current, line, from)) return 'already in a call';
|
|
28
37
|
if (!voiceReady(readVoice())) return 'voice is off or has no API key on the Voice page';
|
|
29
|
-
if (!lineReceives(line) || !mayApprove(line, from)) return 'the caller is not someone who can approve for this channel';
|
|
38
|
+
if (!lineReceives(line) || !mayApprove(line, from) || !senderPermitted(allowlistForLine(line), from)) return 'the caller is not someone who can approve for this channel';
|
|
39
|
+
if (agentIdForLine(line) === undefined) return 'the channel does not belong to this agent';
|
|
30
40
|
return null;
|
|
31
41
|
}
|
|
32
42
|
|
|
@@ -38,15 +48,17 @@ function decline(line: string, callId: string): void {
|
|
|
38
48
|
|
|
39
49
|
function ringing(env: TrainEvent): boolean {
|
|
40
50
|
const sent = Date.parse(str(env.ts));
|
|
41
|
-
|
|
51
|
+
const age = Date.now() - sent;
|
|
52
|
+
return Number.isFinite(age) && age >= -5_000 && age < RING_MS;
|
|
42
53
|
}
|
|
43
54
|
|
|
44
|
-
function invite(env: TrainEvent, body: Record<string, unknown
|
|
55
|
+
function invite(env: TrainEvent, body: Record<string, unknown>, create: MakeCall): void {
|
|
45
56
|
const line = str(env.line);
|
|
46
57
|
const from = str(env.from);
|
|
47
58
|
const callId = str(body.callId);
|
|
48
59
|
const callerPeer = str(body.from);
|
|
49
|
-
|
|
60
|
+
const key = JSON.stringify([line, from, callId]);
|
|
61
|
+
if (callId === '' || callerPeer === '' || invites.has(key)) return;
|
|
50
62
|
if (!ringing(env)) {
|
|
51
63
|
log.info({ line }, 'voice: an invite came in after the caller stopped ringing, not answered');
|
|
52
64
|
return;
|
|
@@ -57,10 +69,15 @@ function invite(env: TrainEvent, body: Record<string, unknown>): void {
|
|
|
57
69
|
decline(line, callId);
|
|
58
70
|
return;
|
|
59
71
|
}
|
|
72
|
+
const agentId = agentIdForLine(line);
|
|
73
|
+
if (agentId === undefined) return;
|
|
74
|
+
invites.add(key);
|
|
75
|
+
if (invites.size > INVITES_MAX) invites.delete(invites.values().next().value ?? '');
|
|
60
76
|
current?.hangUp('the caller started a new call');
|
|
61
|
-
const callerName =
|
|
62
|
-
const
|
|
63
|
-
const
|
|
77
|
+
const callerName = callerNameOf(env);
|
|
78
|
+
const sourceId = sourceIdOf(env);
|
|
79
|
+
const start = { agentId, sourceId, line, lineName: str(env.line_name), direct: env.is_private === true, from, callerName, callId, callerPeer };
|
|
80
|
+
const call = create(start, readVoice(), () => {
|
|
64
81
|
if (current === call) current = null;
|
|
65
82
|
});
|
|
66
83
|
current = call;
|
|
@@ -69,10 +86,19 @@ function invite(env: TrainEvent, body: Record<string, unknown>): void {
|
|
|
69
86
|
});
|
|
70
87
|
}
|
|
71
88
|
|
|
72
|
-
|
|
89
|
+
function authenticated(env: TrainEvent): boolean {
|
|
90
|
+
if (env.sender_verified === false || (env.station !== undefined && env.station !== 'xmtp')) return false;
|
|
91
|
+
const line = str(env.line);
|
|
92
|
+
const match = /^metro:\/\/xmtp\/([^/]+)\/[^/]+$/.exec(line);
|
|
93
|
+
return match !== null && str(env.from).startsWith(`metro://xmtp/${String(match[1])}/user/`) && str(env.from).split('/').length === 6;
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
export function onCallEvent(env: TrainEvent, train = 'xmtp', create: MakeCall = makeCall): void {
|
|
97
|
+
if (train !== 'xmtp' || !authenticated(env)) return;
|
|
73
98
|
const body = record(record(env.payload).call);
|
|
74
|
-
if (contentTypeOf(env) === 'callInvite') invite(env, body);
|
|
75
|
-
else if (current !== null && env.line
|
|
99
|
+
if (contentTypeOf(env) === 'callInvite') invite(env, body, create);
|
|
100
|
+
else if (contentTypeOf(env) === 'callSignal' && current !== null && sameCaller(current, str(env.line), str(env.from)))
|
|
101
|
+
current.signalled(body);
|
|
76
102
|
}
|
|
77
103
|
|
|
78
104
|
export function leaveCallsForShutdown(): Promise<void> {
|
|
@@ -7,7 +7,7 @@ const GATHER_MS = 2_500;
|
|
|
7
7
|
const OPUS_PT = 111;
|
|
8
8
|
const MEDIA = JSON.stringify({ audio: true, video: false, screen: false });
|
|
9
9
|
|
|
10
|
-
interface PeerEvents {
|
|
10
|
+
export interface PeerEvents {
|
|
11
11
|
audio(opus: Buffer): void;
|
|
12
12
|
state(state: string): void;
|
|
13
13
|
}
|
|
@@ -34,6 +34,7 @@ export class Peer {
|
|
|
34
34
|
private readonly channel: RTCDataChannel;
|
|
35
35
|
private sender: RTCRtpSender | null = null;
|
|
36
36
|
private sequence = Math.floor(Math.random() * 0xffff);
|
|
37
|
+
private closed = false;
|
|
37
38
|
|
|
38
39
|
constructor(private readonly events: PeerEvents) {
|
|
39
40
|
this.channel = this.pc.createDataChannel('media', { negotiated: true, id: 0 });
|
|
@@ -43,11 +44,11 @@ export class Peer {
|
|
|
43
44
|
this.pc.onTrack.subscribe((track) => {
|
|
44
45
|
if (track.kind === 'audio')
|
|
45
46
|
track.onReceiveRtp.subscribe((rtp) => {
|
|
46
|
-
this.events.audio(rtp.payload);
|
|
47
|
+
if (!this.closed) this.events.audio(rtp.payload);
|
|
47
48
|
});
|
|
48
49
|
});
|
|
49
50
|
this.pc.connectionStateChange.subscribe((state) => {
|
|
50
|
-
this.events.state(state);
|
|
51
|
+
if (!this.closed) this.events.state(state);
|
|
51
52
|
});
|
|
52
53
|
}
|
|
53
54
|
|
|
@@ -66,17 +67,17 @@ export class Peer {
|
|
|
66
67
|
return sdp;
|
|
67
68
|
}
|
|
68
69
|
|
|
69
|
-
sendOpus(payload: Buffer, timestamp: number, marker: boolean): void {
|
|
70
|
+
sendOpus(payload: Buffer, timestamp: number, marker: boolean): Promise<void> {
|
|
70
71
|
const sender = this.sender;
|
|
71
|
-
if (sender === null) return;
|
|
72
|
+
if (this.closed || sender === null) return Promise.reject(new Error('call audio transport is closed'));
|
|
72
73
|
this.sequence = (this.sequence + 1) & 0xffff;
|
|
73
74
|
const header = new RtpHeader({ payloadType: OPUS_PT, sequenceNumber: this.sequence, timestamp, marker });
|
|
74
|
-
sender.sendRtp(new RtpPacket(header, payload))
|
|
75
|
-
log.debug({ err: errMsg(err) }, 'voice: an audio packet was not sent');
|
|
76
|
-
});
|
|
75
|
+
return sender.sendRtp(new RtpPacket(header, payload));
|
|
77
76
|
}
|
|
78
77
|
|
|
79
78
|
close(): void {
|
|
79
|
+
if (this.closed) return;
|
|
80
|
+
this.closed = true;
|
|
80
81
|
this.pc.close().catch((err: unknown) => {
|
|
81
82
|
log.debug({ err: errMsg(err) }, 'voice: closing the call connection failed');
|
|
82
83
|
});
|
|
@@ -9,7 +9,7 @@ const CHUNK_SAMPLES = STT_RATE / 10;
|
|
|
9
9
|
const HELD_SAMPLES = STT_RATE * 2;
|
|
10
10
|
export const VAD_SILENCE_SECS = 0.5;
|
|
11
11
|
|
|
12
|
-
interface Heard {
|
|
12
|
+
export interface Heard {
|
|
13
13
|
partial(text: string): void;
|
|
14
14
|
committed(text: string): void;
|
|
15
15
|
failed(reason: string): void;
|
|
@@ -54,6 +54,7 @@ export class Scribe {
|
|
|
54
54
|
private readonly ws: WebSocket;
|
|
55
55
|
private buffer: Int16Array[] = [];
|
|
56
56
|
private buffered = 0;
|
|
57
|
+
private closed = false;
|
|
57
58
|
|
|
58
59
|
constructor(apiKey: string, language: string | null, private readonly heard: Heard) {
|
|
59
60
|
this.ws = new WebSocket(sttUrl(language), { headers: { 'xi-api-key': apiKey } });
|
|
@@ -61,14 +62,20 @@ export class Scribe {
|
|
|
61
62
|
this.onMessage(wsText(data));
|
|
62
63
|
});
|
|
63
64
|
this.ws.on('error', (err) => {
|
|
65
|
+
if (this.closed) return;
|
|
64
66
|
log.warn({ err: errMsg(err) }, 'voice: speech to text connection error');
|
|
65
67
|
});
|
|
66
68
|
this.ws.on('close', (code) => {
|
|
67
|
-
if (
|
|
69
|
+
if (this.closed) return;
|
|
70
|
+
this.closed = true;
|
|
71
|
+
this.buffer = [];
|
|
72
|
+
this.buffered = 0;
|
|
73
|
+
this.heard.failed(`speech to text closed (${String(code)})`);
|
|
68
74
|
});
|
|
69
75
|
}
|
|
70
76
|
|
|
71
77
|
send(pcm16k: Int16Array): void {
|
|
78
|
+
if (this.closed) return;
|
|
72
79
|
this.buffer.push(pcm16k);
|
|
73
80
|
this.buffered += pcm16k.length;
|
|
74
81
|
while (this.buffered > HELD_SAMPLES) this.buffered -= this.buffer.shift()?.length ?? this.buffered;
|
|
@@ -87,10 +94,15 @@ export class Scribe {
|
|
|
87
94
|
}
|
|
88
95
|
|
|
89
96
|
close(): void {
|
|
90
|
-
this.
|
|
97
|
+
if (this.closed) return;
|
|
98
|
+
this.closed = true;
|
|
99
|
+
this.buffer = [];
|
|
100
|
+
this.buffered = 0;
|
|
101
|
+
this.ws.terminate();
|
|
91
102
|
}
|
|
92
103
|
|
|
93
104
|
private onMessage(raw: string): void {
|
|
105
|
+
if (this.closed) return;
|
|
94
106
|
const msg = parsed(raw);
|
|
95
107
|
const type = typeof msg.message_type === 'string' ? msg.message_type : '';
|
|
96
108
|
const text = typeof msg.text === 'string' ? msg.text.trim() : '';
|
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
import { randomUUID } from 'node:crypto';
|
|
2
|
+
import { downsample3, OpusIn, type Playback } from './audio.js';
|
|
3
|
+
import { Scribe, type Heard } from './scribe.js';
|
|
4
|
+
import type { SharedCallTransport, SharedSpeechAction } from './shared.js';
|
|
5
|
+
import { SpeechQueue } from './speech-queue.js';
|
|
6
|
+
import { languageOf, type VoiceConfig } from './store.js';
|
|
7
|
+
|
|
8
|
+
const BARGE_MIN_WORDS = 2;
|
|
9
|
+
const wordsIn = (text: string): number => text.split(/\s+/).filter((word) => /\p{L}/u.test(word)).length;
|
|
10
|
+
|
|
11
|
+
type MakeScribe = (heard: Heard) => Pick<Scribe, 'send' | 'close'>;
|
|
12
|
+
|
|
13
|
+
export class SharedTalk implements SharedCallTransport {
|
|
14
|
+
private readonly opus = new OpusIn();
|
|
15
|
+
private readonly scribe: Pick<Scribe, 'send' | 'close'>;
|
|
16
|
+
private readonly speech: SpeechQueue;
|
|
17
|
+
private closed = false;
|
|
18
|
+
|
|
19
|
+
constructor(
|
|
20
|
+
cfg: VoiceConfig,
|
|
21
|
+
send: Playback['send'],
|
|
22
|
+
private readonly ended: (reason: string) => void,
|
|
23
|
+
heard: (text: string, sourceId: string) => void,
|
|
24
|
+
makeScribe: MakeScribe = (events) => new Scribe(cfg.apiKey, languageOf(cfg), events),
|
|
25
|
+
speech = new SpeechQueue(cfg, send),
|
|
26
|
+
) {
|
|
27
|
+
this.speech = speech;
|
|
28
|
+
this.speech.whenFailed(() => {
|
|
29
|
+
this.stop('speech is unavailable');
|
|
30
|
+
});
|
|
31
|
+
this.scribe = makeScribe({
|
|
32
|
+
partial: (text) => {
|
|
33
|
+
this.barge(text);
|
|
34
|
+
},
|
|
35
|
+
committed: (text) => {
|
|
36
|
+
if (this.closed || text.trim() === '') return;
|
|
37
|
+
this.barge(text);
|
|
38
|
+
heard(text, randomUUID());
|
|
39
|
+
},
|
|
40
|
+
failed: (reason) => {
|
|
41
|
+
if (!this.closed) ended(reason);
|
|
42
|
+
},
|
|
43
|
+
});
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
enqueue(action: SharedSpeechAction): boolean {
|
|
47
|
+
return !this.closed && this.speech.enqueue(action);
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
cancel(): void {
|
|
51
|
+
this.speech.cancel();
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
terminate(): void {
|
|
55
|
+
this.stop('the shared conversation ended');
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
connect(): void {
|
|
59
|
+
if (!this.closed) this.speech.connect();
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
hear(packet: Buffer): void {
|
|
63
|
+
if (this.closed) return;
|
|
64
|
+
const pcm = this.opus.decode(packet);
|
|
65
|
+
if (pcm !== null) this.scribe.send(downsample3(pcm));
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
finish(): void {
|
|
69
|
+
if (this.closed) return;
|
|
70
|
+
this.closed = true;
|
|
71
|
+
this.speech.close();
|
|
72
|
+
this.scribe.close();
|
|
73
|
+
this.opus.close();
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
private stop(reason: string): void {
|
|
77
|
+
if (this.closed) return;
|
|
78
|
+
this.finish();
|
|
79
|
+
this.ended(reason);
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
private barge(text: string): void {
|
|
83
|
+
if (!this.closed && this.speech.busy && wordsIn(text) >= BARGE_MIN_WORDS) this.speech.cancel();
|
|
84
|
+
}
|
|
85
|
+
}
|