@stage-labs/metro 0.1.0-beta.264 → 0.1.0-beta.265
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli.js +2 -2
- package/package.json +1 -1
- package/runtime/marketplace/plugin/.claude-plugin/plugin.json +1 -1
- package/runtime/node_modules/@metro-labs/core/package.json +1 -0
- package/runtime/node_modules/@metro-labs/core/src/call.ts +81 -0
- package/runtime/node_modules/@metro-labs/core/src/runner-activity.ts +37 -1
- package/runtime/node_modules/@metro-labs/daemon/src/approvals/pending.ts +35 -2
- package/runtime/node_modules/@metro-labs/daemon/src/boot/boot.ts +1 -1
- package/runtime/node_modules/@metro-labs/daemon/src/channels/inbound.ts +14 -11
- package/runtime/node_modules/@metro-labs/daemon/src/channels/pending.ts +3 -0
- package/runtime/node_modules/@metro-labs/daemon/src/mcp/call-bridge.ts +45 -0
- package/runtime/node_modules/@metro-labs/daemon/src/mcp/call-permission.ts +25 -0
- package/runtime/node_modules/@metro-labs/daemon/src/mcp/call-tools.ts +2 -0
- package/runtime/node_modules/@metro-labs/daemon/src/mcp/event-store.ts +9 -2
- package/runtime/node_modules/@metro-labs/daemon/src/mcp/index.ts +2 -0
- package/runtime/node_modules/@metro-labs/daemon/src/mcp/permission-prompt.ts +2 -1
- package/runtime/node_modules/@metro-labs/daemon/src/mcp/permission-relay.ts +15 -4
- package/runtime/node_modules/@metro-labs/daemon/src/mcp/session.ts +20 -1
- package/runtime/node_modules/@metro-labs/daemon/src/mcp/speech-schema.ts +11 -0
- package/runtime/node_modules/@metro-labs/daemon/src/mcp/speech-send.ts +15 -0
- package/runtime/node_modules/@metro-labs/daemon/src/mcp/tool-schemas.ts +3 -1
- package/runtime/node_modules/@metro-labs/daemon/src/voice/audio.ts +50 -10
- package/runtime/node_modules/@metro-labs/daemon/src/voice/brain.ts +20 -9
- package/runtime/node_modules/@metro-labs/daemon/src/voice/call.ts +125 -51
- package/runtime/node_modules/@metro-labs/daemon/src/voice/calls.ts +41 -15
- package/runtime/node_modules/@metro-labs/daemon/src/voice/peer.ts +9 -8
- package/runtime/node_modules/@metro-labs/daemon/src/voice/scribe.ts +15 -3
- package/runtime/node_modules/@metro-labs/daemon/src/voice/shared-talk.ts +85 -0
- package/runtime/node_modules/@metro-labs/daemon/src/voice/shared.ts +199 -0
- package/runtime/node_modules/@metro-labs/daemon/src/voice/speech-queue.ts +173 -0
- package/runtime/node_modules/@metro-labs/daemon/src/voice/speech.ts +40 -15
- package/runtime/node_modules/@metro-labs/daemon/src/voice/talk.ts +15 -7
- package/runtime/runtime.json +1 -1
- package/runtime/sdk-runner/node_modules/@metro-labs/core/package.json +1 -0
- package/runtime/sdk-runner/node_modules/@metro-labs/core/src/call.ts +81 -0
- package/runtime/sdk-runner/node_modules/@metro-labs/core/src/runner-activity.ts +37 -1
- package/runtime/sdk-runner/runner.json +1 -1
- package/runtime/sdk-runner/src/activity.ts +14 -2
- package/runtime/sdk-runner/src/app.ts +5 -4
- package/runtime/sdk-runner/src/approvals.ts +14 -5
- package/runtime/sdk-runner/src/calls.ts +133 -0
- package/runtime/sdk-runner/src/inbox.ts +58 -26
- package/runtime/sdk-runner/src/input-queue.ts +70 -0
- package/runtime/sdk-runner/src/input-timing.ts +68 -0
- package/runtime/sdk-runner/src/link.ts +27 -4
- package/runtime/sdk-runner/src/main.ts +0 -4
- package/runtime/sdk-runner/src/rules.ts +19 -5
- package/runtime/sdk-runner/src/runner.ts +64 -88
- package/runtime/sdk-runner/src/session-store.ts +14 -1
- package/runtime/sdk-runner/src/session-watch.ts +27 -20
- package/runtime/sdk-runner/src/speech.ts +0 -79
|
@@ -12,6 +12,8 @@ import { BoundedEventStore } from './event-store.js';
|
|
|
12
12
|
import { registerPermissionRelay } from './permission-relay.js';
|
|
13
13
|
import { registerToolHandlers, toolSchemaSignature } from './tool-dispatch.js';
|
|
14
14
|
import { web, type RawGetSink } from './raw-get-stream.js';
|
|
15
|
+
import { sharedCalls } from '../voice/shared.js';
|
|
16
|
+
import { CALL_STATE } from '@metro-labs/core/call';
|
|
15
17
|
|
|
16
18
|
export const channelLog = (...a: unknown[]): void => {
|
|
17
19
|
console.error('[metro-mcp]', ...a);
|
|
@@ -76,9 +78,11 @@ export class McpSession {
|
|
|
76
78
|
private issuedSchema: string | undefined;
|
|
77
79
|
private announcing = false;
|
|
78
80
|
private readonly onClosed: (session: McpSession) => void;
|
|
81
|
+
private readonly kind: SessionKind;
|
|
79
82
|
|
|
80
83
|
private constructor(init: SessionInit) {
|
|
81
84
|
this.id = init.id;
|
|
85
|
+
this.kind = init.kind ?? 'chat';
|
|
82
86
|
this.scope = init.scope;
|
|
83
87
|
this.live = init.live;
|
|
84
88
|
this.issuedSchema = init.adopted ? undefined : toolSchemaSignature();
|
|
@@ -112,6 +116,7 @@ export class McpSession {
|
|
|
112
116
|
senderAllowed,
|
|
113
117
|
approves,
|
|
114
118
|
answerPermission,
|
|
119
|
+
callMeta: (event) => voice ? () => ({}) : sharedCalls.bindChat(event, this.scope),
|
|
115
120
|
});
|
|
116
121
|
if (!voice)
|
|
117
122
|
registerPermissionRelay({
|
|
@@ -167,7 +172,20 @@ export class McpSession {
|
|
|
167
172
|
|
|
168
173
|
bindSink(sink: RawGetSink | undefined): void {
|
|
169
174
|
this.sink = sink;
|
|
170
|
-
if (sink !== undefined)
|
|
175
|
+
if (sink !== undefined) {
|
|
176
|
+
this.announceCallState();
|
|
177
|
+
this.announceToolSchema();
|
|
178
|
+
}
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
private announceCallState(): void {
|
|
182
|
+
if (this.kind !== 'chat' || !this.streamAttached || this.server.getClientVersion()?.name !== 'metro-sdk-runner') return;
|
|
183
|
+
const active = sharedCalls.snapshot();
|
|
184
|
+
const route = active !== null && this.scope.has(active.agentId) && eventInScope(this.scope, active.line) ? active : null;
|
|
185
|
+
this.server.notification({ method: CALL_STATE, params: { route, ...(route === null ? {} : { meta: { line: route.line } }) } }).catch((err: unknown) => {
|
|
186
|
+
channelLog('session: call state notification failed', errMsg(err));
|
|
187
|
+
if (route !== null) sharedCalls.end(route);
|
|
188
|
+
});
|
|
171
189
|
}
|
|
172
190
|
|
|
173
191
|
private get schemaNoticeDue(): boolean {
|
|
@@ -206,6 +224,7 @@ export class McpSession {
|
|
|
206
224
|
async close(): Promise<void> {
|
|
207
225
|
if (this.closed) return;
|
|
208
226
|
this.closed = true;
|
|
227
|
+
if (this.kind === 'chat') sharedCalls.revoke();
|
|
209
228
|
this.onClosed(this);
|
|
210
229
|
forgetPromptsOf(this.server);
|
|
211
230
|
this.dropStream();
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
export const speechSchema = {
|
|
2
|
+
type: 'object',
|
|
3
|
+
description: 'Speak text only in the exact active Stage call, without posting it in chat. Copy callId, generation and sourceId from its authenticated call input or call_id, call_generation and call_source_id from the requesting channel message. Use only for a call reply or an explicit owner request to speak in that call. Normal chat and worker results stay silent. One action per source; retries do not repeat speech. Accepted or queued is not heard. Completed means audio transport drained, not proof the caller heard it.',
|
|
4
|
+
properties: {
|
|
5
|
+
callId: { type: 'string' },
|
|
6
|
+
generation: { type: 'string' },
|
|
7
|
+
sourceId: { type: 'string' },
|
|
8
|
+
},
|
|
9
|
+
required: ['callId', 'generation', 'sourceId'],
|
|
10
|
+
additionalProperties: false,
|
|
11
|
+
} as const;
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
import { speechTarget } from '@metro-labs/core/call';
|
|
2
|
+
import type { ToolResult } from '@metro-labs/core/stations/types';
|
|
3
|
+
import { sharedCalls } from '../voice/shared.js';
|
|
4
|
+
import { allowedAgents, currentIdentity } from './request-identity.js';
|
|
5
|
+
import { errResult, okJson } from './ctx.js';
|
|
6
|
+
|
|
7
|
+
export function sendSpeech(line: string, args: Record<string, unknown>): ToolResult {
|
|
8
|
+
const target = speechTarget(args.speech);
|
|
9
|
+
if (target === null) return errResult('Call speech needs callId, generation and sourceId from its authenticated source.');
|
|
10
|
+
if (['reply_to', 'subject', 'attachments', 'frame', 'wallet', 'account'].some((key) => args[key] !== undefined))
|
|
11
|
+
return errResult('Call speech cannot include chat content, media or an account override. Nothing was sent.');
|
|
12
|
+
if (typeof args.text !== 'string') return errResult('Call speech needs text.');
|
|
13
|
+
const receipt = sharedCalls.speak(line, target, args.text, allowedAgents(currentIdentity()));
|
|
14
|
+
return okJson({ speech: receipt, delivery: 'Audio transport status only. No proof the caller heard it. No chat message was posted.' });
|
|
15
|
+
}
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import type { ToolDef } from './tool-def.js';
|
|
2
2
|
import { attachmentItem, CREATE_UPLOAD_TOOL } from './attachment-schema.js';
|
|
3
|
+
import { speechSchema } from './speech-schema.js';
|
|
3
4
|
|
|
4
5
|
const lineProp = {
|
|
5
6
|
type: 'string',
|
|
@@ -34,7 +35,7 @@ export const COMMON_TOOLS: ToolDef[] = [
|
|
|
34
35
|
group: 'write',
|
|
35
36
|
description:
|
|
36
37
|
'Send a message (and/or media) to a Metro conversation. Args: line, text?, reply_to?, ' +
|
|
37
|
-
'subject?, attachments?, frame?, wallet?. The station is derived from the line. Outlook and Gmail: send to ' +
|
|
38
|
+
'subject?, attachments?, frame?, wallet?, speech?. `speech` speaks text only in an authorized active call, never posts it in chat. The station is derived from the line. Outlook and Gmail: send to ' +
|
|
38
39
|
'metro://<outlook|gmail>/<account>/<email address> to start a new email (subject defaults to the ' +
|
|
39
40
|
'first line of text); the result names the new thread line. Each attachment names EXACTLY ONE ' +
|
|
40
41
|
'source, and the choice matters: `upload` (an `upload_id` from `create_upload`, the ' +
|
|
@@ -54,6 +55,7 @@ export const COMMON_TOOLS: ToolDef[] = [
|
|
|
54
55
|
type: 'string',
|
|
55
56
|
description: 'The message text (optional if sending only media).',
|
|
56
57
|
},
|
|
58
|
+
speech: speechSchema,
|
|
57
59
|
reply_to: {
|
|
58
60
|
type: 'string',
|
|
59
61
|
description: 'Optional message_id to quote/reply to.',
|
|
@@ -5,6 +5,7 @@ const RATE = 48_000;
|
|
|
5
5
|
const FRAME = 960;
|
|
6
6
|
const FRAME_MS = 20;
|
|
7
7
|
const MAX_LATE_FRAMES = 5;
|
|
8
|
+
const MAX_QUEUED_FRAMES = 3_000;
|
|
8
9
|
|
|
9
10
|
interface Frame {
|
|
10
11
|
pcm: Int16Array;
|
|
@@ -55,7 +56,7 @@ export class OpusIn {
|
|
|
55
56
|
}
|
|
56
57
|
|
|
57
58
|
export interface Playback {
|
|
58
|
-
send(opus: Buffer, timestamp: number, marker: boolean): void
|
|
59
|
+
send(opus: Buffer, timestamp: number, marker: boolean): void | Promise<void>;
|
|
59
60
|
played(chars: number): void;
|
|
60
61
|
}
|
|
61
62
|
|
|
@@ -70,11 +71,15 @@ export class Player {
|
|
|
70
71
|
private speaking = false;
|
|
71
72
|
private onIdle: (() => void) | null = null;
|
|
72
73
|
private onFirst: (() => void) | null = null;
|
|
74
|
+
private onFailed: ((reason: string) => void) | null = null;
|
|
75
|
+
private generation = 0;
|
|
76
|
+
private inFlight = 0;
|
|
77
|
+
private closed = false;
|
|
73
78
|
|
|
74
79
|
constructor(private readonly out: Playback) {}
|
|
75
80
|
|
|
76
81
|
get busy(): boolean {
|
|
77
|
-
return this.queue.length > 0 || this.pending.length > 0;
|
|
82
|
+
return this.queue.length > 0 || this.pending.length > 0 || this.inFlight > 0;
|
|
78
83
|
}
|
|
79
84
|
|
|
80
85
|
whenFirstPlayed(fn: () => void): void {
|
|
@@ -85,7 +90,16 @@ export class Player {
|
|
|
85
90
|
this.onIdle = fn;
|
|
86
91
|
}
|
|
87
92
|
|
|
93
|
+
whenFailed(fn: (reason: string) => void): void {
|
|
94
|
+
this.onFailed = fn;
|
|
95
|
+
}
|
|
96
|
+
|
|
88
97
|
push(pcm: Int16Array, chars: number): void {
|
|
98
|
+
if (this.closed) return;
|
|
99
|
+
if (this.queue.length + this.inFlight + Math.ceil((this.pending.length + pcm.length) / FRAME) > MAX_QUEUED_FRAMES) {
|
|
100
|
+
this.failed('audio queue is full');
|
|
101
|
+
return;
|
|
102
|
+
}
|
|
89
103
|
const joined = new Int16Array(this.pending.length + pcm.length);
|
|
90
104
|
joined.set(this.pending);
|
|
91
105
|
joined.set(pcm, this.pending.length);
|
|
@@ -101,7 +115,7 @@ export class Player {
|
|
|
101
115
|
}
|
|
102
116
|
|
|
103
117
|
finish(): void {
|
|
104
|
-
if (this.pending.length === 0) return;
|
|
118
|
+
if (this.closed || this.pending.length === 0) return;
|
|
105
119
|
const pcm = new Int16Array(FRAME);
|
|
106
120
|
pcm.set(this.pending);
|
|
107
121
|
this.queue.push({ pcm, chars: this.pendingChars });
|
|
@@ -110,28 +124,36 @@ export class Player {
|
|
|
110
124
|
}
|
|
111
125
|
|
|
112
126
|
clear(): void {
|
|
127
|
+
this.generation += 1;
|
|
128
|
+
this.inFlight = 0;
|
|
129
|
+
this.speaking = false;
|
|
113
130
|
this.queue.length = 0;
|
|
114
131
|
this.pending = new Int16Array(0);
|
|
115
132
|
this.pendingChars = 0;
|
|
116
133
|
}
|
|
117
134
|
|
|
118
135
|
start(): void {
|
|
119
|
-
if (this.timer !== null) return;
|
|
136
|
+
if (this.closed || this.timer !== null) return;
|
|
120
137
|
this.started = performance.now();
|
|
121
138
|
this.sent = 0;
|
|
122
139
|
this.tick();
|
|
123
140
|
}
|
|
124
141
|
|
|
125
142
|
stop(): void {
|
|
143
|
+
if (this.closed) return;
|
|
144
|
+
this.closed = true;
|
|
126
145
|
if (this.timer !== null) clearTimeout(this.timer);
|
|
127
146
|
this.timer = null;
|
|
147
|
+
this.clear();
|
|
128
148
|
this.encoder.delete();
|
|
129
149
|
}
|
|
130
150
|
|
|
131
151
|
private tick(): void {
|
|
152
|
+
if (this.closed) return;
|
|
132
153
|
const due = Math.floor((performance.now() - this.started) / FRAME_MS) + 1;
|
|
133
154
|
if (due - this.sent > MAX_LATE_FRAMES) this.sent = due - 1;
|
|
134
|
-
while (this.sent < due) this.sendOne();
|
|
155
|
+
while (!this.closed && this.sent < due) this.sendOne();
|
|
156
|
+
if (this.closed) return;
|
|
135
157
|
const next = this.started + this.sent * FRAME_MS;
|
|
136
158
|
this.timer = setTimeout(() => {
|
|
137
159
|
this.tick();
|
|
@@ -143,11 +165,29 @@ export class Player {
|
|
|
143
165
|
const marker = frame !== undefined && !this.speaking;
|
|
144
166
|
this.speaking = frame !== undefined;
|
|
145
167
|
const pcm = frame?.pcm ?? new Int16Array(FRAME);
|
|
146
|
-
|
|
168
|
+
const timestamp = (this.sent * FRAME) >>> 0;
|
|
147
169
|
this.sent += 1;
|
|
148
|
-
|
|
149
|
-
if (
|
|
150
|
-
|
|
151
|
-
|
|
170
|
+
const generation = this.generation;
|
|
171
|
+
if (frame !== undefined) this.inFlight += 1;
|
|
172
|
+
try {
|
|
173
|
+
const sent = this.out.send(this.encoder.encode(bytesOf(pcm), FRAME), timestamp, marker);
|
|
174
|
+
Promise.resolve(sent).then(() => {
|
|
175
|
+
if (this.closed || generation !== this.generation || frame === undefined) return;
|
|
176
|
+
this.inFlight -= 1;
|
|
177
|
+
if (marker) this.onFirst?.();
|
|
178
|
+
if (frame.chars > 0) this.out.played(frame.chars);
|
|
179
|
+
if (!this.busy) this.onIdle?.();
|
|
180
|
+
}).catch((err: unknown) => {
|
|
181
|
+
if (!this.closed && generation === this.generation) this.failed(errMsg(err));
|
|
182
|
+
});
|
|
183
|
+
} catch (err) {
|
|
184
|
+
if (!this.closed && generation === this.generation) this.failed(errMsg(err));
|
|
185
|
+
}
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
private failed(reason: string): void {
|
|
189
|
+
this.clear();
|
|
190
|
+
log.warn({ reason }, 'voice: audio playback failed');
|
|
191
|
+
this.onFailed?.(reason);
|
|
152
192
|
}
|
|
153
193
|
}
|
|
@@ -50,15 +50,21 @@ function streamEvent(msg: Record<string, unknown>, mind: Thinking): void {
|
|
|
50
50
|
if (event.type === 'content_block_start' && block.type === 'tool_use') mind.tool(typeof block.name === 'string' ? block.name : 'tool');
|
|
51
51
|
}
|
|
52
52
|
|
|
53
|
+
function startBrain(model: string): ChildProcessWithoutNullStreams {
|
|
54
|
+
const [command = 'metro', ...args] = metroCli(['voice', '--model', model]);
|
|
55
|
+
const [file, argv] = asAgent(command, args);
|
|
56
|
+
return spawn(file, argv, { stdio: ['pipe', 'pipe', 'pipe'], cwd: '/' });
|
|
57
|
+
}
|
|
58
|
+
|
|
53
59
|
export class Brain {
|
|
54
60
|
private readonly child: ChildProcessWithoutNullStreams;
|
|
55
61
|
private out = '';
|
|
56
62
|
private gone = false;
|
|
63
|
+
private closing = false;
|
|
64
|
+
private killTimer: ReturnType<typeof setTimeout> | undefined;
|
|
57
65
|
|
|
58
|
-
constructor(model: string, private readonly mind: Thinking) {
|
|
59
|
-
|
|
60
|
-
const [file, argv] = asAgent(command, args);
|
|
61
|
-
this.child = spawn(file, argv, { stdio: ['pipe', 'pipe', 'pipe'], cwd: '/' });
|
|
66
|
+
constructor(model: string, private readonly mind: Thinking, launch = startBrain) {
|
|
67
|
+
this.child = launch(model);
|
|
62
68
|
this.child.stdout.setEncoding('utf8');
|
|
63
69
|
this.child.stdout.on('data', (chunk: string) => {
|
|
64
70
|
this.out = drainLines('voice-brain', this.out + chunk, (line) => {
|
|
@@ -91,18 +97,22 @@ export class Brain {
|
|
|
91
97
|
}
|
|
92
98
|
|
|
93
99
|
close(): void {
|
|
94
|
-
if (this.gone) return;
|
|
100
|
+
if (this.gone || this.closing) return;
|
|
101
|
+
this.closing = true;
|
|
95
102
|
this.child.stdin.end();
|
|
96
|
-
|
|
103
|
+
this.child.kill('SIGTERM');
|
|
104
|
+
this.killTimer = setTimeout(() => {
|
|
97
105
|
if (!this.gone) this.child.kill('SIGKILL');
|
|
98
|
-
}, CLOSE_GRACE_MS)
|
|
106
|
+
}, CLOSE_GRACE_MS);
|
|
107
|
+
this.killTimer.unref();
|
|
99
108
|
}
|
|
100
109
|
|
|
101
110
|
private write(message: Record<string, unknown>): void {
|
|
102
|
-
if (!this.gone && this.child.stdin.writable) this.child.stdin.write(`${JSON.stringify(message)}\n`);
|
|
111
|
+
if (!this.gone && !this.closing && this.child.stdin.writable) this.child.stdin.write(`${JSON.stringify(message)}\n`);
|
|
103
112
|
}
|
|
104
113
|
|
|
105
114
|
private read(line: string): void {
|
|
115
|
+
if (this.closing || this.gone) return;
|
|
106
116
|
let parsed: unknown;
|
|
107
117
|
try {
|
|
108
118
|
parsed = JSON.parse(line);
|
|
@@ -130,6 +140,7 @@ export class Brain {
|
|
|
130
140
|
private exit(reason: string): void {
|
|
131
141
|
if (this.gone) return;
|
|
132
142
|
this.gone = true;
|
|
133
|
-
this.
|
|
143
|
+
clearTimeout(this.killTimer);
|
|
144
|
+
if (!this.closing) this.mind.exited(reason);
|
|
134
145
|
}
|
|
135
146
|
}
|
|
@@ -1,9 +1,13 @@
|
|
|
1
1
|
import { randomUUID } from 'node:crypto';
|
|
2
|
+
import type { CallRoute } from '@metro-labs/core/call';
|
|
2
3
|
import { errMsg, log } from '@metro-labs/core/log';
|
|
3
4
|
import { isRecord } from '@metro-labs/core/is-record';
|
|
4
5
|
import { forwardTrainCall } from '../stations/train-call.js';
|
|
5
6
|
import { CallApprovals } from './approvals.js';
|
|
6
|
-
import {
|
|
7
|
+
import type { Playback } from './audio.js';
|
|
8
|
+
import { Peer, type PeerEvents } from './peer.js';
|
|
9
|
+
import { SharedTalk } from './shared-talk.js';
|
|
10
|
+
import { endSharedCall, hearSharedCall, openSharedCall, sharedCallsSelected, type SharedCallTransport } from './shared.js';
|
|
7
11
|
import { Talk } from './talk.js';
|
|
8
12
|
import type { VoiceConfig } from './store.js';
|
|
9
13
|
|
|
@@ -12,8 +16,11 @@ const LONGEST_CALL_MS = 60 * 60_000;
|
|
|
12
16
|
const CONTEXT_MESSAGES = 12;
|
|
13
17
|
const CONTEXT_CHARS = 400;
|
|
14
18
|
const LEAVE_RETRY_MS = 3_000;
|
|
19
|
+
const SIGNAL_WAIT_MS = 2_000;
|
|
15
20
|
|
|
16
|
-
interface CallStart {
|
|
21
|
+
export interface CallStart {
|
|
22
|
+
agentId: string;
|
|
23
|
+
sourceId: string;
|
|
17
24
|
line: string;
|
|
18
25
|
lineName: string;
|
|
19
26
|
direct: boolean;
|
|
@@ -23,6 +30,23 @@ interface CallStart {
|
|
|
23
30
|
callerPeer: string;
|
|
24
31
|
}
|
|
25
32
|
|
|
33
|
+
type CallAudio = Pick<Talk, 'connect' | 'hear' | 'finish'>;
|
|
34
|
+
type CallPeer = Pick<Peer, 'answer' | 'sendOpus' | 'close'>;
|
|
35
|
+
type ContextStart = Omit<CallStart, 'agentId' | 'sourceId'>;
|
|
36
|
+
|
|
37
|
+
export interface CallDependencies {
|
|
38
|
+
train: typeof trainCall;
|
|
39
|
+
peer(events: PeerEvents): CallPeer;
|
|
40
|
+
cli(cfg: VoiceConfig, out: Playback['send'], ended: (reason: string) => void, approvals: CallApprovals): CallAudio & Pick<Talk, 'prime'>;
|
|
41
|
+
shared(cfg: VoiceConfig, out: Playback['send'], ended: (reason: string) => void, heard: (text: string, sourceId: string) => void): CallAudio & SharedCallTransport;
|
|
42
|
+
selected: typeof sharedCallsSelected;
|
|
43
|
+
open: typeof openSharedCall;
|
|
44
|
+
heard: typeof hearSharedCall;
|
|
45
|
+
ended: typeof endSharedCall;
|
|
46
|
+
signalWaitMs: number;
|
|
47
|
+
leaveRetryMs: number;
|
|
48
|
+
}
|
|
49
|
+
|
|
26
50
|
const str = (value: unknown): string => (typeof value === 'string' ? value : '');
|
|
27
51
|
|
|
28
52
|
export async function trainCall(action: string, args: Record<string, unknown>): Promise<unknown> {
|
|
@@ -31,20 +55,33 @@ export async function trainCall(action: string, args: Record<string, unknown>):
|
|
|
31
55
|
return answer.result;
|
|
32
56
|
}
|
|
33
57
|
|
|
34
|
-
|
|
58
|
+
const dependencies: CallDependencies = {
|
|
59
|
+
train: trainCall,
|
|
60
|
+
peer: (events) => new Peer(events),
|
|
61
|
+
cli: (cfg, out, ended, approvals) => new Talk(cfg, out, ended, approvals),
|
|
62
|
+
shared: (cfg, out, ended, heard) => new SharedTalk(cfg, out, ended, heard),
|
|
63
|
+
selected: sharedCallsSelected,
|
|
64
|
+
open: openSharedCall,
|
|
65
|
+
heard: hearSharedCall,
|
|
66
|
+
ended: endSharedCall,
|
|
67
|
+
signalWaitMs: SIGNAL_WAIT_MS,
|
|
68
|
+
leaveRetryMs: LEAVE_RETRY_MS,
|
|
69
|
+
};
|
|
70
|
+
|
|
71
|
+
function speaker(row: Record<string, unknown>, start: ContextStart): string {
|
|
35
72
|
if (row.self === true) return 'you';
|
|
36
73
|
return row.from === start.from ? start.callerName : `another member (${str(row.from)})`;
|
|
37
74
|
}
|
|
38
75
|
|
|
39
|
-
function contextLine(row: unknown, start:
|
|
76
|
+
function contextLine(row: unknown, start: ContextStart): string | null {
|
|
40
77
|
if (!isRecord(row) || typeof row.text !== 'string' || row.text.startsWith('[call')) return null;
|
|
41
78
|
const who = speaker(row, start);
|
|
42
79
|
return `${str(row.ts).slice(11, 16)} ${who}: ${row.text.replace(/\s+/g, ' ').slice(0, CONTEXT_CHARS)}`;
|
|
43
80
|
}
|
|
44
81
|
|
|
45
|
-
async function recentChat(start: CallStart): Promise<string> {
|
|
82
|
+
async function recentChat(start: CallStart, train: typeof trainCall): Promise<string> {
|
|
46
83
|
try {
|
|
47
|
-
const result = await
|
|
84
|
+
const result = await train('read', { line: start.line, limit: CONTEXT_MESSAGES });
|
|
48
85
|
const rows = isRecord(result) && Array.isArray(result.messages) ? result.messages : [];
|
|
49
86
|
return rows.map((row) => contextLine(row, start)).filter((l): l is string => l !== null).join('\n');
|
|
50
87
|
} catch (err) {
|
|
@@ -53,12 +90,12 @@ async function recentChat(start: CallStart): Promise<string> {
|
|
|
53
90
|
}
|
|
54
91
|
}
|
|
55
92
|
|
|
56
|
-
function where(start:
|
|
93
|
+
function where(start: ContextStart): string {
|
|
57
94
|
if (start.lineName !== '') return `the group chat "${start.lineName}"`;
|
|
58
95
|
return start.direct ? `your direct chat with ${start.callerName}` : 'a group chat with no name';
|
|
59
96
|
}
|
|
60
97
|
|
|
61
|
-
export function opening(start:
|
|
98
|
+
export function opening(start: ContextStart, chat: string): string {
|
|
62
99
|
const recent = chat === '' ? 'No earlier messages were found there.' : `Recent messages there before the call, oldest first (times in UTC):\n${chat}`;
|
|
63
100
|
return [
|
|
64
101
|
`[A voice call with ${start.callerName} is starting on Stage, in ${where(start)}.`,
|
|
@@ -70,49 +107,77 @@ export function opening(start: CallStart, chat: string): string {
|
|
|
70
107
|
}
|
|
71
108
|
|
|
72
109
|
export class Call {
|
|
110
|
+
readonly route: CallRoute;
|
|
73
111
|
private readonly selfPeer = randomUUID();
|
|
74
112
|
private readonly startedAt = Date.now();
|
|
75
|
-
private readonly talk:
|
|
76
|
-
private
|
|
113
|
+
private readonly talk: CallAudio;
|
|
114
|
+
private readonly shared: SharedCallTransport | null;
|
|
115
|
+
private readonly prime: ((context: string) => void) | null;
|
|
116
|
+
private readonly deps: CallDependencies;
|
|
117
|
+
private peer: CallPeer | null = null;
|
|
77
118
|
private timer: ReturnType<typeof setTimeout> | null = null;
|
|
78
|
-
private longest: ReturnType<typeof setTimeout>;
|
|
119
|
+
private readonly longest: ReturnType<typeof setTimeout>;
|
|
79
120
|
private over = false;
|
|
121
|
+
private begun = false;
|
|
80
122
|
private left: Promise<void> = Promise.resolve();
|
|
81
123
|
|
|
82
124
|
constructor(
|
|
83
125
|
readonly start: CallStart,
|
|
84
126
|
cfg: VoiceConfig,
|
|
85
127
|
private readonly onOver: () => void,
|
|
128
|
+
deps: Partial<CallDependencies> = {},
|
|
86
129
|
) {
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
130
|
+
this.deps = { ...dependencies, ...deps };
|
|
131
|
+
this.route = Object.freeze({ agentId: start.agentId, line: start.line, from: start.from, callId: start.callId, generation: randomUUID() });
|
|
132
|
+
const out: Playback['send'] = (opus, timestamp, marker) => {
|
|
133
|
+
if (this.over || this.peer === null) return Promise.reject(new Error('call audio is not connected'));
|
|
134
|
+
return this.peer.sendOpus(opus, timestamp, marker);
|
|
135
|
+
};
|
|
136
|
+
const ended = (reason: string): void => {
|
|
137
|
+
this.hangUp(reason);
|
|
138
|
+
};
|
|
139
|
+
if (this.deps.selected()) {
|
|
140
|
+
const shared = this.deps.shared(cfg, out, ended, (text, sourceId) => {
|
|
141
|
+
if (!this.over) this.deps.heard(this.route, text, sourceId);
|
|
142
|
+
});
|
|
143
|
+
this.talk = shared;
|
|
144
|
+
this.shared = shared;
|
|
145
|
+
this.prime = null;
|
|
146
|
+
} else {
|
|
147
|
+
const approvals = new CallApprovals(start.line, async (text) => {
|
|
148
|
+
await this.deps.train('send', { line: start.line, text });
|
|
149
|
+
});
|
|
150
|
+
const cli = this.deps.cli(cfg, out, ended, approvals);
|
|
151
|
+
this.talk = cli;
|
|
152
|
+
this.shared = null;
|
|
153
|
+
this.prime = (context) => {
|
|
154
|
+
cli.prime(context);
|
|
155
|
+
};
|
|
156
|
+
}
|
|
100
157
|
this.longest = setTimeout(() => {
|
|
101
158
|
this.hangUp('the call reached one hour');
|
|
102
159
|
}, LONGEST_CALL_MS);
|
|
103
160
|
}
|
|
104
161
|
|
|
105
162
|
async begin(): Promise<void> {
|
|
163
|
+
if (this.over || this.begun) return;
|
|
164
|
+
this.begun = true;
|
|
106
165
|
log.info({ line: this.start.line, callId: this.start.callId }, 'voice: answering a call');
|
|
107
|
-
|
|
108
|
-
.
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
166
|
+
if (this.shared !== null) {
|
|
167
|
+
if (!this.deps.open(this.route, this.start.sourceId, this.shared)) throw new Error('the shared conversation is unavailable');
|
|
168
|
+
} else {
|
|
169
|
+
recentChat(this.start, this.deps.train).then((chat) => {
|
|
170
|
+
if (!this.over) this.prime?.(opening(this.start, chat));
|
|
171
|
+
}).catch((err: unknown) => {
|
|
112
172
|
log.warn({ err: errMsg(err) }, 'voice: could not start the agent on the call');
|
|
113
173
|
});
|
|
174
|
+
}
|
|
114
175
|
await this.signal({ kind: 'join', callId: this.start.callId, from: this.selfPeer });
|
|
115
|
-
this.
|
|
176
|
+
if (this.over) {
|
|
177
|
+
await this.sendLeave();
|
|
178
|
+
return;
|
|
179
|
+
}
|
|
180
|
+
if (this.peer === null) this.timer = setTimeout(() => {
|
|
116
181
|
this.hangUp('no offer arrived');
|
|
117
182
|
}, OFFER_WAIT_MS);
|
|
118
183
|
}
|
|
@@ -138,9 +203,9 @@ export class Call {
|
|
|
138
203
|
|
|
139
204
|
private async answer(to: string, offer: string): Promise<void> {
|
|
140
205
|
if (this.timer !== null) clearTimeout(this.timer);
|
|
141
|
-
this.peer =
|
|
206
|
+
this.peer = this.deps.peer({
|
|
142
207
|
audio: (opus) => {
|
|
143
|
-
this.talk.hear(opus);
|
|
208
|
+
if (!this.over) this.talk.hear(opus);
|
|
144
209
|
},
|
|
145
210
|
state: (state) => {
|
|
146
211
|
this.stateChanged(state);
|
|
@@ -152,13 +217,14 @@ export class Call {
|
|
|
152
217
|
}
|
|
153
218
|
|
|
154
219
|
private stateChanged(state: string): void {
|
|
220
|
+
if (this.over) return;
|
|
155
221
|
log.info({ state, afterMs: Date.now() - this.startedAt }, 'voice: call connection');
|
|
156
222
|
if (state === 'connected') this.talk.connect();
|
|
157
223
|
else if (state === 'failed' || state === 'closed') this.hangUp(`the connection ${state}`);
|
|
158
224
|
}
|
|
159
225
|
|
|
160
226
|
private signal(signal: Record<string, unknown>): Promise<unknown> {
|
|
161
|
-
return
|
|
227
|
+
return this.deps.train('callSignal', { line: this.start.line, signal });
|
|
162
228
|
}
|
|
163
229
|
|
|
164
230
|
private end(reason: string): void {
|
|
@@ -167,34 +233,42 @@ export class Call {
|
|
|
167
233
|
if (this.timer !== null) clearTimeout(this.timer);
|
|
168
234
|
clearTimeout(this.longest);
|
|
169
235
|
log.info({ reason, seconds: Math.round((Date.now() - this.startedAt) / 1000) }, 'voice: call over');
|
|
170
|
-
this.
|
|
171
|
-
this.
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
.
|
|
177
|
-
|
|
178
|
-
});
|
|
236
|
+
this.talk.finish();
|
|
237
|
+
this.peer?.close();
|
|
238
|
+
if (this.shared !== null) this.deps.ended(this.route);
|
|
239
|
+
this.onOver();
|
|
240
|
+
this.left = this.sendLeave().then(async () => {
|
|
241
|
+
if (this.shared !== null || reason.startsWith('the caller') || reason === 'no offer arrived') return;
|
|
242
|
+
await this.deps.train('send', { line: this.start.line, text: `The voice call ended: ${reason}.` });
|
|
243
|
+
}).catch((err: unknown) => {
|
|
244
|
+
log.warn({ err: errMsg(err) }, 'voice: wrapping up the call failed');
|
|
245
|
+
});
|
|
179
246
|
}
|
|
180
247
|
|
|
181
248
|
private async sendLeave(): Promise<void> {
|
|
182
|
-
const leave = { kind: 'leave', callId: this.start.callId, from: this.selfPeer };
|
|
183
249
|
try {
|
|
184
|
-
await this.
|
|
250
|
+
await this.leaveOnce();
|
|
185
251
|
} catch (err) {
|
|
186
252
|
log.warn({ err: errMsg(err) }, 'voice: could not send the hang-up, trying once more');
|
|
187
|
-
await new Promise((r) => setTimeout(r,
|
|
188
|
-
await this.
|
|
253
|
+
await new Promise((r) => setTimeout(r, this.deps.leaveRetryMs));
|
|
254
|
+
await this.leaveOnce().catch((again: unknown) => {
|
|
189
255
|
log.warn({ err: errMsg(again) }, 'voice: could not send the hang-up; the next start sends it');
|
|
190
256
|
});
|
|
191
257
|
}
|
|
192
258
|
}
|
|
193
259
|
|
|
194
|
-
private async
|
|
195
|
-
this.
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
260
|
+
private async leaveOnce(): Promise<void> {
|
|
261
|
+
const leave = { kind: 'leave', callId: this.start.callId, from: this.selfPeer };
|
|
262
|
+
let timer: ReturnType<typeof setTimeout> | undefined;
|
|
263
|
+
const capped = new Promise<never>((_resolve, reject) => {
|
|
264
|
+
timer = setTimeout(() => {
|
|
265
|
+
reject(new Error('hang-up signal timed out'));
|
|
266
|
+
}, this.deps.signalWaitMs);
|
|
267
|
+
});
|
|
268
|
+
try {
|
|
269
|
+
await Promise.race([this.signal(leave), capped]);
|
|
270
|
+
} finally {
|
|
271
|
+
clearTimeout(timer);
|
|
272
|
+
}
|
|
199
273
|
}
|
|
200
274
|
}
|