@stage-labs/metro 0.1.0-beta.237 → 0.1.0-beta.239
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/runtime/marketplace/plugin/.claude-plugin/plugin.json +1 -1
- package/runtime/node_modules/@metro-labs/daemon/src/boot/boot.ts +4 -1
- package/runtime/node_modules/@metro-labs/daemon/src/voice/call.ts +32 -12
- package/runtime/node_modules/@metro-labs/daemon/src/voice/calls.ts +84 -9
- package/runtime/node_modules/@metro-labs/daemon/src/voice/speech.ts +16 -3
- package/runtime/node_modules/@metro-labs/daemon/src/voice/talk.ts +60 -19
- package/runtime/node_modules/@metro-labs/xmtp/src/actions.ts +2 -0
- package/runtime/node_modules/@metro-labs/xmtp/src/call-leftovers.ts +47 -0
- package/runtime/runtime.json +1 -1
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "metro",
|
|
3
|
-
"version": "0.1.0-beta.
|
|
3
|
+
"version": "0.1.0-beta.239",
|
|
4
4
|
"description": "Metro for Claude Code, installed by the metro daemon on this machine: every connector the agent holds as its own MCP server relayed through the daemon, the orchestrator-only guard on the main thread, and the standing rules loaded at each session start. No vendor credential ever sits in a config file.",
|
|
5
5
|
"author": {
|
|
6
6
|
"name": "Bonustrack",
|
|
@@ -52,7 +52,7 @@ import type { StationName } from '@metro-labs/core/station-names';
|
|
|
52
52
|
import { startUploadReaper } from '../files/upload-store.js';
|
|
53
53
|
import { startAttachReaper } from '../files/attach-reaper.js';
|
|
54
54
|
import { startCloudWatchPublisher } from '../server/cloudwatch.js';
|
|
55
|
-
import { isCallEvent, onCallEvent } from '../voice/calls.js';
|
|
55
|
+
import { isCallEvent, leaveCallsForShutdown, onCallEvent, sweepLeftoverCalls } from '../voice/calls.js';
|
|
56
56
|
|
|
57
57
|
installCrashGuard();
|
|
58
58
|
acquireLock(join(STATE_DIR, '.tail-lock'));
|
|
@@ -137,6 +137,7 @@ installBearerSessions(agentsDir(), localOwner);
|
|
|
137
137
|
await materializeFrom(fileSource);
|
|
138
138
|
forgetOrphans(knownAccounts());
|
|
139
139
|
supervisor.start();
|
|
140
|
+
sweepLeftoverCalls();
|
|
140
141
|
const metroMcp = await createMetroMcp({ liveEvents: liveEvents() });
|
|
141
142
|
webhookServer = await startWebhookServer(
|
|
142
143
|
emit,
|
|
@@ -196,6 +197,7 @@ async function shutdown(): Promise<void> {
|
|
|
196
197
|
if (shuttingDown) return;
|
|
197
198
|
shuttingDown = true;
|
|
198
199
|
log.info('dispatcher shutting down');
|
|
200
|
+
const leaving = leaveCallsForShutdown();
|
|
199
201
|
unwatchSession();
|
|
200
202
|
tunnel?.stop();
|
|
201
203
|
if (webhookServer) {
|
|
@@ -211,6 +213,7 @@ async function shutdown(): Promise<void> {
|
|
|
211
213
|
}),
|
|
212
214
|
]);
|
|
213
215
|
}
|
|
216
|
+
await leaving;
|
|
214
217
|
await supervisor.stop();
|
|
215
218
|
process.exit(exitCode);
|
|
216
219
|
}
|
|
@@ -10,6 +10,7 @@ const OFFER_WAIT_MS = 45_000;
|
|
|
10
10
|
const LONGEST_CALL_MS = 60 * 60_000;
|
|
11
11
|
const CONTEXT_MESSAGES = 12;
|
|
12
12
|
const CONTEXT_CHARS = 400;
|
|
13
|
+
const LEAVE_RETRY_MS = 3_000;
|
|
13
14
|
|
|
14
15
|
interface CallStart {
|
|
15
16
|
line: string;
|
|
@@ -21,7 +22,7 @@ interface CallStart {
|
|
|
21
22
|
|
|
22
23
|
const str = (value: unknown): string => (typeof value === 'string' ? value : '');
|
|
23
24
|
|
|
24
|
-
async function trainCall(action: string, args: Record<string, unknown>): Promise<unknown> {
|
|
25
|
+
export async function trainCall(action: string, args: Record<string, unknown>): Promise<unknown> {
|
|
25
26
|
const answer = await forwardTrainCall('xmtp', action, args);
|
|
26
27
|
if (answer.error !== undefined) throw new Error(answer.error);
|
|
27
28
|
return answer.result;
|
|
@@ -57,6 +58,7 @@ export class Call {
|
|
|
57
58
|
private timer: ReturnType<typeof setTimeout> | null = null;
|
|
58
59
|
private longest: ReturnType<typeof setTimeout>;
|
|
59
60
|
private over = false;
|
|
61
|
+
private left: Promise<void> = Promise.resolve();
|
|
60
62
|
|
|
61
63
|
constructor(
|
|
62
64
|
readonly start: CallStart,
|
|
@@ -93,11 +95,16 @@ export class Call {
|
|
|
93
95
|
this.answer(this.start.callerPeer, raw.sdp).catch((err: unknown) => {
|
|
94
96
|
this.hangUp(`answering failed: ${errMsg(err)}`);
|
|
95
97
|
});
|
|
96
|
-
else if (raw.kind === 'leave' && fromCaller) this.
|
|
98
|
+
else if (raw.kind === 'leave' && fromCaller) this.hangUp('the caller hung up');
|
|
97
99
|
}
|
|
98
100
|
|
|
99
101
|
hangUp(reason: string): void {
|
|
100
|
-
this.end(reason
|
|
102
|
+
this.end(reason);
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
leave(reason: string): Promise<void> {
|
|
106
|
+
this.end(reason);
|
|
107
|
+
return this.left;
|
|
101
108
|
}
|
|
102
109
|
|
|
103
110
|
private async answer(to: string, offer: string): Promise<void> {
|
|
@@ -111,6 +118,7 @@ export class Call {
|
|
|
111
118
|
},
|
|
112
119
|
});
|
|
113
120
|
const sdp = await this.peer.answer(offer);
|
|
121
|
+
if (this.over) return;
|
|
114
122
|
await this.signal({ kind: 'answer', callId: this.start.callId, from: this.selfPeer, to, sdp });
|
|
115
123
|
}
|
|
116
124
|
|
|
@@ -124,13 +132,15 @@ export class Call {
|
|
|
124
132
|
return trainCall('callSignal', { line: this.start.line, signal });
|
|
125
133
|
}
|
|
126
134
|
|
|
127
|
-
private end(reason: string
|
|
135
|
+
private end(reason: string): void {
|
|
128
136
|
if (this.over) return;
|
|
129
137
|
this.over = true;
|
|
130
138
|
if (this.timer !== null) clearTimeout(this.timer);
|
|
131
139
|
clearTimeout(this.longest);
|
|
132
140
|
log.info({ reason, seconds: Math.round((Date.now() - this.startedAt) / 1000) }, 'voice: call over');
|
|
133
|
-
this.
|
|
141
|
+
this.left = this.sendLeave();
|
|
142
|
+
this.left
|
|
143
|
+
.then(() => this.wrapUp(reason))
|
|
134
144
|
.catch((err: unknown) => {
|
|
135
145
|
log.warn({ err: errMsg(err) }, 'voice: wrapping up the call failed');
|
|
136
146
|
})
|
|
@@ -139,16 +149,26 @@ export class Call {
|
|
|
139
149
|
});
|
|
140
150
|
}
|
|
141
151
|
|
|
142
|
-
private async
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
152
|
+
private async sendLeave(): Promise<void> {
|
|
153
|
+
const leave = { kind: 'leave', callId: this.start.callId, from: this.selfPeer };
|
|
154
|
+
try {
|
|
155
|
+
await this.signal(leave);
|
|
156
|
+
} catch (err) {
|
|
157
|
+
log.warn({ err: errMsg(err) }, 'voice: could not send the hang-up, trying once more');
|
|
158
|
+
await new Promise((r) => setTimeout(r, LEAVE_RETRY_MS));
|
|
159
|
+
await this.signal(leave).catch((again: unknown) => {
|
|
160
|
+
log.warn({ err: errMsg(again) }, 'voice: could not send the hang-up; the next start sends it');
|
|
146
161
|
});
|
|
162
|
+
}
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
private async wrapUp(reason: string): Promise<void> {
|
|
147
166
|
this.peer?.close();
|
|
148
167
|
const notes = await this.talk.finish();
|
|
149
168
|
const minutes = Math.max(1, Math.round((Date.now() - this.startedAt) / 60_000));
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
169
|
+
const timing = this.talk.replyTime === null ? '' : `\n${this.talk.replyTime}`;
|
|
170
|
+
const failure = reason.startsWith('the caller') || reason === 'no offer arrived' ? null : `The voice call ended: ${reason}.`;
|
|
171
|
+
const text = [failure, notes === null ? null : `Call notes (${String(minutes)} min):\n${notes}${timing}`].filter((part) => part !== null).join('\n\n');
|
|
172
|
+
if (text !== '') await trainCall('send', { line: this.start.line, text });
|
|
153
173
|
}
|
|
154
174
|
}
|
|
@@ -2,11 +2,15 @@ import { errMsg, log } from '@metro-labs/core/log';
|
|
|
2
2
|
import { isRecord } from '@metro-labs/core/is-record';
|
|
3
3
|
import type { TrainEvent } from '@metro-labs/core/trains/protocol';
|
|
4
4
|
import { lineReceives, mayApprove } from '../agents/map.js';
|
|
5
|
-
import { Call } from './call.js';
|
|
5
|
+
import { Call, trainCall } from './call.js';
|
|
6
6
|
import { readVoice, voiceReady } from './store.js';
|
|
7
7
|
|
|
8
|
-
const
|
|
8
|
+
const RING_MS = 40_000;
|
|
9
9
|
const CALL_TYPES = new Set(['callInvite', 'callSignal']);
|
|
10
|
+
const SHUTDOWN_LEAVE_MS = 2_000;
|
|
11
|
+
const SWEEP_FIRST_MS = 5_000;
|
|
12
|
+
const SWEEP_EVERY_MS = 10_000;
|
|
13
|
+
const SWEEP_TRIES = 12;
|
|
10
14
|
|
|
11
15
|
let current: Call | null = null;
|
|
12
16
|
|
|
@@ -17,26 +21,43 @@ const contentTypeOf = (env: TrainEvent): string => str(record(env.payload).conte
|
|
|
17
21
|
|
|
18
22
|
export const isCallEvent = (env: TrainEvent): boolean => CALL_TYPES.has(contentTypeOf(env));
|
|
19
23
|
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
if (
|
|
24
|
+
const sameCaller = (call: Call, line: string, from: string): boolean => call.start.line === line && call.start.from === from;
|
|
25
|
+
|
|
26
|
+
function refusal(line: string, from: string): string | null {
|
|
27
|
+
if (current !== null && !sameCaller(current, line, from)) return 'already in a call';
|
|
24
28
|
if (!voiceReady(readVoice())) return 'voice is off or has no API key on the Voice page';
|
|
25
29
|
if (!lineReceives(line) || !mayApprove(line, from)) return 'the caller is not someone who can approve for this channel';
|
|
26
30
|
return null;
|
|
27
31
|
}
|
|
28
32
|
|
|
33
|
+
function decline(line: string, callId: string): void {
|
|
34
|
+
trainCall('callSignal', { line, signal: { kind: 'decline', callId } }).catch((err: unknown) => {
|
|
35
|
+
log.warn({ line, err: errMsg(err) }, 'voice: could not decline a call');
|
|
36
|
+
});
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
function ringing(env: TrainEvent): boolean {
|
|
40
|
+
const sent = Date.parse(str(env.ts));
|
|
41
|
+
return !Number.isNaN(sent) && Date.now() - sent < RING_MS;
|
|
42
|
+
}
|
|
43
|
+
|
|
29
44
|
function invite(env: TrainEvent, body: Record<string, unknown>): void {
|
|
30
45
|
const line = str(env.line);
|
|
31
46
|
const from = str(env.from);
|
|
32
47
|
const callId = str(body.callId);
|
|
33
48
|
const callerPeer = str(body.from);
|
|
34
|
-
if (callId === '' || callerPeer === '') return;
|
|
35
|
-
|
|
49
|
+
if (callId === '' || callerPeer === '' || current?.start.callId === callId) return;
|
|
50
|
+
if (!ringing(env)) {
|
|
51
|
+
log.info({ line }, 'voice: an invite came in after the caller stopped ringing, not answered');
|
|
52
|
+
return;
|
|
53
|
+
}
|
|
54
|
+
const refused = refusal(line, from);
|
|
36
55
|
if (refused !== null) {
|
|
37
|
-
log.info({ line, reason: refused }, 'voice: a call came in and is
|
|
56
|
+
log.info({ line, reason: refused }, 'voice: a call came in and is declined');
|
|
57
|
+
decline(line, callId);
|
|
38
58
|
return;
|
|
39
59
|
}
|
|
60
|
+
current?.hangUp('the caller started a new call');
|
|
40
61
|
const callerName = str(env.from_display_name) || str(env.from_name) || 'the caller';
|
|
41
62
|
const call = new Call({ line, from, callerName, callId, callerPeer }, readVoice(), () => {
|
|
42
63
|
if (current === call) current = null;
|
|
@@ -52,3 +73,57 @@ export function onCallEvent(env: TrainEvent): void {
|
|
|
52
73
|
if (contentTypeOf(env) === 'callInvite') invite(env, body);
|
|
53
74
|
else if (current !== null && env.line === current.start.line) current.signalled(body);
|
|
54
75
|
}
|
|
76
|
+
|
|
77
|
+
export function leaveCallsForShutdown(): Promise<void> {
|
|
78
|
+
if (current === null) return Promise.resolve();
|
|
79
|
+
const leaving = current.leave('Metro stopped or restarted');
|
|
80
|
+
const capped = new Promise<void>((resolve) => {
|
|
81
|
+
setTimeout(resolve, SHUTDOWN_LEAVE_MS).unref();
|
|
82
|
+
});
|
|
83
|
+
return Promise.race([leaving, capped]);
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
interface Leftover {
|
|
87
|
+
line: string;
|
|
88
|
+
callId: string;
|
|
89
|
+
peer: string;
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
function leftoversOf(result: unknown): Leftover[] {
|
|
93
|
+
const rows = isRecord(result) && Array.isArray(result.calls) ? result.calls : [];
|
|
94
|
+
return rows
|
|
95
|
+
.map((row) => record(row))
|
|
96
|
+
.map((row) => ({ line: str(row.line), callId: str(row.callId), peer: str(row.peer) }))
|
|
97
|
+
.filter((row) => row.line !== '' && row.callId !== '' && row.peer !== '');
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
export async function leaveLeftoverCalls(): Promise<number> {
|
|
101
|
+
const leftovers = leftoversOf(await trainCall('callLeftovers', {}));
|
|
102
|
+
let left = 0;
|
|
103
|
+
for (const { line, callId, peer } of leftovers) {
|
|
104
|
+
if (current?.start.callId === callId) continue;
|
|
105
|
+
try {
|
|
106
|
+
await trainCall('callSignal', { line, signal: { kind: 'leave', callId, from: peer } });
|
|
107
|
+
log.info({ line, callId }, 'voice: left a call this box was still in');
|
|
108
|
+
left += 1;
|
|
109
|
+
} catch (err) {
|
|
110
|
+
log.warn({ line, err: errMsg(err) }, 'voice: could not leave a call this box was still in');
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
return left;
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
function sweep(tries: number, delayMs: number): void {
|
|
117
|
+
setTimeout(() => {
|
|
118
|
+
leaveLeftoverCalls().catch((err: unknown) => {
|
|
119
|
+
const reason = errMsg(err);
|
|
120
|
+
if (reason.startsWith('no train named')) return;
|
|
121
|
+
if (tries > 1) sweep(tries - 1, SWEEP_EVERY_MS);
|
|
122
|
+
else log.warn({ err: reason }, 'voice: could not check for calls left open');
|
|
123
|
+
});
|
|
124
|
+
}, delayMs).unref();
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
export function sweepLeftoverCalls(): void {
|
|
128
|
+
sweep(SWEEP_TRIES, SWEEP_FIRST_MS);
|
|
129
|
+
}
|
|
@@ -26,14 +26,19 @@ const charsOf = (msg: Record<string, unknown>): number => {
|
|
|
26
26
|
return isRecord(alignment) && Array.isArray(alignment.chars) ? alignment.chars.length : 0;
|
|
27
27
|
};
|
|
28
28
|
|
|
29
|
+
const MAX_REASON = 300;
|
|
30
|
+
|
|
29
31
|
export class Utterance {
|
|
30
32
|
private readonly ws: WebSocket;
|
|
31
33
|
private readonly outbox: string[] = [];
|
|
32
34
|
private listener: Voiced | null = null;
|
|
33
35
|
private carry = 0;
|
|
34
36
|
private closed = false;
|
|
37
|
+
private aborted = false;
|
|
38
|
+
private voiced = false;
|
|
39
|
+
private refusal: string | null = null;
|
|
35
40
|
|
|
36
|
-
constructor(apiKey: string, voiceId: string) {
|
|
41
|
+
constructor(apiKey: string, voiceId: string, failed: (reason: string) => void) {
|
|
37
42
|
this.ws = new WebSocket(ttsUrl(voiceId), { headers: { 'xi-api-key': apiKey } });
|
|
38
43
|
this.ws.on('open', () => {
|
|
39
44
|
this.ws.send(JSON.stringify({ text: ' ' }));
|
|
@@ -43,12 +48,15 @@ export class Utterance {
|
|
|
43
48
|
this.onMessage(wsText(data));
|
|
44
49
|
});
|
|
45
50
|
this.ws.on('error', (err) => {
|
|
51
|
+
this.refusal ??= errMsg(err);
|
|
46
52
|
log.warn({ err: errMsg(err) }, 'voice: text to speech connection error');
|
|
47
53
|
});
|
|
48
|
-
this.ws.on('close', () => {
|
|
54
|
+
this.ws.on('close', (code, why) => {
|
|
49
55
|
this.closed = true;
|
|
50
56
|
this.listener?.done();
|
|
51
57
|
this.listener = null;
|
|
58
|
+
const reason = this.refusal ?? (code === 1000 ? null : (why.toString() || `closed (${String(code)})`));
|
|
59
|
+
if (!this.aborted && !this.voiced && reason !== null) failed(reason.slice(0, MAX_REASON));
|
|
52
60
|
});
|
|
53
61
|
}
|
|
54
62
|
|
|
@@ -69,6 +77,7 @@ export class Utterance {
|
|
|
69
77
|
}
|
|
70
78
|
|
|
71
79
|
abort(): void {
|
|
80
|
+
this.aborted = true;
|
|
72
81
|
this.listener = null;
|
|
73
82
|
this.ws.terminate();
|
|
74
83
|
}
|
|
@@ -82,10 +91,14 @@ export class Utterance {
|
|
|
82
91
|
private onMessage(raw: string): void {
|
|
83
92
|
const msg = parsed(raw);
|
|
84
93
|
if (typeof msg.audio === 'string' && msg.audio !== '') {
|
|
94
|
+
this.voiced = true;
|
|
85
95
|
const pcm = pcmOf(Buffer.from(msg.audio, 'base64'));
|
|
86
96
|
const up = upsample2(pcm, this.carry);
|
|
87
97
|
this.carry = pcm.at(-1) ?? this.carry;
|
|
88
98
|
this.listener?.audio(up, charsOf(msg));
|
|
89
|
-
} else if (msg.isFinal !== true)
|
|
99
|
+
} else if (msg.isFinal !== true) {
|
|
100
|
+
if (typeof msg.message === 'string' && msg.message !== '') this.refusal ??= msg.message;
|
|
101
|
+
log.warn({ msg: raw.slice(0, MAX_REASON) }, 'voice: text to speech said');
|
|
102
|
+
}
|
|
90
103
|
}
|
|
91
104
|
}
|
|
@@ -6,7 +6,7 @@ import { Utterance } from './speech.js';
|
|
|
6
6
|
import { Chunker } from './speakable.js';
|
|
7
7
|
import { brainModel, voiceIdOf, type VoiceConfig } from './store.js';
|
|
8
8
|
|
|
9
|
-
const
|
|
9
|
+
const BARGE_MIN_WORDS = 2;
|
|
10
10
|
const NOTES_WAIT_MS = 60_000;
|
|
11
11
|
const NOTES_PROMPT =
|
|
12
12
|
'[The call has ended. Write short call notes for the chat: what was discussed, what was decided and what you will follow up on. Plain text, at most 6 short lines, no title and no greeting. Reply with the notes only.]';
|
|
@@ -17,10 +17,14 @@ interface Reply {
|
|
|
17
17
|
heard: number;
|
|
18
18
|
chunker: Chunker;
|
|
19
19
|
askedAt: number | null;
|
|
20
|
+
saidAt: number | null;
|
|
20
21
|
firstText: number | null;
|
|
21
22
|
firstAudio: number | null;
|
|
23
|
+
firstPlayed: number | null;
|
|
22
24
|
}
|
|
23
25
|
|
|
26
|
+
const wordsIn = (text: string): number => text.split(/\s+/).filter((word) => /\p{L}/u.test(word)).length;
|
|
27
|
+
|
|
24
28
|
const since = (start: number | null, at: number | null): number | null =>
|
|
25
29
|
start === null || at === null ? null : Math.round(at - start);
|
|
26
30
|
|
|
@@ -43,11 +47,15 @@ export class Talk implements Thinking {
|
|
|
43
47
|
private muted = false;
|
|
44
48
|
private busy = false;
|
|
45
49
|
private askedAt: number | null = null;
|
|
50
|
+
private saidAt: number | null = null;
|
|
51
|
+
private lastHeardAt: number | null = null;
|
|
52
|
+
private readonly delays: number[] = [];
|
|
46
53
|
private notes: ((text: string) => void) | null = null;
|
|
47
54
|
private notesStarted = false;
|
|
48
55
|
private live = false;
|
|
49
56
|
private finished = false;
|
|
50
57
|
private heardCaller = false;
|
|
58
|
+
private spoke = false;
|
|
51
59
|
|
|
52
60
|
constructor(
|
|
53
61
|
private readonly cfg: VoiceConfig,
|
|
@@ -76,7 +84,7 @@ export class Talk implements Thinking {
|
|
|
76
84
|
this.ended(reason);
|
|
77
85
|
},
|
|
78
86
|
});
|
|
79
|
-
this.spare =
|
|
87
|
+
this.spare = this.utterance();
|
|
80
88
|
}
|
|
81
89
|
|
|
82
90
|
prime(context: string): void {
|
|
@@ -86,7 +94,7 @@ export class Talk implements Thinking {
|
|
|
86
94
|
|
|
87
95
|
connect(): void {
|
|
88
96
|
this.live = true;
|
|
89
|
-
this.spare ??=
|
|
97
|
+
this.spare ??= this.utterance();
|
|
90
98
|
this.player.start();
|
|
91
99
|
}
|
|
92
100
|
|
|
@@ -115,8 +123,19 @@ export class Talk implements Thinking {
|
|
|
115
123
|
this.busy = true;
|
|
116
124
|
if (this.notes !== null) this.notesStarted = true;
|
|
117
125
|
if (this.notes !== null || this.finished) return;
|
|
118
|
-
this.reply = {
|
|
126
|
+
this.reply = {
|
|
127
|
+
utterance: this.takeUtterance(),
|
|
128
|
+
said: '',
|
|
129
|
+
heard: 0,
|
|
130
|
+
chunker: new Chunker(),
|
|
131
|
+
askedAt: this.askedAt,
|
|
132
|
+
saidAt: this.saidAt,
|
|
133
|
+
firstText: null,
|
|
134
|
+
firstAudio: null,
|
|
135
|
+
firstPlayed: null,
|
|
136
|
+
};
|
|
119
137
|
this.askedAt = null;
|
|
138
|
+
this.saidAt = null;
|
|
120
139
|
}
|
|
121
140
|
|
|
122
141
|
text(delta: string): void {
|
|
@@ -158,14 +177,15 @@ export class Talk implements Thinking {
|
|
|
158
177
|
}
|
|
159
178
|
|
|
160
179
|
private takeUtterance(): Utterance {
|
|
161
|
-
const ready = this.spare?.usable === true ? this.spare :
|
|
162
|
-
this.spare = this.live ?
|
|
180
|
+
const ready = this.spare?.usable === true ? this.spare : this.utterance();
|
|
181
|
+
this.spare = this.live ? this.utterance() : null;
|
|
163
182
|
if (ready === this.spare) this.spare = null;
|
|
164
183
|
const reply = (): Reply | null => this.reply;
|
|
165
184
|
ready.attach({
|
|
166
185
|
audio: (pcm, chars) => {
|
|
167
186
|
const current = reply();
|
|
168
187
|
if (current?.utterance !== ready) return;
|
|
188
|
+
this.spoke = true;
|
|
169
189
|
current.firstAudio ??= performance.now();
|
|
170
190
|
this.player.push(pcm, chars);
|
|
171
191
|
},
|
|
@@ -176,6 +196,13 @@ export class Talk implements Thinking {
|
|
|
176
196
|
return ready;
|
|
177
197
|
}
|
|
178
198
|
|
|
199
|
+
private utterance(): Utterance {
|
|
200
|
+
return new Utterance(this.cfg.apiKey, voiceIdOf(this.cfg), (reason) => {
|
|
201
|
+
log.warn({ reason }, 'voice: text to speech gave no audio');
|
|
202
|
+
if (!this.spoke && !this.finished) this.ended(`ElevenLabs could not speak (${reason})`);
|
|
203
|
+
});
|
|
204
|
+
}
|
|
205
|
+
|
|
179
206
|
private speak(reply: Reply, parts: string[]): void {
|
|
180
207
|
for (const part of parts) {
|
|
181
208
|
reply.said += `${part} `;
|
|
@@ -187,32 +214,46 @@ export class Talk implements Thinking {
|
|
|
187
214
|
if (this.reply !== null) this.reply.heard += chars;
|
|
188
215
|
}
|
|
189
216
|
|
|
217
|
+
get replyTime(): string | null {
|
|
218
|
+
if (this.delays.length === 0) return null;
|
|
219
|
+
const sorted = [...this.delays].sort((a, b) => a - b);
|
|
220
|
+
const middle = sorted[Math.floor(sorted.length / 2)] ?? 0;
|
|
221
|
+
return `Reply time, from your last word heard to my first word sent: about ${(middle / 1000).toFixed(1)} s (middle of ${String(sorted.length)}).`;
|
|
222
|
+
}
|
|
223
|
+
|
|
190
224
|
private firstPlayed(): void {
|
|
191
225
|
const reply = this.reply;
|
|
192
|
-
if (reply?.
|
|
226
|
+
if (reply?.firstPlayed !== null || reply.saidAt === null) return;
|
|
227
|
+
reply.firstPlayed = performance.now();
|
|
228
|
+
this.delays.push(reply.firstPlayed - reply.saidAt);
|
|
229
|
+
log.info({ fromLastWord: since(reply.saidAt, reply.firstPlayed), fromCommit: since(reply.askedAt, reply.firstPlayed) }, 'voice: first reply audio sent to the caller, in ms');
|
|
193
230
|
}
|
|
194
231
|
|
|
195
|
-
private
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
232
|
+
private interrupts(text: string): boolean {
|
|
233
|
+
return !this.muted && this.player.busy && wordsIn(text) >= BARGE_MIN_WORDS;
|
|
234
|
+
}
|
|
235
|
+
|
|
236
|
+
private cutOff(): void {
|
|
237
|
+
this.cut = heardNote(this.reply);
|
|
238
|
+
this.reply?.utterance.abort();
|
|
201
239
|
this.player.clear();
|
|
202
240
|
this.muted = true;
|
|
203
241
|
}
|
|
204
242
|
|
|
243
|
+
private partial(text: string): void {
|
|
244
|
+
if (text !== '') this.lastHeardAt = performance.now();
|
|
245
|
+
if (this.interrupts(text)) this.cutOff();
|
|
246
|
+
}
|
|
247
|
+
|
|
205
248
|
private committed(text: string): void {
|
|
206
249
|
this.heardCaller = true;
|
|
250
|
+
if (this.interrupts(text)) this.cutOff();
|
|
207
251
|
const note = this.cut;
|
|
208
|
-
const now = this.
|
|
252
|
+
const now = this.muted;
|
|
209
253
|
this.cut = null;
|
|
210
|
-
if (now && !this.muted) {
|
|
211
|
-
this.reply?.utterance.abort();
|
|
212
|
-
this.player.clear();
|
|
213
|
-
this.muted = true;
|
|
214
|
-
}
|
|
215
254
|
this.askedAt = performance.now();
|
|
255
|
+
this.saidAt = this.lastHeardAt ?? this.askedAt;
|
|
256
|
+
this.lastHeardAt = null;
|
|
216
257
|
this.busy = true;
|
|
217
258
|
this.brain.tell(note === null ? text : `${note}\n${text}`, now);
|
|
218
259
|
}
|
|
@@ -14,6 +14,7 @@ import { resolveMsgId } from './wire.js';
|
|
|
14
14
|
import { emitOutbound } from './emit.js';
|
|
15
15
|
import { CallSignalCodec, FrameCodec, PollCodec, buildPollContent, encodeDeleteMessage } from './codecs.js';
|
|
16
16
|
import { buildFrameContent } from './frames.js';
|
|
17
|
+
import { callLeftovers } from './call-leftovers.js';
|
|
17
18
|
import { convHandlers } from './actions-conv.js';
|
|
18
19
|
import { messagingAliases } from '@metro-labs/core/stations/messaging-normalize';
|
|
19
20
|
import { TrainError } from '@metro-labs/core/train-error';
|
|
@@ -233,6 +234,7 @@ const handlers: Record<string, (id: string, args: Args) => Promise<void>> = {
|
|
|
233
234
|
reply,
|
|
234
235
|
delete: remove,
|
|
235
236
|
callSignal,
|
|
237
|
+
callLeftovers: (id) => callLeftovers(id),
|
|
236
238
|
sendAttachment,
|
|
237
239
|
sendImage,
|
|
238
240
|
...convHandlers,
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
import type { DecodedMessage } from '@xmtp/node-sdk';
|
|
2
|
+
import { isRecord } from '@metro-labs/core/is-record';
|
|
3
|
+
import { TrainError } from '@metro-labs/core/train-error';
|
|
4
|
+
import { respond } from '@metro-labs/core/stations/station-runtime';
|
|
5
|
+
import { accounts, lineOf, type Account } from './accounts.js';
|
|
6
|
+
import { isCallSignal } from './codecs.js';
|
|
7
|
+
|
|
8
|
+
const CALL_WINDOW_MS = 2 * 60 * 60_000;
|
|
9
|
+
|
|
10
|
+
interface Leftover {
|
|
11
|
+
line: string;
|
|
12
|
+
callId: string;
|
|
13
|
+
peer: string;
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
function ownSignal(acct: Account, m: DecodedMessage): Record<string, unknown> | null {
|
|
17
|
+
if (m.senderInboxId !== acct.inboxId || !isCallSignal(m)) return null;
|
|
18
|
+
return isRecord(m.content) ? m.content : null;
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
function openJoins(acct: Account, line: string, messages: DecodedMessage[]): Leftover[] {
|
|
22
|
+
const open = new Map<string, string>();
|
|
23
|
+
for (const m of messages) {
|
|
24
|
+
const signal = ownSignal(acct, m);
|
|
25
|
+
if (signal === null || typeof signal.from !== 'string' || typeof signal.callId !== 'string') continue;
|
|
26
|
+
if (signal.kind === 'join') open.set(signal.from, signal.callId);
|
|
27
|
+
else if (signal.kind === 'leave') open.delete(signal.from);
|
|
28
|
+
}
|
|
29
|
+
return [...open].map(([peer, callId]) => ({ line, callId, peer }));
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
async function leftoversOf(acct: Account, sinceNs: bigint): Promise<Leftover[]> {
|
|
33
|
+
const found: Leftover[] = [];
|
|
34
|
+
for (const conv of await acct.client.conversations.list()) {
|
|
35
|
+
const messages = await conv.messages({ sentAfterNs: sinceNs, direction: 0 });
|
|
36
|
+
found.push(...openJoins(acct, lineOf(acct.cfg.id, conv.id), messages));
|
|
37
|
+
}
|
|
38
|
+
return found;
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
export async function callLeftovers(id: string): Promise<void> {
|
|
42
|
+
if (accounts.size === 0) throw new TrainError('UNAVAILABLE', 'no XMTP account is ready yet');
|
|
43
|
+
const sinceNs = BigInt(Date.now() - CALL_WINDOW_MS) * 1_000_000n;
|
|
44
|
+
const calls: Leftover[] = [];
|
|
45
|
+
for (const acct of accounts.values()) calls.push(...(await leftoversOf(acct, sinceNs)));
|
|
46
|
+
respond(id, { result: { calls } });
|
|
47
|
+
}
|
package/runtime/runtime.json
CHANGED