@stage-labs/metro 0.1.0-beta.240 → 0.1.0-beta.242
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli.js +3 -2
- package/dist/mcp-config.js +6 -4
- package/dist/voice.js +27 -10
- package/package.json +1 -1
- package/runtime/marketplace/plugin/.claude-plugin/plugin.json +1 -1
- package/runtime/marketplace/plugin/bin/guard.mjs +5 -1
- package/runtime/marketplace/plugin/bin/session-start.mjs +1 -1
- package/runtime/node_modules/@metro-labs/daemon/src/boot/paths.ts +2 -2
- package/runtime/node_modules/@metro-labs/daemon/src/mcp/index.ts +8 -4
- package/runtime/node_modules/@metro-labs/daemon/src/mcp/instructions.ts +11 -0
- package/runtime/node_modules/@metro-labs/daemon/src/mcp/session-slot.ts +7 -3
- package/runtime/node_modules/@metro-labs/daemon/src/mcp/session.ts +17 -16
- package/runtime/node_modules/@metro-labs/daemon/src/routes/http.ts +2 -1
- package/runtime/node_modules/@metro-labs/daemon/src/voice/approvals.ts +43 -0
- package/runtime/node_modules/@metro-labs/daemon/src/voice/brain.ts +61 -21
- package/runtime/node_modules/@metro-labs/daemon/src/voice/call.ts +37 -11
- package/runtime/node_modules/@metro-labs/daemon/src/voice/calls.ts +2 -1
- package/runtime/node_modules/@metro-labs/daemon/src/voice/scribe.ts +4 -3
- package/runtime/node_modules/@metro-labs/daemon/src/voice/store.ts +20 -1
- package/runtime/node_modules/@metro-labs/daemon/src/voice/talk.ts +32 -51
- package/runtime/node_modules/@metro-labs/telegram/src/client.ts +2 -2
- package/runtime/node_modules/@metro-labs/xmtp/src/actions-conv.ts +2 -1
- package/runtime/runtime.json +1 -1
package/dist/cli.js
CHANGED
|
@@ -41,8 +41,9 @@ const USAGE = `metro: run your agent on this machine
|
|
|
41
41
|
metro voice --model <model>
|
|
42
42
|
the agent's side of a voice call, which the daemon starts when a call
|
|
43
43
|
comes in: Claude Code reads the call's words as stream-json on stdin and
|
|
44
|
-
streams its replies on stdout, with the agent's memory, skills
|
|
45
|
-
|
|
44
|
+
streams its replies on stdout, with the agent's memory, skills, plugin,
|
|
45
|
+
connectors and metro tools (on /mcp/voice, so the chat session keeps its
|
|
46
|
+
slot) but no metro channel; tool approvals come back on stdout
|
|
46
47
|
metro update update to the newest published version (--check only reports)
|
|
47
48
|
metro version print this CLI's version
|
|
48
49
|
|
package/dist/mcp-config.js
CHANGED
|
@@ -2,20 +2,22 @@ import { mkdtempSync, rmSync, writeFileSync } from 'node:fs';
|
|
|
2
2
|
import { tmpdir } from 'node:os';
|
|
3
3
|
import { join } from 'node:path';
|
|
4
4
|
const METRO_SERVER = 'metro';
|
|
5
|
-
|
|
5
|
+
const CHAT_PATH = '/mcp';
|
|
6
|
+
export const VOICE_MCP_PATH = '/mcp/voice';
|
|
7
|
+
export function metroMcpConfig(agentKey, port, path = CHAT_PATH) {
|
|
6
8
|
const servers = {
|
|
7
9
|
[METRO_SERVER]: {
|
|
8
10
|
type: 'http',
|
|
9
|
-
url: `http://127.0.0.1:${String(port)}
|
|
11
|
+
url: `http://127.0.0.1:${String(port)}${path}`,
|
|
10
12
|
headers: { Authorization: `Bearer ${agentKey}` },
|
|
11
13
|
},
|
|
12
14
|
};
|
|
13
15
|
return `${JSON.stringify({ mcpServers: servers }, null, 2)}\n`;
|
|
14
16
|
}
|
|
15
|
-
export function writeMcpConfig(agentKey, port, base = tmpdir()) {
|
|
17
|
+
export function writeMcpConfig(agentKey, port, base = tmpdir(), mcpPath = CHAT_PATH) {
|
|
16
18
|
const dir = mkdtempSync(join(base, 'metro-claude-'));
|
|
17
19
|
const path = join(dir, 'mcp.json');
|
|
18
|
-
writeFileSync(path, metroMcpConfig(agentKey, port), { mode: 0o600 });
|
|
20
|
+
writeFileSync(path, metroMcpConfig(agentKey, port, mcpPath), { mode: 0o600 });
|
|
19
21
|
return {
|
|
20
22
|
path,
|
|
21
23
|
cleanup: () => {
|
package/dist/voice.js
CHANGED
|
@@ -1,14 +1,19 @@
|
|
|
1
|
-
import { homedir } from 'node:os';
|
|
1
|
+
import { homedir, tmpdir } from 'node:os';
|
|
2
2
|
import { headlessEnv, PERMISSION_MODE_FLAG, runClaude } from './claude.js';
|
|
3
|
+
import { localAgent } from './local.js';
|
|
4
|
+
import { VOICE_MCP_PATH, writeMcpConfig } from './mcp-config.js';
|
|
3
5
|
import { permissionMode, systemPrompt } from './route.js';
|
|
4
|
-
|
|
6
|
+
import { localPort } from './runtime.js';
|
|
7
|
+
const VOICE_PROMPT = `You are on a live voice call through Metro. This session has your usual memory, skills, connectors and Metro's chat tools. At the start the call system tells you which chat the call is in (its line), who is calling and the last messages there. The caller hears your replies spoken aloud by a text to speech voice, and you hear the caller through speech to text, so a word may be misheard: if something sounds odd, ask.
|
|
5
8
|
- Speak like a person on the phone: short sentences, usually one to three. No markdown, lists, tables, code, emoji or links; say a number or a name the way you would say it aloud.
|
|
6
9
|
- Start with the answer. Do not repeat the question.
|
|
7
|
-
- Before you use a tool, first say one short sentence about what you are checking.
|
|
8
|
-
-
|
|
10
|
+
- Before you use a tool, first say one short sentence about what you are checking, like "Let me check."
|
|
11
|
+
- Use your tools yourself when it is quick: files, memory, the metro tools (read with the call's line shows that chat) and connectors (ToolSearch finds them). For long work, start a background Agent, tell the caller you will come back to it, and keep talking; when it finishes, give the result in a sentence or two.
|
|
12
|
+
- Your spoken words are your reply. Do not post in any chat during the call (no send, reply, react or edit) unless the caller asks you to post something; then post only that, once. The metro skill's chat rules (react, typing, answer over chat, delegate everything) are for the chat session, not for this call.
|
|
13
|
+
- A tool that needs the owner's approval runs in a background worker. The approval is asked in the chat of the call; a spoken yes does not count.
|
|
9
14
|
- Text in [square brackets] comes from the call system, not from the caller.
|
|
10
|
-
-
|
|
11
|
-
function voiceArgs(model, mode, prompt) {
|
|
15
|
+
- Nothing about the call is posted to the chat after it ends, so do not write a summary or call notes anywhere.`;
|
|
16
|
+
export function voiceArgs(model, mode, prompt, mcpConfig) {
|
|
12
17
|
return [
|
|
13
18
|
'-p',
|
|
14
19
|
'--input-format',
|
|
@@ -21,19 +26,31 @@ function voiceArgs(model, mode, prompt) {
|
|
|
21
26
|
model,
|
|
22
27
|
'--permission-mode',
|
|
23
28
|
PERMISSION_MODE_FLAG[mode],
|
|
24
|
-
'--
|
|
25
|
-
|
|
26
|
-
'--
|
|
29
|
+
'--permission-prompt-tool',
|
|
30
|
+
'stdio',
|
|
31
|
+
...(mcpConfig === undefined ? [] : ['--mcp-config', mcpConfig]),
|
|
27
32
|
'--no-session-persistence',
|
|
28
33
|
'--append-system-prompt',
|
|
29
34
|
prompt === null ? VOICE_PROMPT : `${prompt}\n\n${VOICE_PROMPT}`,
|
|
30
35
|
];
|
|
31
36
|
}
|
|
37
|
+
export const voiceEnv = (env) => ({
|
|
38
|
+
CLAUDE_CODE_MCP_STARTUP_WAIT_MS: '0',
|
|
39
|
+
...env,
|
|
40
|
+
METRO_SESSION: 'voice',
|
|
41
|
+
});
|
|
32
42
|
export async function voiceSession(argv) {
|
|
33
43
|
const at = argv.indexOf('--model');
|
|
34
44
|
const model = at < 0 ? '' : (argv[at + 1] ?? '');
|
|
35
45
|
if (model === '')
|
|
36
46
|
throw new Error('metro voice needs --model <model>');
|
|
37
47
|
process.chdir(homedir());
|
|
38
|
-
|
|
48
|
+
const agent = localAgent();
|
|
49
|
+
const mcp = agent === null ? null : writeMcpConfig(agent.key, localPort(), tmpdir(), VOICE_MCP_PATH);
|
|
50
|
+
try {
|
|
51
|
+
return await runClaude(voiceArgs(model, permissionMode(), systemPrompt(), mcp?.path), voiceEnv(await headlessEnv()));
|
|
52
|
+
}
|
|
53
|
+
finally {
|
|
54
|
+
mcp?.cleanup();
|
|
55
|
+
}
|
|
39
56
|
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "metro",
|
|
3
|
-
"version": "0.1.0-beta.
|
|
3
|
+
"version": "0.1.0-beta.242",
|
|
4
4
|
"description": "Metro for Claude Code, installed by the metro daemon on this machine: every connector the agent holds as its own MCP server relayed through the daemon, the orchestrator-only guard on the main thread, and the standing rules loaded at each session start. No vendor credential ever sits in a config file.",
|
|
5
5
|
"author": {
|
|
6
6
|
"name": "Bonustrack",
|
|
@@ -23,6 +23,7 @@ const ORCHESTRATION = new Set([
|
|
|
23
23
|
const IMAGE = /\.(png|jpe?g|gif|webp|bmp)$/i;
|
|
24
24
|
const MCP = /^mcp__/;
|
|
25
25
|
const METRO_MCP = /^mcp__metro__/;
|
|
26
|
+
const VOICE = process.env.METRO_SESSION === 'voice';
|
|
26
27
|
|
|
27
28
|
function answer(decision, reason) {
|
|
28
29
|
process.stdout.write(
|
|
@@ -48,7 +49,10 @@ function verdict(payload, input) {
|
|
|
48
49
|
return `${tool} blocks the session waiting on the terminal, and nobody is watching it. Decide with your best judgement and state the assumption, or ask over chat with the metro tools and keep working in the meantime.`;
|
|
49
50
|
if (typeof payload.agent_id === 'string' && payload.agent_id !== '') return null;
|
|
50
51
|
if (tool === 'Agent' && input.run_in_background === false)
|
|
51
|
-
return
|
|
52
|
+
return VOICE
|
|
53
|
+
? 'On a voice call a foreground subagent leaves the caller in silence until it finishes. Re-issue the identical Agent call with run_in_background: true, tell the caller you will come back to it, and keep talking.'
|
|
54
|
+
: 'A foreground subagent blinds the main thread: it cannot see or acknowledge inbound metro messages until the agent finishes. Re-issue the identical Agent call with run_in_background: true, then wait for the task notification.';
|
|
55
|
+
if (VOICE) return null;
|
|
52
56
|
if (ORCHESTRATION.has(tool)) return null;
|
|
53
57
|
if (METRO_MCP.test(tool)) return null;
|
|
54
58
|
if (MCP.test(tool))
|
|
@@ -24,7 +24,7 @@ function guidance() {
|
|
|
24
24
|
return '';
|
|
25
25
|
}
|
|
26
26
|
|
|
27
|
-
const text = guidance();
|
|
27
|
+
const text = process.env.METRO_SESSION === 'voice' ? '' : guidance();
|
|
28
28
|
if (text !== '')
|
|
29
29
|
process.stdout.write(
|
|
30
30
|
`${JSON.stringify({ hookSpecificOutput: { hookEventName: 'SessionStart', additionalContext: `This machine runs as a metro agent. Standing rules, kept in the metro skill:\n\n${text}` } })}\n`,
|
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
import { execFileSync } from 'node:child_process';
|
|
2
2
|
import {
|
|
3
3
|
closeSync,
|
|
4
|
-
mkdirSync,
|
|
5
4
|
openSync,
|
|
6
5
|
readFileSync,
|
|
7
6
|
unlinkSync,
|
|
@@ -11,10 +10,11 @@ import { homedir } from 'node:os';
|
|
|
11
10
|
import { join } from 'node:path';
|
|
12
11
|
import { fileURLToPath } from 'node:url';
|
|
13
12
|
import { log } from '@metro-labs/core/log';
|
|
13
|
+
import { ensureSecureDir } from '@metro-labs/core/secure-fs';
|
|
14
14
|
|
|
15
15
|
export const STATE_DIR =
|
|
16
16
|
process.env.METRO_STATE_DIR ?? join(homedir(), '.cache', 'metro');
|
|
17
|
-
|
|
17
|
+
ensureSecureDir(STATE_DIR);
|
|
18
18
|
|
|
19
19
|
|
|
20
20
|
export function trainsDir(): string {
|
|
@@ -2,6 +2,7 @@ import type { IncomingMessage, ServerResponse } from 'node:http';
|
|
|
2
2
|
import {
|
|
3
3
|
BodyTooLargeError,
|
|
4
4
|
readBody as readBodyBuffer,
|
|
5
|
+
VOICE_MCP_PATH,
|
|
5
6
|
} from '../routes/http.js';
|
|
6
7
|
import {
|
|
7
8
|
authenticate,
|
|
@@ -90,16 +91,19 @@ async function serveGet(
|
|
|
90
91
|
if (served) session.replayMissed();
|
|
91
92
|
}
|
|
92
93
|
|
|
93
|
-
|
|
94
|
+
const isVoicePath = (req: IncomingMessage): boolean => (req.url ?? '').split('?')[0] === VOICE_MCP_PATH;
|
|
95
|
+
|
|
96
|
+
let activeSlots: SessionSlot[] = [];
|
|
94
97
|
|
|
95
98
|
export async function createMetroMcp(options: { liveEvents?: boolean } = {}): Promise<{
|
|
96
99
|
httpHandler: (req: IncomingMessage, res: ServerResponse) => Promise<void>;
|
|
97
100
|
startInbound: () => void;
|
|
98
101
|
setLiveEvents: (on: boolean) => void;
|
|
99
102
|
}> {
|
|
100
|
-
await
|
|
103
|
+
await Promise.all(activeSlots.map((s) => s.close()));
|
|
101
104
|
const slot = new SessionSlot(options.liveEvents);
|
|
102
|
-
|
|
105
|
+
const voiceSlot = new SessionSlot(false, 'voice');
|
|
106
|
+
activeSlots = [slot, voiceSlot];
|
|
103
107
|
|
|
104
108
|
const httpHandler = async (
|
|
105
109
|
req: IncomingMessage,
|
|
@@ -114,7 +118,7 @@ export async function createMetroMcp(options: { liveEvents?: boolean } = {}): Pr
|
|
|
114
118
|
if (!parsed.ok) return;
|
|
115
119
|
await runWithIdentity(identity, async () => {
|
|
116
120
|
const session = await resolveSession(
|
|
117
|
-
slot,
|
|
121
|
+
isVoicePath(req) ? voiceSlot : slot,
|
|
118
122
|
req,
|
|
119
123
|
parsed.body,
|
|
120
124
|
identity,
|
|
@@ -17,3 +17,14 @@ export const MCP_INSTRUCTIONS =
|
|
|
17
17
|
'needing approval must run in a background worker: on the main thread it is refused, so ' +
|
|
18
18
|
'delegate that exact call and keep answering; the worker waits for the owner. Approval ' +
|
|
19
19
|
'prompts are relayed to the chat; the owner answers "yes <id>"/"no <id>".';
|
|
20
|
+
|
|
21
|
+
export const VOICE_MCP_INSTRUCTIONS =
|
|
22
|
+
"Metro's chat tools, opened for a live voice call. The caller hears what you say, so do not " +
|
|
23
|
+
'answer them with `send` or `reply` and do not post the call in a chat; post, react or act on ' +
|
|
24
|
+
'a channel only when the caller asks for it, once. `read` with a `line` (passed verbatim) ' +
|
|
25
|
+
'shows that chat; `group_info` and `list_members` show who is in it; `get_profile` with a ' +
|
|
26
|
+
'`from` names a sender. The owner may block a tool on an account or make it need approval ' +
|
|
27
|
+
'(`list_accounts` shows each `policy`). A blocked call errors. A call needing approval must ' +
|
|
28
|
+
'run in a background worker: on the main thread it is refused, so delegate that exact call ' +
|
|
29
|
+
'and keep talking; the worker waits while the owner is asked in the chat of the call, where ' +
|
|
30
|
+
'they answer "yes <id>"/"no <id>".';
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { randomUUID } from 'node:crypto';
|
|
2
2
|
import { currentBusSeq } from '@metro-labs/core/events';
|
|
3
3
|
import { newReplayLedger } from '../channels/relay.js';
|
|
4
|
-
import { McpSession, channelLog } from './session.js';
|
|
4
|
+
import { McpSession, channelLog, type SessionKind } from './session.js';
|
|
5
5
|
import { allowedAgents, type RequestIdentity } from './request-identity.js';
|
|
6
6
|
|
|
7
7
|
export class SessionSlot {
|
|
@@ -10,7 +10,10 @@ export class SessionSlot {
|
|
|
10
10
|
private started = false;
|
|
11
11
|
private live: boolean;
|
|
12
12
|
|
|
13
|
-
constructor(
|
|
13
|
+
constructor(
|
|
14
|
+
live = true,
|
|
15
|
+
private readonly kind: SessionKind = 'chat',
|
|
16
|
+
) {
|
|
14
17
|
this.live = live;
|
|
15
18
|
}
|
|
16
19
|
|
|
@@ -50,6 +53,7 @@ export class SessionSlot {
|
|
|
50
53
|
if (this.ledger.startAt < 0) this.ledger.startAt = currentBusSeq();
|
|
51
54
|
const id = adoptId ?? randomUUID();
|
|
52
55
|
const session = await McpSession.open({
|
|
56
|
+
kind: this.kind,
|
|
53
57
|
id,
|
|
54
58
|
scope: allowedAgents(identity),
|
|
55
59
|
adopted: adoptId !== undefined,
|
|
@@ -61,7 +65,7 @@ export class SessionSlot {
|
|
|
61
65
|
});
|
|
62
66
|
this.session = session;
|
|
63
67
|
if (this.started && this.live) session.startChannel();
|
|
64
|
-
channelLog('session: opened', 'id', id, 'adopted', adoptId !== undefined);
|
|
68
|
+
channelLog('session: opened', 'id', id, 'kind', this.kind, 'adopted', adoptId !== undefined);
|
|
65
69
|
return session;
|
|
66
70
|
}
|
|
67
71
|
|
|
@@ -7,7 +7,7 @@ import { errMsg } from '@metro-labs/core/log';
|
|
|
7
7
|
import { allowlistForLine, mayApprove, senderPermitted } from '../agents/map.js';
|
|
8
8
|
import { accountStationNames, stationByName } from '../stations/registry.js';
|
|
9
9
|
import { eventInScope } from '../agents/scope.js';
|
|
10
|
-
import { MCP_INSTRUCTIONS } from './instructions.js';
|
|
10
|
+
import { MCP_INSTRUCTIONS, VOICE_MCP_INSTRUCTIONS } from './instructions.js';
|
|
11
11
|
import { BoundedEventStore } from './event-store.js';
|
|
12
12
|
import { registerPermissionRelay } from './permission-relay.js';
|
|
13
13
|
import { registerToolHandlers, toolSchemaSignature } from './tool-dispatch.js';
|
|
@@ -47,7 +47,12 @@ function makeTransport(
|
|
|
47
47
|
return t;
|
|
48
48
|
}
|
|
49
49
|
|
|
50
|
+
export type SessionKind = 'chat' | 'voice';
|
|
51
|
+
|
|
52
|
+
const CHANNEL_CAPABILITIES = { experimental: { 'claude/channel': {}, 'claude/channel/permission': {} } };
|
|
53
|
+
|
|
50
54
|
export interface SessionInit {
|
|
55
|
+
kind?: SessionKind;
|
|
51
56
|
id: string;
|
|
52
57
|
scope: Set<string>;
|
|
53
58
|
adopted: boolean;
|
|
@@ -80,17 +85,12 @@ export class McpSession {
|
|
|
80
85
|
this.onClosed = init.onClosed;
|
|
81
86
|
this.eventStore = new BoundedEventStore();
|
|
82
87
|
this.transport = makeTransport(init.id, this.eventStore, init.adopted);
|
|
88
|
+
const voice = init.kind === 'voice';
|
|
83
89
|
this.server = new Server(
|
|
84
90
|
{ name: 'metro', version: '0.1.0' },
|
|
85
91
|
{
|
|
86
|
-
capabilities: {
|
|
87
|
-
|
|
88
|
-
'claude/channel': {},
|
|
89
|
-
'claude/channel/permission': {},
|
|
90
|
-
},
|
|
91
|
-
tools: { listChanged: true },
|
|
92
|
-
},
|
|
93
|
-
instructions: MCP_INSTRUCTIONS,
|
|
92
|
+
capabilities: { ...(voice ? {} : CHANNEL_CAPABILITIES), tools: { listChanged: true } },
|
|
93
|
+
instructions: voice ? VOICE_MCP_INSTRUCTIONS : MCP_INSTRUCTIONS,
|
|
94
94
|
},
|
|
95
95
|
);
|
|
96
96
|
registerToolHandlers(this.server, {
|
|
@@ -113,13 +113,14 @@ export class McpSession {
|
|
|
113
113
|
approves,
|
|
114
114
|
answerPermission,
|
|
115
115
|
});
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
116
|
+
if (!voice)
|
|
117
|
+
registerPermissionRelay({
|
|
118
|
+
mcp: this.server,
|
|
119
|
+
relay: this.relay,
|
|
120
|
+
inScope: (line) => this.inScope(line),
|
|
121
|
+
live: this.live,
|
|
122
|
+
log: channelLog,
|
|
123
|
+
});
|
|
123
124
|
this.channel = new ChannelRelay({
|
|
124
125
|
relay: this.relay,
|
|
125
126
|
log: channelLog,
|
|
@@ -38,6 +38,7 @@ import { applyMcpCors, handleMcpPreflight } from '@metro-labs/http/cors';
|
|
|
38
38
|
import { METRO_VERSION } from '@metro-labs/core/version';
|
|
39
39
|
|
|
40
40
|
const LRU_CAP = 2_000;
|
|
41
|
+
export const VOICE_MCP_PATH = '/mcp/voice';
|
|
41
42
|
|
|
42
43
|
function dedupKey(
|
|
43
44
|
e: Pick<MetroEvent, 'station' | 'line' | 'messageId'>,
|
|
@@ -213,7 +214,7 @@ export async function startWebhookServer(
|
|
|
213
214
|
|
|
214
215
|
function isMcpPath(req: IncomingMessage): boolean {
|
|
215
216
|
const path = (req.url ?? '').split('?')[0];
|
|
216
|
-
return path === '/' || path === '/mcp';
|
|
217
|
+
return path === '/' || path === '/mcp' || path === VOICE_MCP_PATH;
|
|
217
218
|
}
|
|
218
219
|
|
|
219
220
|
function flatHeaders(req: IncomingMessage): Record<string, string> {
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
import { randomInt } from 'node:crypto';
|
|
2
|
+
import { errMsg, log } from '@metro-labs/core/log';
|
|
3
|
+
import { forgetPromptsOf, holdPrompt, type Behavior } from '../approvals/pending.js';
|
|
4
|
+
import { promptBody } from '../mcp/permission-prompt.js';
|
|
5
|
+
import type { ToolAsk } from './brain.js';
|
|
6
|
+
|
|
7
|
+
const ID_LETTERS = 'abcdefghijkmnopqrstuvwxyz';
|
|
8
|
+
const ID_LENGTH = 5;
|
|
9
|
+
|
|
10
|
+
export const approvalId = (): string => Array.from({ length: ID_LENGTH }, () => ID_LETTERS[randomInt(ID_LETTERS.length)]).join('');
|
|
11
|
+
|
|
12
|
+
export class CallApprovals {
|
|
13
|
+
private closed = false;
|
|
14
|
+
|
|
15
|
+
constructor(
|
|
16
|
+
private readonly line: string,
|
|
17
|
+
private readonly post: (text: string) => Promise<void>,
|
|
18
|
+
) {}
|
|
19
|
+
|
|
20
|
+
ask(ask: ToolAsk, answer: (behavior: Behavior) => void): string | null {
|
|
21
|
+
if (this.closed) return null;
|
|
22
|
+
const id = approvalId();
|
|
23
|
+
const preview = JSON.stringify(ask.input);
|
|
24
|
+
holdPrompt(
|
|
25
|
+
{ requestId: id, tool: ask.tool, description: ask.description, preview, line: this.line, at: Date.now() },
|
|
26
|
+
this,
|
|
27
|
+
(behavior) => {
|
|
28
|
+
answer(behavior);
|
|
29
|
+
return Promise.resolve();
|
|
30
|
+
},
|
|
31
|
+
this.post,
|
|
32
|
+
);
|
|
33
|
+
this.post(promptBody({ request_id: id, tool_name: ask.tool, description: ask.description, input_preview: preview })).catch((err: unknown) => {
|
|
34
|
+
log.warn({ err: errMsg(err) }, 'voice: could not ask for an approval in the chat of the call');
|
|
35
|
+
});
|
|
36
|
+
return id;
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
close(): void {
|
|
40
|
+
this.closed = true;
|
|
41
|
+
forgetPromptsOf(this);
|
|
42
|
+
}
|
|
43
|
+
}
|
|
@@ -6,16 +6,40 @@ import { asAgent } from '../agent-user/user.js';
|
|
|
6
6
|
import { metroCli } from '../claude/session.js';
|
|
7
7
|
|
|
8
8
|
const CLOSE_GRACE_MS = 10_000;
|
|
9
|
+
const DENIED = 'The owner did not approve this.';
|
|
10
|
+
|
|
11
|
+
export interface ToolAsk {
|
|
12
|
+
requestId: string;
|
|
13
|
+
tool: string;
|
|
14
|
+
input: Record<string, unknown>;
|
|
15
|
+
description: string;
|
|
16
|
+
fromWorker: boolean;
|
|
17
|
+
}
|
|
9
18
|
|
|
10
19
|
export interface Thinking {
|
|
11
20
|
turnStarted(): void;
|
|
12
21
|
text(delta: string): void;
|
|
13
22
|
tool(name: string): void;
|
|
14
|
-
turnEnded(
|
|
23
|
+
turnEnded(): void;
|
|
24
|
+
asked(ask: ToolAsk): void;
|
|
15
25
|
exited(reason: string): void;
|
|
16
26
|
}
|
|
17
27
|
|
|
18
28
|
const record = (value: unknown): Record<string, unknown> => (isRecord(value) ? value : {});
|
|
29
|
+
const str = (value: unknown): string => (typeof value === 'string' ? value : '');
|
|
30
|
+
|
|
31
|
+
export function toolAsk(msg: Record<string, unknown>): ToolAsk | null {
|
|
32
|
+
const request = record(msg.request);
|
|
33
|
+
const requestId = str(msg.request_id);
|
|
34
|
+
if (request.subtype !== 'can_use_tool' || requestId === '' || str(request.tool_name) === '') return null;
|
|
35
|
+
return {
|
|
36
|
+
requestId,
|
|
37
|
+
tool: str(request.tool_name),
|
|
38
|
+
input: record(request.input),
|
|
39
|
+
description: str(request.description) || str(request.decision_reason),
|
|
40
|
+
fromWorker: str(request.agent_id) !== '',
|
|
41
|
+
};
|
|
42
|
+
}
|
|
19
43
|
|
|
20
44
|
function streamEvent(msg: Record<string, unknown>, mind: Thinking): void {
|
|
21
45
|
if (msg.parent_tool_use_id !== null && msg.parent_tool_use_id !== undefined) return;
|
|
@@ -26,19 +50,6 @@ function streamEvent(msg: Record<string, unknown>, mind: Thinking): void {
|
|
|
26
50
|
if (event.type === 'content_block_start' && block.type === 'tool_use') mind.tool(typeof block.name === 'string' ? block.name : 'tool');
|
|
27
51
|
}
|
|
28
52
|
|
|
29
|
-
function readBrainLine(line: string, mind: Thinking): void {
|
|
30
|
-
let parsed: unknown;
|
|
31
|
-
try {
|
|
32
|
-
parsed = JSON.parse(line);
|
|
33
|
-
} catch {
|
|
34
|
-
return;
|
|
35
|
-
}
|
|
36
|
-
const msg = record(parsed);
|
|
37
|
-
if (msg.type === 'system' && msg.subtype === 'init') mind.turnStarted();
|
|
38
|
-
else if (msg.type === 'stream_event') streamEvent(msg, mind);
|
|
39
|
-
else if (msg.type === 'result') mind.turnEnded(typeof msg.result === 'string' ? msg.result : '');
|
|
40
|
-
}
|
|
41
|
-
|
|
42
53
|
export class Brain {
|
|
43
54
|
private readonly child: ChildProcessWithoutNullStreams;
|
|
44
55
|
private out = '';
|
|
@@ -51,7 +62,7 @@ export class Brain {
|
|
|
51
62
|
this.child.stdout.setEncoding('utf8');
|
|
52
63
|
this.child.stdout.on('data', (chunk: string) => {
|
|
53
64
|
this.out = drainLines('voice-brain', this.out + chunk, (line) => {
|
|
54
|
-
|
|
65
|
+
this.read(line);
|
|
55
66
|
});
|
|
56
67
|
});
|
|
57
68
|
this.child.stderr.setEncoding('utf8');
|
|
@@ -69,14 +80,14 @@ export class Brain {
|
|
|
69
80
|
});
|
|
70
81
|
}
|
|
71
82
|
|
|
72
|
-
get alive(): boolean {
|
|
73
|
-
return !this.gone;
|
|
74
|
-
}
|
|
75
|
-
|
|
76
83
|
tell(text: string, now = false): void {
|
|
77
|
-
if (this.gone) return;
|
|
78
84
|
const message = { type: 'user', message: { role: 'user', content: text }, parent_tool_use_id: null, session_id: '' };
|
|
79
|
-
this.
|
|
85
|
+
this.write(now ? { ...message, priority: 'now' } : message);
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
answer(ask: ToolAsk, behavior: 'allow' | 'deny'): void {
|
|
89
|
+
const response = behavior === 'allow' ? { behavior, updatedInput: ask.input } : { behavior, message: DENIED };
|
|
90
|
+
this.write({ type: 'control_response', response: { subtype: 'success', request_id: ask.requestId, response } });
|
|
80
91
|
}
|
|
81
92
|
|
|
82
93
|
close(): void {
|
|
@@ -87,6 +98,35 @@ export class Brain {
|
|
|
87
98
|
}, CLOSE_GRACE_MS).unref();
|
|
88
99
|
}
|
|
89
100
|
|
|
101
|
+
private write(message: Record<string, unknown>): void {
|
|
102
|
+
if (!this.gone && this.child.stdin.writable) this.child.stdin.write(`${JSON.stringify(message)}\n`);
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
private read(line: string): void {
|
|
106
|
+
let parsed: unknown;
|
|
107
|
+
try {
|
|
108
|
+
parsed = JSON.parse(line);
|
|
109
|
+
} catch {
|
|
110
|
+
return;
|
|
111
|
+
}
|
|
112
|
+
const msg = record(parsed);
|
|
113
|
+
if (msg.type === 'system' && msg.subtype === 'init') this.mind.turnStarted();
|
|
114
|
+
else if (msg.type === 'stream_event') streamEvent(msg, this.mind);
|
|
115
|
+
else if (msg.type === 'result') this.mind.turnEnded();
|
|
116
|
+
else if (msg.type === 'control_request') this.control(msg);
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
private control(msg: Record<string, unknown>): void {
|
|
120
|
+
const ask = toolAsk(msg);
|
|
121
|
+
if (ask !== null) {
|
|
122
|
+
this.mind.asked(ask);
|
|
123
|
+
return;
|
|
124
|
+
}
|
|
125
|
+
const subtype = str(record(msg.request).subtype);
|
|
126
|
+
log.warn({ subtype }, 'voice: the agent session asked for something the call does not handle');
|
|
127
|
+
this.write({ type: 'control_response', response: { subtype: 'error', request_id: str(msg.request_id), error: `not supported on a voice call: ${subtype}` } });
|
|
128
|
+
}
|
|
129
|
+
|
|
90
130
|
private exit(reason: string): void {
|
|
91
131
|
if (this.gone) return;
|
|
92
132
|
this.gone = true;
|
|
@@ -2,6 +2,7 @@ import { randomUUID } from 'node:crypto';
|
|
|
2
2
|
import { errMsg, log } from '@metro-labs/core/log';
|
|
3
3
|
import { isRecord } from '@metro-labs/core/is-record';
|
|
4
4
|
import { forwardTrainCall } from '../stations/train-call.js';
|
|
5
|
+
import { CallApprovals } from './approvals.js';
|
|
5
6
|
import { Peer } from './peer.js';
|
|
6
7
|
import { Talk } from './talk.js';
|
|
7
8
|
import type { VoiceConfig } from './store.js';
|
|
@@ -14,6 +15,8 @@ const LEAVE_RETRY_MS = 3_000;
|
|
|
14
15
|
|
|
15
16
|
interface CallStart {
|
|
16
17
|
line: string;
|
|
18
|
+
lineName: string;
|
|
19
|
+
direct: boolean;
|
|
17
20
|
from: string;
|
|
18
21
|
callerName: string;
|
|
19
22
|
callId: string;
|
|
@@ -28,9 +31,14 @@ export async function trainCall(action: string, args: Record<string, unknown>):
|
|
|
28
31
|
return answer.result;
|
|
29
32
|
}
|
|
30
33
|
|
|
34
|
+
function speaker(row: Record<string, unknown>, start: CallStart): string {
|
|
35
|
+
if (row.self === true) return 'you';
|
|
36
|
+
return row.from === start.from ? start.callerName : `another member (${str(row.from)})`;
|
|
37
|
+
}
|
|
38
|
+
|
|
31
39
|
function contextLine(row: unknown, start: CallStart): string | null {
|
|
32
40
|
if (!isRecord(row) || typeof row.text !== 'string' || row.text.startsWith('[call')) return null;
|
|
33
|
-
const who = row
|
|
41
|
+
const who = speaker(row, start);
|
|
34
42
|
return `${str(row.ts).slice(11, 16)} ${who}: ${row.text.replace(/\s+/g, ' ').slice(0, CONTEXT_CHARS)}`;
|
|
35
43
|
}
|
|
36
44
|
|
|
@@ -45,9 +53,20 @@ async function recentChat(start: CallStart): Promise<string> {
|
|
|
45
53
|
}
|
|
46
54
|
}
|
|
47
55
|
|
|
48
|
-
function
|
|
49
|
-
|
|
50
|
-
return `
|
|
56
|
+
function where(start: CallStart): string {
|
|
57
|
+
if (start.lineName !== '') return `the group chat "${start.lineName}"`;
|
|
58
|
+
return start.direct ? `your direct chat with ${start.callerName}` : 'a group chat with no name';
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
export function opening(start: CallStart, chat: string): string {
|
|
62
|
+
const recent = chat === '' ? 'No earlier messages were found there.' : `Recent messages there before the call, oldest first (times in UTC):\n${chat}`;
|
|
63
|
+
return [
|
|
64
|
+
`[A voice call with ${start.callerName} is starting on Stage, in ${where(start)}.`,
|
|
65
|
+
`Line of this chat: ${start.line} (pass it verbatim to the metro tools, e.g. read, to look at or act on this chat).`,
|
|
66
|
+
`Caller: ${start.callerName} (${start.from}).`,
|
|
67
|
+
recent,
|
|
68
|
+
`Greet ${start.callerName} in a few words.]`,
|
|
69
|
+
].join('\n');
|
|
51
70
|
}
|
|
52
71
|
|
|
53
72
|
export class Call {
|
|
@@ -65,6 +84,9 @@ export class Call {
|
|
|
65
84
|
cfg: VoiceConfig,
|
|
66
85
|
private readonly onOver: () => void,
|
|
67
86
|
) {
|
|
87
|
+
const approvals = new CallApprovals(start.line, async (text) => {
|
|
88
|
+
await trainCall('send', { line: start.line, text });
|
|
89
|
+
});
|
|
68
90
|
this.talk = new Talk(
|
|
69
91
|
cfg,
|
|
70
92
|
(opus, timestamp, marker) => {
|
|
@@ -73,6 +95,7 @@ export class Call {
|
|
|
73
95
|
(reason) => {
|
|
74
96
|
this.hangUp(reason);
|
|
75
97
|
},
|
|
98
|
+
approvals,
|
|
76
99
|
);
|
|
77
100
|
this.longest = setTimeout(() => {
|
|
78
101
|
this.hangUp('the call reached one hour');
|
|
@@ -81,7 +104,13 @@ export class Call {
|
|
|
81
104
|
|
|
82
105
|
async begin(): Promise<void> {
|
|
83
106
|
log.info({ line: this.start.line, callId: this.start.callId }, 'voice: answering a call');
|
|
84
|
-
|
|
107
|
+
recentChat(this.start)
|
|
108
|
+
.then((chat) => {
|
|
109
|
+
if (!this.over) this.talk.prime(opening(this.start, chat));
|
|
110
|
+
})
|
|
111
|
+
.catch((err: unknown) => {
|
|
112
|
+
log.warn({ err: errMsg(err) }, 'voice: could not start the agent on the call');
|
|
113
|
+
});
|
|
85
114
|
await this.signal({ kind: 'join', callId: this.start.callId, from: this.selfPeer });
|
|
86
115
|
this.timer = setTimeout(() => {
|
|
87
116
|
this.hangUp('no offer arrived');
|
|
@@ -164,11 +193,8 @@ export class Call {
|
|
|
164
193
|
|
|
165
194
|
private async wrapUp(reason: string): Promise<void> {
|
|
166
195
|
this.peer?.close();
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
const failure = reason.startsWith('the caller') || reason === 'no offer arrived' ? null : `The voice call ended: ${reason}.`;
|
|
171
|
-
const text = [failure, notes === null ? null : `Call notes (${String(minutes)} min):\n${notes}${timing}`].filter((part) => part !== null).join('\n\n');
|
|
172
|
-
if (text !== '') await trainCall('send', { line: this.start.line, text });
|
|
196
|
+
this.talk.finish();
|
|
197
|
+
if (reason.startsWith('the caller') || reason === 'no offer arrived') return;
|
|
198
|
+
await trainCall('send', { line: this.start.line, text: `The voice call ended: ${reason}.` });
|
|
173
199
|
}
|
|
174
200
|
}
|
|
@@ -59,7 +59,8 @@ function invite(env: TrainEvent, body: Record<string, unknown>): void {
|
|
|
59
59
|
}
|
|
60
60
|
current?.hangUp('the caller started a new call');
|
|
61
61
|
const callerName = str(env.from_display_name) || str(env.from_name) || 'the caller';
|
|
62
|
-
const
|
|
62
|
+
const start = { line, lineName: str(env.line_name), direct: env.is_private === true, from, callerName, callId, callerPeer };
|
|
63
|
+
const call = new Call(start, readVoice(), () => {
|
|
63
64
|
if (current === call) current = null;
|
|
64
65
|
});
|
|
65
66
|
current = call;
|
|
@@ -39,13 +39,14 @@ export function parsed(raw: string): Record<string, unknown> {
|
|
|
39
39
|
}
|
|
40
40
|
}
|
|
41
41
|
|
|
42
|
-
function sttUrl(): string {
|
|
42
|
+
export function sttUrl(language: string | null): string {
|
|
43
43
|
const query = new URLSearchParams({
|
|
44
44
|
model_id: 'scribe_v2_realtime',
|
|
45
45
|
audio_format: 'pcm_16000',
|
|
46
46
|
commit_strategy: 'vad',
|
|
47
47
|
vad_silence_threshold_secs: String(VAD_SILENCE_SECS),
|
|
48
48
|
});
|
|
49
|
+
if (language !== null) query.set('language_code', language);
|
|
49
50
|
return `${elevenLabs()}/v1/speech-to-text/realtime?${query.toString()}`;
|
|
50
51
|
}
|
|
51
52
|
|
|
@@ -54,8 +55,8 @@ export class Scribe {
|
|
|
54
55
|
private buffer: Int16Array[] = [];
|
|
55
56
|
private buffered = 0;
|
|
56
57
|
|
|
57
|
-
constructor(apiKey: string, private readonly heard: Heard) {
|
|
58
|
-
this.ws = new WebSocket(sttUrl(), { headers: { 'xi-api-key': apiKey } });
|
|
58
|
+
constructor(apiKey: string, language: string | null, private readonly heard: Heard) {
|
|
59
|
+
this.ws = new WebSocket(sttUrl(language), { headers: { 'xi-api-key': apiKey } });
|
|
59
60
|
this.ws.on('message', (data) => {
|
|
60
61
|
this.onMessage(wsText(data));
|
|
61
62
|
});
|
|
@@ -13,9 +13,14 @@ export interface VoiceConfig {
|
|
|
13
13
|
apiKey: string;
|
|
14
14
|
voiceId: string;
|
|
15
15
|
model: string;
|
|
16
|
+
language: string;
|
|
16
17
|
enabled: boolean;
|
|
17
18
|
}
|
|
18
19
|
|
|
20
|
+
const DEFAULT_LANGUAGE = 'en';
|
|
21
|
+
const AUTO_LANGUAGE = 'auto';
|
|
22
|
+
const LANGUAGE_RE = /^(?:[a-z]{2,3}|auto)$/;
|
|
23
|
+
|
|
19
24
|
const DEFAULT_VOICE_ID = 'EXAVITQu4vr4xnSDxMaL';
|
|
20
25
|
const DEFAULT_VOICE_MODEL = 'anthropic:claude-sonnet-5-5';
|
|
21
26
|
|
|
@@ -34,6 +39,7 @@ export function parseVoice(raw: unknown): VoiceConfig {
|
|
|
34
39
|
apiKey: text(r.apiKey),
|
|
35
40
|
voiceId: text(r.voiceId),
|
|
36
41
|
model: text(r.model),
|
|
42
|
+
language: LANGUAGE_RE.test(text(r.language)) ? text(r.language) : '',
|
|
37
43
|
enabled: r.enabled !== false,
|
|
38
44
|
};
|
|
39
45
|
}
|
|
@@ -54,6 +60,11 @@ export function brainModel(cfg: VoiceConfig, models: ModelConfig = readModelConf
|
|
|
54
60
|
return models.connections.some((c) => c.provider === 'anthropic') ? model : anthropic;
|
|
55
61
|
}
|
|
56
62
|
|
|
63
|
+
export function languageOf(cfg: VoiceConfig): string | null {
|
|
64
|
+
const language = cfg.language === '' ? DEFAULT_LANGUAGE : cfg.language;
|
|
65
|
+
return language === AUTO_LANGUAGE ? null : language;
|
|
66
|
+
}
|
|
67
|
+
|
|
57
68
|
export const voiceReady = (cfg: VoiceConfig): boolean => cfg.enabled && cfg.apiKey !== '';
|
|
58
69
|
|
|
59
70
|
export function publicVoice(cfg: VoiceConfig): Record<string, unknown> {
|
|
@@ -63,8 +74,9 @@ export function publicVoice(cfg: VoiceConfig): Record<string, unknown> {
|
|
|
63
74
|
hasKey: cfg.apiKey !== '',
|
|
64
75
|
voiceId: cfg.voiceId,
|
|
65
76
|
model: cfg.model,
|
|
77
|
+
language: cfg.language,
|
|
66
78
|
enabled: cfg.enabled,
|
|
67
|
-
defaults: { voiceId: DEFAULT_VOICE_ID, model: DEFAULT_VOICE_MODEL },
|
|
79
|
+
defaults: { voiceId: DEFAULT_VOICE_ID, model: DEFAULT_VOICE_MODEL, language: DEFAULT_LANGUAGE },
|
|
68
80
|
};
|
|
69
81
|
}
|
|
70
82
|
|
|
@@ -82,6 +94,12 @@ function keyOf(body: Record<string, unknown>, current: string): string {
|
|
|
82
94
|
return next === '' ? current : next;
|
|
83
95
|
}
|
|
84
96
|
|
|
97
|
+
function languageField(body: Record<string, unknown>, current: string): string {
|
|
98
|
+
const next = field(body, 'language', current).toLowerCase();
|
|
99
|
+
if (next !== '' && !LANGUAGE_RE.test(next)) throw new ApiError("language is a two or three letter ISO 639 code, or 'auto'", 400);
|
|
100
|
+
return next;
|
|
101
|
+
}
|
|
102
|
+
|
|
85
103
|
export function patchVoice(cfg: VoiceConfig, body: unknown): VoiceConfig {
|
|
86
104
|
if (!isRecord(body)) throw new ApiError('the body must be a JSON object', 400);
|
|
87
105
|
if (body.provider !== undefined && !isProvider(body.provider))
|
|
@@ -92,6 +110,7 @@ export function patchVoice(cfg: VoiceConfig, body: unknown): VoiceConfig {
|
|
|
92
110
|
apiKey: keyOf(body, cfg.apiKey),
|
|
93
111
|
voiceId: field(body, 'voiceId', cfg.voiceId),
|
|
94
112
|
model: field(body, 'model', cfg.model),
|
|
113
|
+
language: languageField(body, cfg.language),
|
|
95
114
|
enabled: typeof body.enabled === 'boolean' ? body.enabled : cfg.enabled,
|
|
96
115
|
};
|
|
97
116
|
}
|
|
@@ -1,15 +1,13 @@
|
|
|
1
1
|
import { log } from '@metro-labs/core/log';
|
|
2
2
|
import { downsample3, OpusIn, Player, type Playback } from './audio.js';
|
|
3
|
-
import {
|
|
3
|
+
import type { CallApprovals } from './approvals.js';
|
|
4
|
+
import { Brain, type Thinking, type ToolAsk } from './brain.js';
|
|
4
5
|
import { Scribe, VAD_SILENCE_SECS } from './scribe.js';
|
|
5
6
|
import { Utterance } from './speech.js';
|
|
6
7
|
import { Chunker } from './speakable.js';
|
|
7
|
-
import { brainModel, voiceIdOf, type VoiceConfig } from './store.js';
|
|
8
|
+
import { brainModel, languageOf, voiceIdOf, type VoiceConfig } from './store.js';
|
|
8
9
|
|
|
9
10
|
const BARGE_MIN_WORDS = 2;
|
|
10
|
-
const NOTES_WAIT_MS = 60_000;
|
|
11
|
-
const NOTES_PROMPT =
|
|
12
|
-
'[The call has ended. Write short call notes for the chat: what was discussed, what was decided and what you will follow up on. Plain text, at most 6 short lines, no title and no greeting. Reply with the notes only.]';
|
|
13
11
|
|
|
14
12
|
interface Reply {
|
|
15
13
|
utterance: Utterance;
|
|
@@ -23,6 +21,11 @@ interface Reply {
|
|
|
23
21
|
firstPlayed: number | null;
|
|
24
22
|
}
|
|
25
23
|
|
|
24
|
+
const toolName = (tool: string): string => tool.split('__').pop() ?? tool;
|
|
25
|
+
|
|
26
|
+
const approvalNote = (tool: string, id: string): string =>
|
|
27
|
+
`[The owner was asked in the chat of this call to approve ${toolName(tool)}; they answer "yes ${id}" or "no ${id}" there. Unless you already said so, tell the caller in one short sentence that it waits for their answer in the chat.]`;
|
|
28
|
+
|
|
26
29
|
const wordsIn = (text: string): number => text.split(/\s+/).filter((word) => /\p{L}/u.test(word)).length;
|
|
27
30
|
|
|
28
31
|
const since = (start: number | null, at: number | null): number | null =>
|
|
@@ -45,22 +48,19 @@ export class Talk implements Thinking {
|
|
|
45
48
|
private reply: Reply | null = null;
|
|
46
49
|
private cut: string | null = null;
|
|
47
50
|
private muted = false;
|
|
48
|
-
private busy = false;
|
|
49
51
|
private askedAt: number | null = null;
|
|
50
52
|
private saidAt: number | null = null;
|
|
51
53
|
private lastHeardAt: number | null = null;
|
|
52
54
|
private readonly delays: number[] = [];
|
|
53
|
-
private notes: ((text: string) => void) | null = null;
|
|
54
|
-
private notesStarted = false;
|
|
55
55
|
private live = false;
|
|
56
56
|
private finished = false;
|
|
57
|
-
private heardCaller = false;
|
|
58
57
|
private spoke = false;
|
|
59
58
|
|
|
60
59
|
constructor(
|
|
61
60
|
private readonly cfg: VoiceConfig,
|
|
62
61
|
out: (opus: Buffer, timestamp: number, marker: boolean) => void,
|
|
63
62
|
private readonly ended: (reason: string) => void,
|
|
63
|
+
private readonly approvals: CallApprovals,
|
|
64
64
|
) {
|
|
65
65
|
const playback: Playback = {
|
|
66
66
|
send: out,
|
|
@@ -73,7 +73,7 @@ export class Talk implements Thinking {
|
|
|
73
73
|
this.firstPlayed();
|
|
74
74
|
});
|
|
75
75
|
this.brain = new Brain(brainModel(cfg), this);
|
|
76
|
-
this.scribe = new Scribe(cfg.apiKey, {
|
|
76
|
+
this.scribe = new Scribe(cfg.apiKey, languageOf(cfg), {
|
|
77
77
|
partial: (text) => {
|
|
78
78
|
this.partial(text);
|
|
79
79
|
},
|
|
@@ -88,7 +88,6 @@ export class Talk implements Thinking {
|
|
|
88
88
|
}
|
|
89
89
|
|
|
90
90
|
prime(context: string): void {
|
|
91
|
-
this.busy = true;
|
|
92
91
|
this.brain.tell(context);
|
|
93
92
|
}
|
|
94
93
|
|
|
@@ -104,25 +103,23 @@ export class Talk implements Thinking {
|
|
|
104
103
|
if (pcm !== null) this.scribe.send(downsample3(pcm));
|
|
105
104
|
}
|
|
106
105
|
|
|
107
|
-
|
|
108
|
-
if (this.finished) return
|
|
106
|
+
finish(): void {
|
|
107
|
+
if (this.finished) return;
|
|
109
108
|
this.finished = true;
|
|
110
109
|
this.live = false;
|
|
110
|
+
this.approvals.close();
|
|
111
111
|
this.player.stop();
|
|
112
112
|
this.scribe.close();
|
|
113
113
|
this.reply?.utterance.abort();
|
|
114
114
|
this.spare?.abort();
|
|
115
115
|
this.opus.close();
|
|
116
|
-
const notes = this.heardCaller && this.brain.alive ? await this.callNotes() : null;
|
|
117
116
|
this.brain.close();
|
|
118
|
-
|
|
117
|
+
this.logReplyTime();
|
|
119
118
|
}
|
|
120
119
|
|
|
121
120
|
turnStarted(): void {
|
|
122
121
|
this.muted = false;
|
|
123
|
-
this.
|
|
124
|
-
if (this.notes !== null) this.notesStarted = true;
|
|
125
|
-
if (this.notes !== null || this.finished) return;
|
|
122
|
+
if (this.finished) return;
|
|
126
123
|
this.reply = {
|
|
127
124
|
utterance: this.takeUtterance(),
|
|
128
125
|
said: '',
|
|
@@ -140,7 +137,7 @@ export class Talk implements Thinking {
|
|
|
140
137
|
|
|
141
138
|
text(delta: string): void {
|
|
142
139
|
const reply = this.reply;
|
|
143
|
-
if (this.muted || reply === null
|
|
140
|
+
if (this.muted || reply === null) return;
|
|
144
141
|
reply.firstText ??= performance.now();
|
|
145
142
|
this.speak(reply, reply.chunker.add(delta));
|
|
146
143
|
}
|
|
@@ -150,15 +147,7 @@ export class Talk implements Thinking {
|
|
|
150
147
|
if (this.reply !== null && !this.muted) this.speak(this.reply, this.reply.chunker.flush());
|
|
151
148
|
}
|
|
152
149
|
|
|
153
|
-
turnEnded(
|
|
154
|
-
this.busy = false;
|
|
155
|
-
const notes = this.notes;
|
|
156
|
-
if (notes !== null) {
|
|
157
|
-
if (!this.notesStarted) return;
|
|
158
|
-
this.notes = null;
|
|
159
|
-
notes(text);
|
|
160
|
-
return;
|
|
161
|
-
}
|
|
150
|
+
turnEnded(): void {
|
|
162
151
|
const reply = this.reply;
|
|
163
152
|
if (reply === null || this.muted) return;
|
|
164
153
|
this.speak(reply, reply.chunker.flush());
|
|
@@ -169,10 +158,20 @@ export class Talk implements Thinking {
|
|
|
169
158
|
);
|
|
170
159
|
}
|
|
171
160
|
|
|
161
|
+
asked(ask: ToolAsk): void {
|
|
162
|
+
const id = this.finished ? null : this.approvals.ask(ask, (behavior) => {
|
|
163
|
+
this.brain.answer(ask, behavior);
|
|
164
|
+
});
|
|
165
|
+
if (id === null) {
|
|
166
|
+
this.brain.answer(ask, 'deny');
|
|
167
|
+
return;
|
|
168
|
+
}
|
|
169
|
+
log.info({ tool: ask.tool, worker: ask.fromWorker }, 'voice: a tool waits for the owner’s approval in the chat of the call');
|
|
170
|
+
this.brain.tell(approvalNote(ask.tool, id));
|
|
171
|
+
}
|
|
172
|
+
|
|
172
173
|
exited(reason: string): void {
|
|
173
174
|
log.warn({ reason }, 'voice: the agent session for the call ended');
|
|
174
|
-
this.notes?.('');
|
|
175
|
-
this.notes = null;
|
|
176
175
|
this.ended(`agent session ${reason}`);
|
|
177
176
|
}
|
|
178
177
|
|
|
@@ -214,11 +213,10 @@ export class Talk implements Thinking {
|
|
|
214
213
|
if (this.reply !== null) this.reply.heard += chars;
|
|
215
214
|
}
|
|
216
215
|
|
|
217
|
-
|
|
218
|
-
if (this.delays.length === 0) return
|
|
216
|
+
private logReplyTime(): void {
|
|
217
|
+
if (this.delays.length === 0) return;
|
|
219
218
|
const sorted = [...this.delays].sort((a, b) => a - b);
|
|
220
|
-
|
|
221
|
-
return `Reply time, from your last word heard to my first word sent: about ${(middle / 1000).toFixed(1)} s (middle of ${String(sorted.length)}).`;
|
|
219
|
+
log.info({ middleMs: Math.round(sorted[Math.floor(sorted.length / 2)] ?? 0), replies: sorted.length }, 'voice: reply time of the call, from the caller’s last word heard to the first reply audio sent');
|
|
222
220
|
}
|
|
223
221
|
|
|
224
222
|
private firstPlayed(): void {
|
|
@@ -246,7 +244,6 @@ export class Talk implements Thinking {
|
|
|
246
244
|
}
|
|
247
245
|
|
|
248
246
|
private committed(text: string): void {
|
|
249
|
-
this.heardCaller = true;
|
|
250
247
|
if (this.interrupts(text)) this.cutOff();
|
|
251
248
|
const note = this.cut;
|
|
252
249
|
const now = this.muted;
|
|
@@ -254,22 +251,6 @@ export class Talk implements Thinking {
|
|
|
254
251
|
this.askedAt = performance.now();
|
|
255
252
|
this.saidAt = this.lastHeardAt ?? this.askedAt;
|
|
256
253
|
this.lastHeardAt = null;
|
|
257
|
-
this.busy = true;
|
|
258
254
|
this.brain.tell(note === null ? text : `${note}\n${text}`, now);
|
|
259
255
|
}
|
|
260
|
-
|
|
261
|
-
private callNotes(): Promise<string | null> {
|
|
262
|
-
return new Promise((resolve) => {
|
|
263
|
-
const timer = setTimeout(() => {
|
|
264
|
-
this.notes = null;
|
|
265
|
-
resolve(null);
|
|
266
|
-
}, NOTES_WAIT_MS);
|
|
267
|
-
this.notesStarted = false;
|
|
268
|
-
this.notes = (text) => {
|
|
269
|
-
clearTimeout(timer);
|
|
270
|
-
resolve(text.trim() === '' ? null : text.trim());
|
|
271
|
-
};
|
|
272
|
-
this.brain.tell(NOTES_PROMPT, this.busy);
|
|
273
|
-
});
|
|
274
|
-
}
|
|
275
256
|
}
|
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
import { homedir } from 'node:os';
|
|
2
2
|
import { join } from 'node:path';
|
|
3
|
-
import { mkdirSync } from 'node:fs';
|
|
4
3
|
import { TelegramClient } from '@mtcute/bun';
|
|
5
4
|
import { TrainError } from '@metro-labs/core/train-error';
|
|
6
5
|
import { errMsg } from '@metro-labs/core/log';
|
|
6
|
+
import { ensureSecureDir } from '@metro-labs/core/secure-fs';
|
|
7
7
|
import type { UserAccount } from './types.js';
|
|
8
8
|
|
|
9
9
|
const STATE_DIR =
|
|
@@ -25,7 +25,7 @@ export interface UserClient {
|
|
|
25
25
|
|
|
26
26
|
function storagePath(accountId: string): string {
|
|
27
27
|
const dir = join(STATE_DIR, 'telegram');
|
|
28
|
-
|
|
28
|
+
ensureSecureDir(dir);
|
|
29
29
|
return join(dir, `${accountId}.session`);
|
|
30
30
|
}
|
|
31
31
|
|
|
@@ -57,7 +57,7 @@ const DELETED = { text: '[deletedMessage]', contentType: 'deletedMessage' };
|
|
|
57
57
|
|
|
58
58
|
async function read(id: string, args: Args): Promise<void> {
|
|
59
59
|
const { line, limit, before } = args as { line: string; limit?: number; before?: string };
|
|
60
|
-
const { conv } = await convOf(line);
|
|
60
|
+
const { acct, conv } = await convOf(line);
|
|
61
61
|
if (!conv)
|
|
62
62
|
throw new TrainError('NOT_FOUND', `conversation not found for ${line}`);
|
|
63
63
|
const lim = Math.min(Math.max(1, limit ?? 20), 200);
|
|
@@ -73,6 +73,7 @@ async function read(id: string, args: Args): Promise<void> {
|
|
|
73
73
|
id: m.id,
|
|
74
74
|
ts: new Date(Number(m.sentAtNs / 1_000_000n)).toISOString(),
|
|
75
75
|
from: `metro://xmtp/${acctId}/user/${m.senderInboxId}`,
|
|
76
|
+
...(m.senderInboxId === acct.inboxId ? { self: true } : {}),
|
|
76
77
|
...(deleted.has(m.id) ? DELETED : { text: textOf(m), contentType: m.contentType?.typeId ?? 'unknown' }),
|
|
77
78
|
}));
|
|
78
79
|
respond(id, { result: { line, count: messages.length, messages } });
|
package/runtime/runtime.json
CHANGED