@stage-labs/metro 0.1.0-beta.263 → 0.1.0-beta.265
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli.js +2 -2
- package/dist/runtime-install.js +22 -5
- package/package.json +1 -1
- package/runtime/marketplace/plugin/.claude-plugin/plugin.json +1 -1
- package/runtime/marketplace/plugin/bin/policy.mjs +1 -1
- package/runtime/node_modules/@metro-labs/core/package.json +1 -0
- package/runtime/node_modules/@metro-labs/core/src/call.ts +81 -0
- package/runtime/node_modules/@metro-labs/core/src/runner-activity.ts +37 -1
- package/runtime/node_modules/@metro-labs/core/src/stations/train-boot.ts +22 -1
- package/runtime/node_modules/@metro-labs/daemon/src/approvals/pending.ts +35 -2
- package/runtime/node_modules/@metro-labs/daemon/src/boot/boot.ts +1 -1
- package/runtime/node_modules/@metro-labs/daemon/src/channels/inbound.ts +14 -11
- package/runtime/node_modules/@metro-labs/daemon/src/channels/pending.ts +3 -0
- package/runtime/node_modules/@metro-labs/daemon/src/mcp/call-bridge.ts +45 -0
- package/runtime/node_modules/@metro-labs/daemon/src/mcp/call-permission.ts +25 -0
- package/runtime/node_modules/@metro-labs/daemon/src/mcp/call-tools.ts +2 -0
- package/runtime/node_modules/@metro-labs/daemon/src/mcp/event-store.ts +9 -2
- package/runtime/node_modules/@metro-labs/daemon/src/mcp/index.ts +2 -0
- package/runtime/node_modules/@metro-labs/daemon/src/mcp/permission-prompt.ts +2 -1
- package/runtime/node_modules/@metro-labs/daemon/src/mcp/permission-relay.ts +15 -4
- package/runtime/node_modules/@metro-labs/daemon/src/mcp/policy-gate.ts +3 -3
- package/runtime/node_modules/@metro-labs/daemon/src/mcp/session.ts +20 -1
- package/runtime/node_modules/@metro-labs/daemon/src/mcp/speech-schema.ts +11 -0
- package/runtime/node_modules/@metro-labs/daemon/src/mcp/speech-send.ts +15 -0
- package/runtime/node_modules/@metro-labs/daemon/src/mcp/tool-schemas.ts +3 -1
- package/runtime/node_modules/@metro-labs/daemon/src/stations/runtime-deps.ts +23 -7
- package/runtime/node_modules/@metro-labs/daemon/src/voice/audio.ts +50 -10
- package/runtime/node_modules/@metro-labs/daemon/src/voice/brain.ts +20 -9
- package/runtime/node_modules/@metro-labs/daemon/src/voice/call.ts +125 -51
- package/runtime/node_modules/@metro-labs/daemon/src/voice/calls.ts +41 -15
- package/runtime/node_modules/@metro-labs/daemon/src/voice/peer.ts +9 -8
- package/runtime/node_modules/@metro-labs/daemon/src/voice/scribe.ts +15 -3
- package/runtime/node_modules/@metro-labs/daemon/src/voice/shared-talk.ts +85 -0
- package/runtime/node_modules/@metro-labs/daemon/src/voice/shared.ts +199 -0
- package/runtime/node_modules/@metro-labs/daemon/src/voice/speech-queue.ts +173 -0
- package/runtime/node_modules/@metro-labs/daemon/src/voice/speech.ts +40 -15
- package/runtime/node_modules/@metro-labs/daemon/src/voice/talk.ts +15 -7
- package/runtime/node_modules/@metro-labs/whatsapp/src/actions.ts +32 -1
- package/runtime/node_modules/@metro-labs/whatsapp/src/app-state.ts +49 -0
- package/runtime/node_modules/@metro-labs/whatsapp/src/client-history.ts +31 -0
- package/runtime/node_modules/@metro-labs/whatsapp/src/client-types.ts +46 -0
- package/runtime/node_modules/@metro-labs/whatsapp/src/client.ts +75 -77
- package/runtime/node_modules/@metro-labs/whatsapp/src/history-events.ts +127 -0
- package/runtime/node_modules/@metro-labs/whatsapp/src/history-files.ts +39 -0
- package/runtime/node_modules/@metro-labs/whatsapp/src/history-identity.ts +58 -0
- package/runtime/node_modules/@metro-labs/whatsapp/src/history-message.ts +161 -0
- package/runtime/node_modules/@metro-labs/whatsapp/src/history-state.ts +344 -0
- package/runtime/node_modules/@metro-labs/whatsapp/src/history-types.ts +91 -0
- package/runtime/node_modules/@metro-labs/whatsapp/src/history.ts +195 -0
- package/runtime/node_modules/@metro-labs/whatsapp/src/index.ts +1 -1
- package/runtime/node_modules/@metro-labs/whatsapp/src/members.ts +94 -0
- package/runtime/node_modules/@metro-labs/whatsapp/src/messages.ts +70 -0
- package/runtime/node_modules/@metro-labs/whatsapp/src/send-expiry.ts +65 -0
- package/runtime/node_modules/@metro-labs/whatsapp/src/station.ts +4 -0
- package/runtime/patches/5bca2c07d5852d3f58453cff0448dd85da13d3665b61e3d758905b298d432d7c.patch +31 -0
- package/runtime/runtime.json +1 -1
- package/runtime/sdk-runner/node_modules/@metro-labs/core/package.json +1 -0
- package/runtime/sdk-runner/node_modules/@metro-labs/core/src/call.ts +81 -0
- package/runtime/sdk-runner/node_modules/@metro-labs/core/src/runner-activity.ts +37 -1
- package/runtime/sdk-runner/node_modules/@metro-labs/core/src/stations/train-boot.ts +22 -1
- package/runtime/sdk-runner/runner.json +1 -1
- package/runtime/sdk-runner/src/activity.ts +14 -2
- package/runtime/sdk-runner/src/app.ts +5 -4
- package/runtime/sdk-runner/src/approvals.ts +14 -5
- package/runtime/sdk-runner/src/calls.ts +133 -0
- package/runtime/sdk-runner/src/inbox.ts +58 -26
- package/runtime/sdk-runner/src/input-queue.ts +70 -0
- package/runtime/sdk-runner/src/input-timing.ts +68 -0
- package/runtime/sdk-runner/src/link.ts +27 -4
- package/runtime/sdk-runner/src/main.ts +0 -4
- package/runtime/sdk-runner/src/rules.ts +19 -5
- package/runtime/sdk-runner/src/runner.ts +64 -88
- package/runtime/sdk-runner/src/session-store.ts +14 -1
- package/runtime/sdk-runner/src/session-watch.ts +27 -20
- package/runtime/stations.json +3 -0
- package/runtime/sdk-runner/src/speech.ts +0 -79
|
@@ -0,0 +1,199 @@
|
|
|
1
|
+
import { randomUUID } from 'node:crypto';
|
|
2
|
+
import { sameCall, type CallNotice, type CallRoute, type CallSource, type SpeechStatus, type SpeechTarget } from '@metro-labs/core/call';
|
|
3
|
+
import { str } from '@metro-labs/core/str';
|
|
4
|
+
import { log } from '@metro-labs/core/log';
|
|
5
|
+
import { agentIdForLine, allowlistForLine, lineReceives, mayApprove, senderPermitted, accountFromLine } from '../agents/map.js';
|
|
6
|
+
import { eventInScope } from '../agents/scope.js';
|
|
7
|
+
import { harnessRunner } from '../claude/runner.js';
|
|
8
|
+
import { policyFor, type Access } from '../policy/policy.js';
|
|
9
|
+
|
|
10
|
+
export interface SharedSpeechAction {
|
|
11
|
+
actionId: string;
|
|
12
|
+
text: string;
|
|
13
|
+
isValid(): boolean;
|
|
14
|
+
status(status: SpeechStatus): void;
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
export interface SharedCallTransport {
|
|
18
|
+
enqueue(action: SharedSpeechAction): boolean;
|
|
19
|
+
terminate(): void;
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
export interface SpeechReceipt {
|
|
23
|
+
actionId: string;
|
|
24
|
+
sourceId: string;
|
|
25
|
+
status: SpeechStatus;
|
|
26
|
+
duplicate: boolean;
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
interface Action extends SpeechReceipt {
|
|
30
|
+
text: string;
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
interface Active {
|
|
34
|
+
route: CallRoute;
|
|
35
|
+
at: number;
|
|
36
|
+
transport: SharedCallTransport;
|
|
37
|
+
sources: Set<string>;
|
|
38
|
+
actions: Map<string, Action>;
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
export interface SharedCallBridge {
|
|
42
|
+
available(route: CallRoute): boolean;
|
|
43
|
+
notify(notice: CallNotice): boolean;
|
|
44
|
+
revoked(route: CallRoute): void;
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
const SOURCES_MAX = 1024;
|
|
48
|
+
const TEXT_MAX = 32_000;
|
|
49
|
+
const sourceValid = (sourceId: string): boolean => sourceId.length > 0 && sourceId.length <= 2048;
|
|
50
|
+
|
|
51
|
+
const authorized = (route: CallRoute): boolean =>
|
|
52
|
+
agentIdForLine(route.line) === route.agentId && eventInScope(new Set([route.agentId]), route.line) && lineReceives(route.line) &&
|
|
53
|
+
mayApprove(route.line, route.from) && senderPermitted(allowlistForLine(route.line), route.from);
|
|
54
|
+
|
|
55
|
+
function access(route: CallRoute): Access {
|
|
56
|
+
const account = accountFromLine(route.line);
|
|
57
|
+
return account === undefined ? 'deny' : policyFor({ kind: 'channel', station: account.station, account: account.accountId }, { name: 'send', group: 'write' });
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
export class SharedCalls {
|
|
61
|
+
private active: Active | null = null;
|
|
62
|
+
private bridge: SharedCallBridge | null = null;
|
|
63
|
+
|
|
64
|
+
constructor(private readonly permitted = authorized, private readonly policy = access) {}
|
|
65
|
+
|
|
66
|
+
connect(bridge: SharedCallBridge): void {
|
|
67
|
+
this.disconnect();
|
|
68
|
+
this.bridge = bridge;
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
disconnect(): void {
|
|
72
|
+
this.revoke();
|
|
73
|
+
this.bridge = null;
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
revoke(): void {
|
|
77
|
+
const active = this.active;
|
|
78
|
+
if (active !== null) this.end(active.route);
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
open(route: CallRoute, sourceId: string, transport: SharedCallTransport): boolean {
|
|
82
|
+
if (!this.permitted(route) || this.bridge?.available(route) !== true || !sourceValid(sourceId)) return false;
|
|
83
|
+
if (this.active !== null && sameCall(this.active.route, route)) return this.active.transport === transport && this.active.sources.has(sourceId);
|
|
84
|
+
if (this.active !== null) this.end(this.active.route);
|
|
85
|
+
this.active = { route, at: Date.now(), transport, sources: new Set([sourceId]), actions: new Map() };
|
|
86
|
+
if (this.bridge.notify({ type: 'started', route, sourceId })) return true;
|
|
87
|
+
this.end(route);
|
|
88
|
+
return false;
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
snapshot(): CallRoute | null {
|
|
92
|
+
const route = this.active?.route;
|
|
93
|
+
return route !== undefined && this.current(route) !== null ? route : null;
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
valid(source: CallSource): boolean {
|
|
97
|
+
return this.current(source.route)?.sources.has(source.sourceId) === true;
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
private current(route: CallRoute): Active | null {
|
|
101
|
+
const active = this.active;
|
|
102
|
+
return active !== null && sameCall(active.route, route) && this.permitted(route) && this.bridge?.available(route) === true ? active : null;
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
heard(route: CallRoute, text: string, sourceId: string): void {
|
|
106
|
+
if (text.trim() === '' || text.length > TEXT_MAX || !this.addSource(route, sourceId)) return;
|
|
107
|
+
if (this.bridge?.notify({ type: 'heard', route, text, sourceId }) !== true) this.end(route);
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
chat(event: Record<string, unknown>, scope: Set<string>): Record<string, string> {
|
|
111
|
+
const active = this.active;
|
|
112
|
+
if (active === null || event.senderVerified === false || !scope.has(active.route.agentId)) return {};
|
|
113
|
+
const { route } = active;
|
|
114
|
+
if (event.line !== route.line || event.from !== route.from || Date.parse(str(event.ts)) < active.at || !Number.isFinite(Date.parse(str(event.ts)))) return {};
|
|
115
|
+
const sourceId = str(event.messageId);
|
|
116
|
+
if (!this.addSource(route, sourceId, true)) return {};
|
|
117
|
+
return { call_id: route.callId, call_generation: route.generation, call_source_id: sourceId };
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
bindChat(event: Record<string, unknown>, scope: Set<string>): () => Record<string, string> {
|
|
121
|
+
const meta = this.chat(event, scope);
|
|
122
|
+
const route = this.active?.route;
|
|
123
|
+
const sourceId = meta.call_source_id;
|
|
124
|
+
return () => route !== undefined && sourceId !== undefined && this.valid({ route, sourceId }) ? meta : {};
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
source(line: string, target: SpeechTarget, scope: Set<string>): CallSource | null {
|
|
128
|
+
const route = this.active?.route;
|
|
129
|
+
if (route === undefined || route.line !== line || route.callId !== target.callId || route.generation !== target.generation || !scope.has(route.agentId)) return null;
|
|
130
|
+
const source = { route, sourceId: target.sourceId };
|
|
131
|
+
return this.valid(source) ? source : null;
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
speak(line: string, target: SpeechTarget, text: string, scope: Set<string>): SpeechReceipt {
|
|
135
|
+
const source = this.source(line, target, scope);
|
|
136
|
+
const active = this.active;
|
|
137
|
+
if (source === null || active === null) throw new Error('The call or its authorized source has ended or changed. Nothing was spoken.');
|
|
138
|
+
if (text.trim() === '' || text.length > TEXT_MAX) throw new Error('Call speech needs 1 to 32000 characters of text.');
|
|
139
|
+
const previous = active.actions.get(source.sourceId);
|
|
140
|
+
if (previous !== undefined) {
|
|
141
|
+
if (previous.text !== text) throw new Error('This source already has a different speech action. Nothing was repeated.');
|
|
142
|
+
return { actionId: previous.actionId, sourceId: previous.sourceId, status: previous.status, duplicate: true };
|
|
143
|
+
}
|
|
144
|
+
const accessAtAcceptance = this.policy(source.route);
|
|
145
|
+
if (accessAtAcceptance === 'deny') throw new Error('The owner blocked sending on this channel.');
|
|
146
|
+
const action: Action = { actionId: randomUUID(), sourceId: source.sourceId, text, status: 'accepted', duplicate: false };
|
|
147
|
+
active.actions.set(source.sourceId, action);
|
|
148
|
+
this.status(source.route, action, 'accepted');
|
|
149
|
+
const queued = active.transport.enqueue({
|
|
150
|
+
actionId: action.actionId,
|
|
151
|
+
text,
|
|
152
|
+
isValid: () => this.valid(source) && this.policy(source.route) === accessAtAcceptance,
|
|
153
|
+
status: (status) => { this.status(source.route, action, status); },
|
|
154
|
+
});
|
|
155
|
+
if (!queued) this.status(source.route, action, 'failed');
|
|
156
|
+
else if (action.status === 'accepted') this.status(source.route, action, 'queued');
|
|
157
|
+
return { actionId: action.actionId, sourceId: action.sourceId, status: action.status, duplicate: false };
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
end(route: CallRoute): void {
|
|
161
|
+
const active = this.active;
|
|
162
|
+
if (active === null || !sameCall(active.route, route)) return;
|
|
163
|
+
this.active = null;
|
|
164
|
+
this.bridge?.revoked(route);
|
|
165
|
+
active.transport.terminate();
|
|
166
|
+
for (const action of active.actions.values())
|
|
167
|
+
if (!this.terminal(action.status)) this.status(route, action, 'interrupted');
|
|
168
|
+
this.bridge?.notify({ type: 'ended', route });
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
private addSource(route: CallRoute, sourceId: string, duplicate = false): boolean {
|
|
172
|
+
const active = this.current(route);
|
|
173
|
+
if (active === null || !sourceValid(sourceId)) return false;
|
|
174
|
+
if (active.sources.has(sourceId)) return duplicate;
|
|
175
|
+
if (active.sources.size >= SOURCES_MAX) {
|
|
176
|
+
log.warn('sdk-call: the call source limit was reached; ending the call');
|
|
177
|
+
this.end(route);
|
|
178
|
+
return false;
|
|
179
|
+
}
|
|
180
|
+
active.sources.add(sourceId);
|
|
181
|
+
return true;
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
private terminal(status: SpeechStatus): boolean {
|
|
185
|
+
return status === 'completed' || status === 'interrupted' || status === 'failed';
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
private status(route: CallRoute, action: Action, status: SpeechStatus): void {
|
|
189
|
+
if (this.terminal(action.status) || (status === 'started' && action.status === 'started')) return;
|
|
190
|
+
action.status = status;
|
|
191
|
+
this.bridge?.notify({ type: 'speech', route, sourceId: action.sourceId, actionId: action.actionId, status });
|
|
192
|
+
}
|
|
193
|
+
}
|
|
194
|
+
|
|
195
|
+
export const sharedCalls = new SharedCalls();
|
|
196
|
+
export const sharedCallsSelected = (): boolean => harnessRunner() === 'sdk';
|
|
197
|
+
export const openSharedCall = (route: CallRoute, sourceId: string, transport: SharedCallTransport): boolean => sharedCalls.open(route, sourceId, transport);
|
|
198
|
+
export const hearSharedCall = (route: CallRoute, text: string, sourceId: string): void => { sharedCalls.heard(route, text, sourceId); };
|
|
199
|
+
export const endSharedCall = (route: CallRoute): void => { sharedCalls.end(route); };
|
|
@@ -0,0 +1,173 @@
|
|
|
1
|
+
import type { SpeechStatus } from '@metro-labs/core/call';
|
|
2
|
+
import { errMsg, log } from '@metro-labs/core/log';
|
|
3
|
+
import { Player, type Playback } from './audio.js';
|
|
4
|
+
import { Utterance } from './speech.js';
|
|
5
|
+
import type { SharedSpeechAction } from './shared.js';
|
|
6
|
+
import { voiceIdOf, type VoiceConfig } from './store.js';
|
|
7
|
+
|
|
8
|
+
const MAX_QUEUED = 8;
|
|
9
|
+
const MAX_TEXT = 32_000;
|
|
10
|
+
const SPEECH_MS = 60_000;
|
|
11
|
+
|
|
12
|
+
type Speech = Pick<Utterance, 'attach' | 'say' | 'end' | 'abort'>;
|
|
13
|
+
type MakeSpeech = (failed: (reason: string) => void) => Speech;
|
|
14
|
+
|
|
15
|
+
interface Playing {
|
|
16
|
+
action: SharedSpeechAction;
|
|
17
|
+
utterance: Speech | null;
|
|
18
|
+
timer: ReturnType<typeof setTimeout>;
|
|
19
|
+
synthesized: boolean;
|
|
20
|
+
started: boolean;
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
export class SpeechQueue {
|
|
24
|
+
private readonly player: Player;
|
|
25
|
+
private readonly waiting: SharedSpeechAction[] = [];
|
|
26
|
+
private current: Playing | null = null;
|
|
27
|
+
private live = false;
|
|
28
|
+
private closed = false;
|
|
29
|
+
private onFailed: () => void = () => undefined;
|
|
30
|
+
|
|
31
|
+
constructor(
|
|
32
|
+
cfg: VoiceConfig,
|
|
33
|
+
send: Playback['send'],
|
|
34
|
+
private readonly makeSpeech: MakeSpeech = (failed) => new Utterance(cfg.apiKey, voiceIdOf(cfg), failed),
|
|
35
|
+
private readonly timeoutMs = SPEECH_MS,
|
|
36
|
+
) {
|
|
37
|
+
this.player = new Player({
|
|
38
|
+
send: (opus, timestamp, marker) => {
|
|
39
|
+
if (this.current !== null && !this.valid(this.current)) return;
|
|
40
|
+
return send(opus, timestamp, marker);
|
|
41
|
+
},
|
|
42
|
+
played: () => undefined,
|
|
43
|
+
});
|
|
44
|
+
this.player.whenFirstPlayed(() => {
|
|
45
|
+
const current = this.current;
|
|
46
|
+
if (current === null || current.started) return;
|
|
47
|
+
current.started = true;
|
|
48
|
+
current.action.status('started');
|
|
49
|
+
});
|
|
50
|
+
this.player.whenIdle(() => {
|
|
51
|
+
this.drained();
|
|
52
|
+
});
|
|
53
|
+
this.player.whenFailed((reason) => {
|
|
54
|
+
this.fail(reason);
|
|
55
|
+
});
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
get busy(): boolean {
|
|
59
|
+
return this.player.busy;
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
whenFailed(failed: () => void): void {
|
|
63
|
+
this.onFailed = failed;
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
enqueue(action: SharedSpeechAction): boolean {
|
|
67
|
+
if (this.closed || action.text.trim() === '' || action.text.length > MAX_TEXT) return false;
|
|
68
|
+
if (this.waiting.length + (this.current === null ? 0 : 1) >= MAX_QUEUED || !action.isValid()) return false;
|
|
69
|
+
this.waiting.push(action);
|
|
70
|
+
queueMicrotask(() => {
|
|
71
|
+
this.next();
|
|
72
|
+
});
|
|
73
|
+
return true;
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
connect(): void {
|
|
77
|
+
if (this.closed || this.live) return;
|
|
78
|
+
this.live = true;
|
|
79
|
+
this.player.start();
|
|
80
|
+
this.next();
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
cancel(): void {
|
|
84
|
+
const waiting = this.waiting.splice(0);
|
|
85
|
+
this.settle('interrupted');
|
|
86
|
+
for (const action of waiting) action.status('interrupted');
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
close(): void {
|
|
90
|
+
if (this.closed) return;
|
|
91
|
+
this.closed = true;
|
|
92
|
+
this.live = false;
|
|
93
|
+
this.cancel();
|
|
94
|
+
this.player.stop();
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
private next(): void {
|
|
98
|
+
if (!this.live || this.closed || this.current !== null) return;
|
|
99
|
+
const action = this.waiting.shift();
|
|
100
|
+
if (action === undefined) return;
|
|
101
|
+
if (!action.isValid()) {
|
|
102
|
+
action.status('failed');
|
|
103
|
+
this.next();
|
|
104
|
+
return;
|
|
105
|
+
}
|
|
106
|
+
const playing: Playing = {
|
|
107
|
+
action, utterance: null, synthesized: false, started: false,
|
|
108
|
+
timer: setTimeout(() => {
|
|
109
|
+
if (this.current === playing) this.fail('speech timed out');
|
|
110
|
+
}, this.timeoutMs),
|
|
111
|
+
};
|
|
112
|
+
this.current = playing;
|
|
113
|
+
try {
|
|
114
|
+
const utterance = this.makeSpeech((reason) => {
|
|
115
|
+
if (this.current === playing) this.fail(reason);
|
|
116
|
+
});
|
|
117
|
+
if (this.current !== playing) {
|
|
118
|
+
utterance.abort();
|
|
119
|
+
return;
|
|
120
|
+
}
|
|
121
|
+
playing.utterance = utterance;
|
|
122
|
+
utterance.attach({
|
|
123
|
+
audio: (pcm, chars) => {
|
|
124
|
+
if (this.valid(playing)) this.player.push(pcm, chars);
|
|
125
|
+
},
|
|
126
|
+
done: () => {
|
|
127
|
+
if (!this.valid(playing)) return;
|
|
128
|
+
playing.synthesized = true;
|
|
129
|
+
this.player.finish();
|
|
130
|
+
this.drained();
|
|
131
|
+
},
|
|
132
|
+
});
|
|
133
|
+
utterance.say(action.text);
|
|
134
|
+
utterance.end();
|
|
135
|
+
} catch (err) {
|
|
136
|
+
if (this.current === playing) this.fail(errMsg(err));
|
|
137
|
+
}
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
private valid(playing: Playing): boolean {
|
|
141
|
+
if (this.closed || this.current !== playing) return false;
|
|
142
|
+
if (playing.action.isValid()) return true;
|
|
143
|
+
this.settle('failed');
|
|
144
|
+
return false;
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
private drained(): void {
|
|
148
|
+
const current = this.current;
|
|
149
|
+
if (current === null || !current.synthesized || this.player.busy) return;
|
|
150
|
+
if (current.started) this.settle('completed');
|
|
151
|
+
else this.fail('no audio received');
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
private fail(reason: string): void {
|
|
155
|
+
log.warn({ reason }, 'voice: queued speech failed');
|
|
156
|
+
this.settle('failed');
|
|
157
|
+
this.onFailed();
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
private settle(status: SpeechStatus): void {
|
|
161
|
+
const current = this.current;
|
|
162
|
+
this.current = null;
|
|
163
|
+
this.player.clear();
|
|
164
|
+
if (current !== null) {
|
|
165
|
+
clearTimeout(current.timer);
|
|
166
|
+
current.utterance?.abort();
|
|
167
|
+
current.action.status(status);
|
|
168
|
+
}
|
|
169
|
+
queueMicrotask(() => {
|
|
170
|
+
this.next();
|
|
171
|
+
});
|
|
172
|
+
}
|
|
173
|
+
}
|
|
@@ -6,7 +6,7 @@ import { elevenLabs, parsed, wsText } from './scribe.js';
|
|
|
6
6
|
|
|
7
7
|
const TTS_MODEL = 'eleven_flash_v2_5';
|
|
8
8
|
|
|
9
|
-
interface Voiced {
|
|
9
|
+
export interface Voiced {
|
|
10
10
|
audio(pcm48k: Int16Array, chars: number): void;
|
|
11
11
|
done(): void;
|
|
12
12
|
}
|
|
@@ -37,10 +37,12 @@ export class Utterance {
|
|
|
37
37
|
private aborted = false;
|
|
38
38
|
private voiced = false;
|
|
39
39
|
private refusal: string | null = null;
|
|
40
|
+
private finished = false;
|
|
40
41
|
|
|
41
|
-
constructor(apiKey: string, voiceId: string, failed: (reason: string) => void) {
|
|
42
|
+
constructor(apiKey: string, voiceId: string, private readonly failed: (reason: string) => void) {
|
|
42
43
|
this.ws = new WebSocket(ttsUrl(voiceId), { headers: { 'xi-api-key': apiKey } });
|
|
43
44
|
this.ws.on('open', () => {
|
|
45
|
+
if (this.aborted) return;
|
|
44
46
|
this.ws.send(JSON.stringify({ text: ' ' }));
|
|
45
47
|
for (const line of this.outbox.splice(0)) this.ws.send(line);
|
|
46
48
|
});
|
|
@@ -48,15 +50,14 @@ export class Utterance {
|
|
|
48
50
|
this.onMessage(wsText(data));
|
|
49
51
|
});
|
|
50
52
|
this.ws.on('error', (err) => {
|
|
53
|
+
if (this.aborted || this.finished) return;
|
|
51
54
|
this.refusal ??= errMsg(err);
|
|
52
55
|
log.warn({ err: errMsg(err) }, 'voice: text to speech connection error');
|
|
53
56
|
});
|
|
54
57
|
this.ws.on('close', (code, why) => {
|
|
55
58
|
this.closed = true;
|
|
56
|
-
this.listener?.done();
|
|
57
|
-
this.listener = null;
|
|
58
59
|
const reason = this.refusal ?? (code === 1000 ? null : (why.toString() || `closed (${String(code)})`));
|
|
59
|
-
|
|
60
|
+
this.complete(reason);
|
|
60
61
|
});
|
|
61
62
|
}
|
|
62
63
|
|
|
@@ -65,7 +66,7 @@ export class Utterance {
|
|
|
65
66
|
}
|
|
66
67
|
|
|
67
68
|
attach(listener: Voiced): void {
|
|
68
|
-
this.listener = listener;
|
|
69
|
+
if (!this.aborted && !this.finished) this.listener = listener;
|
|
69
70
|
}
|
|
70
71
|
|
|
71
72
|
say(text: string): void {
|
|
@@ -77,28 +78,52 @@ export class Utterance {
|
|
|
77
78
|
}
|
|
78
79
|
|
|
79
80
|
abort(): void {
|
|
81
|
+
if (this.aborted) return;
|
|
80
82
|
this.aborted = true;
|
|
81
83
|
this.listener = null;
|
|
84
|
+
this.outbox.length = 0;
|
|
82
85
|
this.ws.terminate();
|
|
83
86
|
}
|
|
84
87
|
|
|
88
|
+
private complete(reason: string | null): void {
|
|
89
|
+
if (this.aborted || this.finished) return;
|
|
90
|
+
this.finished = true;
|
|
91
|
+
const listener = this.listener;
|
|
92
|
+
this.listener = null;
|
|
93
|
+
this.outbox.length = 0;
|
|
94
|
+
const failure = reason ?? (listener !== null && !this.voiced ? 'no audio received' : null);
|
|
95
|
+
if (failure !== null) this.failed(failure.slice(0, MAX_REASON));
|
|
96
|
+
else listener?.done();
|
|
97
|
+
}
|
|
98
|
+
|
|
85
99
|
private post(msg: Record<string, unknown>): void {
|
|
100
|
+
if (this.aborted || this.closed || this.finished) return;
|
|
86
101
|
const line = JSON.stringify(msg);
|
|
87
102
|
if (this.ws.readyState === WebSocket.OPEN) this.ws.send(line);
|
|
88
103
|
else this.outbox.push(line);
|
|
89
104
|
}
|
|
90
105
|
|
|
106
|
+
private refused(msg: Record<string, unknown>, raw: string): void {
|
|
107
|
+
if (typeof msg.message === 'string' && msg.message !== '') this.refusal ??= msg.message;
|
|
108
|
+
log.warn({ msg: raw.slice(0, MAX_REASON) }, 'voice: text to speech said');
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
private audio(encoded: string, chars: number): void {
|
|
112
|
+
this.voiced = true;
|
|
113
|
+
const pcm = pcmOf(Buffer.from(encoded, 'base64'));
|
|
114
|
+
const up = upsample2(pcm, this.carry);
|
|
115
|
+
this.carry = pcm.at(-1) ?? this.carry;
|
|
116
|
+
this.listener?.audio(up, chars);
|
|
117
|
+
}
|
|
118
|
+
|
|
91
119
|
private onMessage(raw: string): void {
|
|
120
|
+
if (this.aborted || this.finished) return;
|
|
92
121
|
const msg = parsed(raw);
|
|
93
|
-
if (typeof msg.audio === 'string' && msg.audio !== '')
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
this.
|
|
98
|
-
this.listener?.audio(up, charsOf(msg));
|
|
99
|
-
} else if (msg.isFinal !== true) {
|
|
100
|
-
if (typeof msg.message === 'string' && msg.message !== '') this.refusal ??= msg.message;
|
|
101
|
-
log.warn({ msg: raw.slice(0, MAX_REASON) }, 'voice: text to speech said');
|
|
122
|
+
if (typeof msg.audio === 'string' && msg.audio !== '') this.audio(msg.audio, charsOf(msg));
|
|
123
|
+
else if (msg.isFinal !== true) this.refused(msg, raw);
|
|
124
|
+
if (this.refusal !== null || msg.isFinal === true) {
|
|
125
|
+
this.complete(this.refusal);
|
|
126
|
+
this.ws.close(1000);
|
|
102
127
|
}
|
|
103
128
|
}
|
|
104
129
|
}
|
|
@@ -58,7 +58,7 @@ export class Talk implements Thinking {
|
|
|
58
58
|
|
|
59
59
|
constructor(
|
|
60
60
|
private readonly cfg: VoiceConfig,
|
|
61
|
-
out:
|
|
61
|
+
out: Playback['send'],
|
|
62
62
|
private readonly ended: (reason: string) => void,
|
|
63
63
|
private readonly approvals: CallApprovals,
|
|
64
64
|
) {
|
|
@@ -72,6 +72,9 @@ export class Talk implements Thinking {
|
|
|
72
72
|
this.player.whenFirstPlayed(() => {
|
|
73
73
|
this.firstPlayed();
|
|
74
74
|
});
|
|
75
|
+
this.player.whenFailed((reason) => {
|
|
76
|
+
if (!this.finished) this.ended(`audio playback failed: ${reason}`);
|
|
77
|
+
});
|
|
75
78
|
this.brain = new Brain(brainModel(cfg), this);
|
|
76
79
|
this.scribe = new Scribe(cfg.apiKey, languageOf(cfg), {
|
|
77
80
|
partial: (text) => {
|
|
@@ -88,10 +91,11 @@ export class Talk implements Thinking {
|
|
|
88
91
|
}
|
|
89
92
|
|
|
90
93
|
prime(context: string): void {
|
|
91
|
-
this.brain.tell(context);
|
|
94
|
+
if (!this.finished) this.brain.tell(context);
|
|
92
95
|
}
|
|
93
96
|
|
|
94
97
|
connect(): void {
|
|
98
|
+
if (this.finished || this.live) return;
|
|
95
99
|
this.live = true;
|
|
96
100
|
this.spare ??= this.utterance();
|
|
97
101
|
this.player.start();
|
|
@@ -118,8 +122,8 @@ export class Talk implements Thinking {
|
|
|
118
122
|
}
|
|
119
123
|
|
|
120
124
|
turnStarted(): void {
|
|
121
|
-
this.muted = false;
|
|
122
125
|
if (this.finished) return;
|
|
126
|
+
this.muted = false;
|
|
123
127
|
this.reply = {
|
|
124
128
|
utterance: this.takeUtterance(),
|
|
125
129
|
said: '',
|
|
@@ -137,19 +141,20 @@ export class Talk implements Thinking {
|
|
|
137
141
|
|
|
138
142
|
text(delta: string): void {
|
|
139
143
|
const reply = this.reply;
|
|
140
|
-
if (this.muted || reply === null) return;
|
|
144
|
+
if (this.finished || this.muted || reply === null) return;
|
|
141
145
|
reply.firstText ??= performance.now();
|
|
142
146
|
this.speak(reply, reply.chunker.add(delta));
|
|
143
147
|
}
|
|
144
148
|
|
|
145
149
|
tool(name: string): void {
|
|
150
|
+
if (this.finished) return;
|
|
146
151
|
log.info({ tool: name }, 'voice: the agent uses a tool during the call');
|
|
147
152
|
if (this.reply !== null && !this.muted) this.speak(this.reply, this.reply.chunker.flush());
|
|
148
153
|
}
|
|
149
154
|
|
|
150
155
|
turnEnded(): void {
|
|
151
156
|
const reply = this.reply;
|
|
152
|
-
if (reply === null || this.muted) return;
|
|
157
|
+
if (this.finished || reply === null || this.muted) return;
|
|
153
158
|
this.speak(reply, reply.chunker.flush());
|
|
154
159
|
reply.utterance.end();
|
|
155
160
|
log.info(
|
|
@@ -171,6 +176,7 @@ export class Talk implements Thinking {
|
|
|
171
176
|
}
|
|
172
177
|
|
|
173
178
|
exited(reason: string): void {
|
|
179
|
+
if (this.finished) return;
|
|
174
180
|
log.warn({ reason }, 'voice: the agent session for the call ended');
|
|
175
181
|
this.ended(`agent session ${reason}`);
|
|
176
182
|
}
|
|
@@ -183,13 +189,13 @@ export class Talk implements Thinking {
|
|
|
183
189
|
ready.attach({
|
|
184
190
|
audio: (pcm, chars) => {
|
|
185
191
|
const current = reply();
|
|
186
|
-
if (current?.utterance !== ready) return;
|
|
192
|
+
if (this.finished || this.muted || current?.utterance !== ready) return;
|
|
187
193
|
this.spoke = true;
|
|
188
194
|
current.firstAudio ??= performance.now();
|
|
189
195
|
this.player.push(pcm, chars);
|
|
190
196
|
},
|
|
191
197
|
done: () => {
|
|
192
|
-
this.player.finish();
|
|
198
|
+
if (!this.finished && !this.muted && this.reply?.utterance === ready) this.player.finish();
|
|
193
199
|
},
|
|
194
200
|
});
|
|
195
201
|
return ready;
|
|
@@ -239,11 +245,13 @@ export class Talk implements Thinking {
|
|
|
239
245
|
}
|
|
240
246
|
|
|
241
247
|
private partial(text: string): void {
|
|
248
|
+
if (this.finished) return;
|
|
242
249
|
if (text !== '') this.lastHeardAt = performance.now();
|
|
243
250
|
if (this.interrupts(text)) this.cutOff();
|
|
244
251
|
}
|
|
245
252
|
|
|
246
253
|
private committed(text: string): void {
|
|
254
|
+
if (this.finished) return;
|
|
247
255
|
if (this.interrupts(text)) this.cutOff();
|
|
248
256
|
const note = this.cut;
|
|
249
257
|
const now = this.muted;
|
|
@@ -8,7 +8,7 @@ import {
|
|
|
8
8
|
type StationHandler,
|
|
9
9
|
} from '@metro-labs/core/stations/station-runtime';
|
|
10
10
|
import { messagingAliases } from '@metro-labs/core/stations/messaging-normalize';
|
|
11
|
-
import { accountFor, accounts, targetOf } from './accounts.js';
|
|
11
|
+
import { accountFor, accounts, lineOf, targetOf } from './accounts.js';
|
|
12
12
|
import { phoneNumberOf, senderLookup } from './resolve.js';
|
|
13
13
|
import { assertImage, fieldsOf, parseProfileChange, type ProfileApplied } from '@metro-labs/core/stations/profile';
|
|
14
14
|
import type { WAClient } from './client.js';
|
|
@@ -117,6 +117,35 @@ function makeSend(clientFor: ClientFor): StationHandler {
|
|
|
117
117
|
};
|
|
118
118
|
}
|
|
119
119
|
|
|
120
|
+
function makeRead(clientFor: ClientFor): StationHandler {
|
|
121
|
+
return async (id, args) => {
|
|
122
|
+
const { accountId, client, jid } = resolve(args, clientFor);
|
|
123
|
+
const page = await guard(() => client.read(jid, {
|
|
124
|
+
limit: typeof args.limit === 'number' ? args.limit : undefined,
|
|
125
|
+
before: str(args.before),
|
|
126
|
+
since: str(args.since),
|
|
127
|
+
}));
|
|
128
|
+
const messages = page.messages.map((row) => ({
|
|
129
|
+
id: row.messageId,
|
|
130
|
+
ts: row.timestamp,
|
|
131
|
+
...(row.senderJid ? { from: lineOf(accountId, `user/${row.senderJid}`) } : {}),
|
|
132
|
+
self: row.fromMe,
|
|
133
|
+
text: row.text,
|
|
134
|
+
...(row.truncated ? { truncated: true } : {}),
|
|
135
|
+
...(row.attachments ? { attachments: row.attachments } : {}),
|
|
136
|
+
}));
|
|
137
|
+
respond(id, { result: { ...page, line: lineOf(accountId, jid), account: accountId, messages } });
|
|
138
|
+
};
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
function makeListMembers(clientFor: ClientFor): StationHandler {
|
|
142
|
+
return async (id, args) => {
|
|
143
|
+
const { client, jid } = resolve({ line: args.line }, clientFor);
|
|
144
|
+
const result = await guard(() => client.listMembers(jid, typeof args.limit === 'number' ? args.limit : undefined));
|
|
145
|
+
respond(id, { result });
|
|
146
|
+
};
|
|
147
|
+
}
|
|
148
|
+
|
|
120
149
|
function makeReact(clientFor: ClientFor): StationHandler {
|
|
121
150
|
return async (id, args) => {
|
|
122
151
|
const { accountId, client, jid } = resolve(args, clientFor);
|
|
@@ -208,6 +237,8 @@ export function makeHandleCall(
|
|
|
208
237
|
handlers: {
|
|
209
238
|
accounts: makeAccounts(clientFor),
|
|
210
239
|
send: makeSend(clientFor),
|
|
240
|
+
read: makeRead(clientFor),
|
|
241
|
+
listMembers: makeListMembers(clientFor),
|
|
211
242
|
react: makeReact(clientFor),
|
|
212
243
|
typing: makeTyping(clientFor),
|
|
213
244
|
edit: makeEdit(clientFor),
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
import { errMsg, log } from '@metro-labs/core/log';
|
|
2
|
+
import { isPnUser, isLidUser, makeEventBuffer, processSyncAction, type BaileysEventEmitter, type Contact } from 'baileys';
|
|
3
|
+
import type { History } from './history.js';
|
|
4
|
+
import { baileysLogger } from './logger.js';
|
|
5
|
+
|
|
6
|
+
interface AppStateHooks {
|
|
7
|
+
history: History;
|
|
8
|
+
current(): boolean;
|
|
9
|
+
}
|
|
10
|
+
|
|
11
|
+
export function supportsHistoryDeletion(processAction = processSyncAction): boolean {
|
|
12
|
+
const ev = makeEventBuffer(baileysLogger('history-support'));
|
|
13
|
+
const id = 'history-support@g.us';
|
|
14
|
+
const messageRange = { lastMessageTimestamp: 1 };
|
|
15
|
+
let received = 0;
|
|
16
|
+
ev.on('chats.clear', (event) => {
|
|
17
|
+
if (event.id === id && event.messageRange === messageRange) received++;
|
|
18
|
+
});
|
|
19
|
+
try {
|
|
20
|
+
for (const initial of [false, true]) {
|
|
21
|
+
for (const type of ['clearChat', 'deleteChat']) {
|
|
22
|
+
const value = type === 'clearChat' ? { clearChatAction: { messageRange } } : { deleteChatAction: { messageRange } };
|
|
23
|
+
const before = received;
|
|
24
|
+
processAction({ index: [type, id], syncAction: { value } }, ev, { id }, initial ? { accountSettings: { unarchiveChats: false } } : undefined);
|
|
25
|
+
if (received !== before + 1) return false;
|
|
26
|
+
}
|
|
27
|
+
}
|
|
28
|
+
return true;
|
|
29
|
+
} catch (err) {
|
|
30
|
+
log.warn({ err: errMsg(err) }, 'whatsapp: local history deletion support check failed');
|
|
31
|
+
return false;
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
export function noteHistoryContact(history: History, contact: Partial<Contact>): void {
|
|
36
|
+
const pn = contact.phoneNumber ?? (isPnUser(contact.id) ? contact.id : undefined);
|
|
37
|
+
const lid = contact.lid ?? (isLidUser(contact.id) ? contact.id : undefined);
|
|
38
|
+
if (pn && lid) history.alias(pn, lid);
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
export function bindAppState(ev: BaileysEventEmitter, hooks: AppStateHooks): void {
|
|
42
|
+
ev.on('lid-mapping.update', ({ pn, lid }) => {
|
|
43
|
+
if (hooks.current()) hooks.history.alias(pn, lid);
|
|
44
|
+
});
|
|
45
|
+
ev.on('chats.clear', ({ id, messageRange }) => {
|
|
46
|
+
if (!hooks.current()) return;
|
|
47
|
+
hooks.history.clearRange(id, messageRange ?? {});
|
|
48
|
+
});
|
|
49
|
+
}
|