@inetafrica/open-claudia 3.1.10 → 3.1.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +2 -0
- package/CHANGELOG.md +7 -0
- package/channels/kazee/adapter.js +4 -0
- package/channels/kazee/call/audio.js +133 -0
- package/channels/kazee/call/janus-client.js +136 -0
- package/channels/kazee/call/manager.js +109 -0
- package/channels/kazee/call/session.js +158 -0
- package/channels/kazee/call/speech-client.js +79 -0
- package/core/actions.js +21 -0
- package/core/handlers.js +43 -0
- package/core/media.js +50 -3
- package/core/router.js +33 -1
- package/core/runner.js +3 -1
- package/core/tts-voice.js +98 -0
- package/package.json +8 -2
- package/test-provider-language.js +13 -1
package/.env.example
CHANGED
|
@@ -13,6 +13,8 @@ KAZEE_BOT_USER_ID=
|
|
|
13
13
|
KAZEE_OWNER_USER_ID=
|
|
14
14
|
# Set to 1 to log every inbound Kazee socket event (debug aid; noisy).
|
|
15
15
|
KAZEE_DEBUG_EVENTS=
|
|
16
|
+
# Optional override; Kazee calls default to the production Janus WebSocket.
|
|
17
|
+
KAZEE_JANUS_URL=wss://janus.kazee.africa/ws
|
|
16
18
|
|
|
17
19
|
WORKSPACE=/path/to/your/workspace
|
|
18
20
|
# Both provider paths are optional. Leave blank to discover the CLI on PATH.
|
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,12 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## v3.1.13 — Owner-only Kazee calls reach Open Claudia
|
|
4
|
+
|
|
5
|
+
- Accepts incoming audio calls from the configured Kazee owner only; external, video, malformed, and concurrent calls are rejected before any media session is created.
|
|
6
|
+
- Joins the existing Janus VideoRoom as a headless publisher/subscriber using pure-TypeScript WebRTC, decodes inbound Opus, commits VAD-delimited 16 kHz PCM to hosted streaming STT, and routes final utterances through the owner's normal Kazee identity and conversation.
|
|
7
|
+
- Speaks only the settled, guardrail-vetted answer through hosted Pocket TTS PCM and Opus RTP. Provider narration and transient progress previews are suppressed in calls; audio is never persisted.
|
|
8
|
+
- Adds deterministic audio, Janus transaction, WebRTC RTP, speech framing, owner gate, lifecycle, and AI-routing tests. Updates `ws` to the patched 8.21 line across direct and Socket.IO use.
|
|
9
|
+
|
|
3
10
|
## v3.1.10 — `say` synthesizes in the bot process, not the stripped subprocess
|
|
4
11
|
|
|
5
12
|
- v3.1.9 carried this exact fix but its build never published: the version pin in `test-provider-language.js` still read `3.1.8`, so the guard failed and the docker stage was skipped. Re-cut as v3.1.10 per "never retag a failed version."
|
|
@@ -13,6 +13,7 @@ const { canonicalForChannel } = require("../../core/identity");
|
|
|
13
13
|
const { inlineKeyboardToPortable } = require("../types");
|
|
14
14
|
const { createKazeeSocket } = require("./socket");
|
|
15
15
|
const { normalize } = require("./format");
|
|
16
|
+
const { KazeeCallManager } = require("./call/manager");
|
|
16
17
|
|
|
17
18
|
class KazeeAdapter {
|
|
18
19
|
constructor({ id = "kazee", url, token, ownerUserId, botUserId }) {
|
|
@@ -27,6 +28,7 @@ class KazeeAdapter {
|
|
|
27
28
|
this.botUserId = botUserId || null;
|
|
28
29
|
this._listeners = { message: new Set(), action: new Set() };
|
|
29
30
|
this._socket = null;
|
|
31
|
+
this._callManager = new KazeeCallManager({ adapter: this });
|
|
30
32
|
}
|
|
31
33
|
|
|
32
34
|
on(event, fn) {
|
|
@@ -67,11 +69,13 @@ class KazeeAdapter {
|
|
|
67
69
|
}
|
|
68
70
|
|
|
69
71
|
async stop() {
|
|
72
|
+
await this._callManager.stop();
|
|
70
73
|
try { this._socket?.disconnect?.(); } catch (e) {}
|
|
71
74
|
this._socket = null;
|
|
72
75
|
}
|
|
73
76
|
|
|
74
77
|
_wireSocket(socket) {
|
|
78
|
+
this._callManager.wire(socket);
|
|
75
79
|
if (process.env.KAZEE_DEBUG_EVENTS) {
|
|
76
80
|
socket.onAny((event, ...args) => {
|
|
77
81
|
try { console.log(`[kazee event] ${event}`, JSON.stringify(args).slice(0, 400)); }
|
|
@@ -0,0 +1,133 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
|
|
3
|
+
const OpusScript = require("opusscript");
|
|
4
|
+
|
|
5
|
+
const WEBRTC_RATE = 48000;
|
|
6
|
+
const STT_RATE = 16000;
|
|
7
|
+
const FRAME_MS = 20;
|
|
8
|
+
const WEBRTC_SAMPLES = WEBRTC_RATE * FRAME_MS / 1000;
|
|
9
|
+
|
|
10
|
+
class OpusAudioCodec {
|
|
11
|
+
constructor({ channels = 1 } = {}) {
|
|
12
|
+
this.channels = channels;
|
|
13
|
+
this.encoder = new OpusScript(WEBRTC_RATE, channels, OpusScript.Application.VOIP);
|
|
14
|
+
this.decoder = new OpusScript(WEBRTC_RATE, channels, OpusScript.Application.VOIP);
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
encode(pcm48k) {
|
|
18
|
+
if (!Buffer.isBuffer(pcm48k)) pcm48k = Buffer.from(pcm48k);
|
|
19
|
+
const samples = pcm48k.length / 2 / this.channels;
|
|
20
|
+
if (samples !== WEBRTC_SAMPLES) {
|
|
21
|
+
throw new Error(`Opus input must be exactly ${WEBRTC_SAMPLES} samples per channel (20ms)`);
|
|
22
|
+
}
|
|
23
|
+
return Buffer.from(this.encoder.encode(pcm48k, samples));
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
decode(opus) {
|
|
27
|
+
return Buffer.from(this.decoder.decode(Buffer.from(opus)));
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
close() {
|
|
31
|
+
this.encoder?.delete();
|
|
32
|
+
this.decoder?.delete();
|
|
33
|
+
this.encoder = null;
|
|
34
|
+
this.decoder = null;
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
// WebRTC Opus is 48 kHz. Faster Whisper's streaming endpoint accepts PCM at
|
|
39
|
+
// any declared rate, but 16 kHz cuts bandwidth and inference conversion work.
|
|
40
|
+
function downsample48To16(pcm48k) {
|
|
41
|
+
if (!Buffer.isBuffer(pcm48k)) pcm48k = Buffer.from(pcm48k);
|
|
42
|
+
const input = new Int16Array(pcm48k.buffer, pcm48k.byteOffset, Math.floor(pcm48k.length / 2));
|
|
43
|
+
const output = Buffer.allocUnsafe(Math.floor(input.length / 3) * 2);
|
|
44
|
+
const samples = new Int16Array(output.buffer, output.byteOffset, output.length / 2);
|
|
45
|
+
for (let i = 0, j = 0; j < samples.length; i += 3, j += 1) {
|
|
46
|
+
samples[j] = Math.round((input[i] + input[i + 1] + input[i + 2]) / 3);
|
|
47
|
+
}
|
|
48
|
+
return output;
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
function upsampleTo48(pcm, sourceRate) {
|
|
52
|
+
if (!Buffer.isBuffer(pcm)) pcm = Buffer.from(pcm);
|
|
53
|
+
if (sourceRate === WEBRTC_RATE) return Buffer.from(pcm);
|
|
54
|
+
if (!Number.isFinite(sourceRate) || sourceRate < 8000 || sourceRate > WEBRTC_RATE) {
|
|
55
|
+
throw new Error(`Unsupported PCM source rate: ${sourceRate}`);
|
|
56
|
+
}
|
|
57
|
+
const input = new Int16Array(pcm.buffer, pcm.byteOffset, Math.floor(pcm.length / 2));
|
|
58
|
+
const outputLength = Math.max(0, Math.round(input.length * WEBRTC_RATE / sourceRate));
|
|
59
|
+
const output = Buffer.allocUnsafe(outputLength * 2);
|
|
60
|
+
const samples = new Int16Array(output.buffer, output.byteOffset, outputLength);
|
|
61
|
+
const scale = sourceRate / WEBRTC_RATE;
|
|
62
|
+
for (let i = 0; i < outputLength; i += 1) {
|
|
63
|
+
const position = i * scale;
|
|
64
|
+
const left = Math.floor(position);
|
|
65
|
+
const right = Math.min(left + 1, input.length - 1);
|
|
66
|
+
const fraction = position - left;
|
|
67
|
+
samples[i] = Math.round(input[left] + (input[right] - input[left]) * fraction);
|
|
68
|
+
}
|
|
69
|
+
return output;
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
function rmsDbfs(pcm) {
|
|
73
|
+
if (!pcm?.length) return -Infinity;
|
|
74
|
+
const samples = new Int16Array(pcm.buffer, pcm.byteOffset, Math.floor(pcm.length / 2));
|
|
75
|
+
let sum = 0;
|
|
76
|
+
for (const sample of samples) {
|
|
77
|
+
const normalized = sample / 32768;
|
|
78
|
+
sum += normalized * normalized;
|
|
79
|
+
}
|
|
80
|
+
if (!sum) return -Infinity;
|
|
81
|
+
return 20 * Math.log10(Math.sqrt(sum / samples.length));
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
class TurnDetector {
|
|
85
|
+
constructor({ thresholdDb = -38, onsetFrames = 3, silenceMs = 700, maxTurnMs = 30000 } = {}) {
|
|
86
|
+
this.thresholdDb = thresholdDb;
|
|
87
|
+
this.onsetFrames = onsetFrames;
|
|
88
|
+
this.silenceFrames = Math.ceil(silenceMs / FRAME_MS);
|
|
89
|
+
this.maxFrames = Math.ceil(maxTurnMs / FRAME_MS);
|
|
90
|
+
this.reset();
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
reset() {
|
|
94
|
+
this.active = false;
|
|
95
|
+
this.loudFrames = 0;
|
|
96
|
+
this.quietFrames = 0;
|
|
97
|
+
this.turnFrames = 0;
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
push(pcm48k) {
|
|
101
|
+
const loud = rmsDbfs(pcm48k) >= this.thresholdDb;
|
|
102
|
+
if (!this.active) {
|
|
103
|
+
this.loudFrames = loud ? this.loudFrames + 1 : 0;
|
|
104
|
+
if (this.loudFrames >= this.onsetFrames) {
|
|
105
|
+
this.active = true;
|
|
106
|
+
this.quietFrames = 0;
|
|
107
|
+
this.turnFrames = this.loudFrames;
|
|
108
|
+
return { type: "start" };
|
|
109
|
+
}
|
|
110
|
+
return null;
|
|
111
|
+
}
|
|
112
|
+
this.turnFrames += 1;
|
|
113
|
+
this.quietFrames = loud ? 0 : this.quietFrames + 1;
|
|
114
|
+
if (this.quietFrames >= this.silenceFrames || this.turnFrames >= this.maxFrames) {
|
|
115
|
+
const reason = this.turnFrames >= this.maxFrames ? "max-duration" : "silence";
|
|
116
|
+
this.reset();
|
|
117
|
+
return { type: "commit", reason };
|
|
118
|
+
}
|
|
119
|
+
return null;
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
module.exports = {
|
|
124
|
+
FRAME_MS,
|
|
125
|
+
STT_RATE,
|
|
126
|
+
WEBRTC_RATE,
|
|
127
|
+
WEBRTC_SAMPLES,
|
|
128
|
+
OpusAudioCodec,
|
|
129
|
+
TurnDetector,
|
|
130
|
+
downsample48To16,
|
|
131
|
+
rmsDbfs,
|
|
132
|
+
upsampleTo48,
|
|
133
|
+
};
|
|
@@ -0,0 +1,136 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
|
|
3
|
+
const crypto = require("crypto");
|
|
4
|
+
const WebSocket = require("ws");
|
|
5
|
+
|
|
6
|
+
function transactionId() {
|
|
7
|
+
return crypto.randomBytes(8).toString("hex");
|
|
8
|
+
}
|
|
9
|
+
|
|
10
|
+
class JanusClient {
|
|
11
|
+
constructor({ url, WebSocketImpl = WebSocket, requestTimeoutMs = 10000, keepaliveMs = 25000 } = {}) {
|
|
12
|
+
if (!url) throw new Error("Janus URL is required");
|
|
13
|
+
this.url = url;
|
|
14
|
+
this.WebSocketImpl = WebSocketImpl;
|
|
15
|
+
this.requestTimeoutMs = requestTimeoutMs;
|
|
16
|
+
this.keepaliveMs = keepaliveMs;
|
|
17
|
+
this.socket = null;
|
|
18
|
+
this.sessionId = null;
|
|
19
|
+
this.pending = new Map();
|
|
20
|
+
this.handleListeners = new Map();
|
|
21
|
+
this.keepalive = null;
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
async connect() {
|
|
25
|
+
if (this.socket) return;
|
|
26
|
+
const socket = new this.WebSocketImpl(this.url, "janus-protocol");
|
|
27
|
+
this.socket = socket;
|
|
28
|
+
socket.on("message", (data) => this._onMessage(data));
|
|
29
|
+
socket.on("close", () => this._onClose(new Error("Janus socket closed")));
|
|
30
|
+
socket.on("error", (error) => this._onClose(error));
|
|
31
|
+
await new Promise((resolve, reject) => {
|
|
32
|
+
socket.once("open", resolve);
|
|
33
|
+
socket.once("error", reject);
|
|
34
|
+
});
|
|
35
|
+
const created = await this.request({ janus: "create" }, { withSession: false });
|
|
36
|
+
this.sessionId = created?.data?.id;
|
|
37
|
+
if (!this.sessionId) throw new Error("Janus create response did not include a session id");
|
|
38
|
+
this.keepalive = setInterval(() => {
|
|
39
|
+
this.request({ janus: "keepalive" }).catch((error) => console.error("Janus keepalive:", error.message));
|
|
40
|
+
}, this.keepaliveMs);
|
|
41
|
+
this.keepalive.unref?.();
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
async attach(plugin = "janus.plugin.videoroom") {
|
|
45
|
+
const response = await this.request({ janus: "attach", plugin });
|
|
46
|
+
const handleId = response?.data?.id;
|
|
47
|
+
if (!handleId) throw new Error("Janus attach response did not include a handle id");
|
|
48
|
+
return handleId;
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
onHandle(handleId, listener) {
|
|
52
|
+
const id = String(handleId);
|
|
53
|
+
if (!this.handleListeners.has(id)) this.handleListeners.set(id, new Set());
|
|
54
|
+
this.handleListeners.get(id).add(listener);
|
|
55
|
+
return () => this.handleListeners.get(id)?.delete(listener);
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
message(handleId, body, jsep) {
|
|
59
|
+
return this.request({ janus: "message", handle_id: handleId, body, ...(jsep ? { jsep } : {}) });
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
async detach(handleId) {
|
|
63
|
+
if (!handleId || !this.sessionId) return;
|
|
64
|
+
try { await this.request({ janus: "detach", handle_id: handleId }); }
|
|
65
|
+
finally { this.handleListeners.delete(String(handleId)); }
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
request(payload, { withSession = true } = {}) {
|
|
69
|
+
if (!this.socket || this.socket.readyState !== this.WebSocketImpl.OPEN) {
|
|
70
|
+
return Promise.reject(new Error("Janus socket is not open"));
|
|
71
|
+
}
|
|
72
|
+
const transaction = transactionId();
|
|
73
|
+
const frame = {
|
|
74
|
+
...payload,
|
|
75
|
+
...(withSession && this.sessionId ? { session_id: this.sessionId } : {}),
|
|
76
|
+
transaction,
|
|
77
|
+
};
|
|
78
|
+
return new Promise((resolve, reject) => {
|
|
79
|
+
const timer = setTimeout(() => {
|
|
80
|
+
this.pending.delete(transaction);
|
|
81
|
+
reject(new Error(`Janus ${payload.janus} timed out`));
|
|
82
|
+
}, this.requestTimeoutMs);
|
|
83
|
+
this.pending.set(transaction, { resolve, reject, timer });
|
|
84
|
+
this.socket.send(JSON.stringify(frame), (error) => {
|
|
85
|
+
if (!error) return;
|
|
86
|
+
clearTimeout(timer);
|
|
87
|
+
this.pending.delete(transaction);
|
|
88
|
+
reject(error);
|
|
89
|
+
});
|
|
90
|
+
});
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
async close() {
|
|
94
|
+
if (this.keepalive) clearInterval(this.keepalive);
|
|
95
|
+
this.keepalive = null;
|
|
96
|
+
if (this.sessionId && this.socket?.readyState === this.WebSocketImpl.OPEN) {
|
|
97
|
+
try { await this.request({ janus: "destroy" }); } catch (_) {}
|
|
98
|
+
}
|
|
99
|
+
this.sessionId = null;
|
|
100
|
+
const socket = this.socket;
|
|
101
|
+
this.socket = null;
|
|
102
|
+
if (socket && socket.readyState < this.WebSocketImpl.CLOSING) socket.close();
|
|
103
|
+
this._onClose(new Error("Janus client closed"));
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
_onMessage(raw) {
|
|
107
|
+
let frame;
|
|
108
|
+
try { frame = JSON.parse(raw.toString()); }
|
|
109
|
+
catch (_) { return; }
|
|
110
|
+
const pending = frame.transaction && this.pending.get(frame.transaction);
|
|
111
|
+
if (pending && frame.janus !== "ack") {
|
|
112
|
+
clearTimeout(pending.timer);
|
|
113
|
+
this.pending.delete(frame.transaction);
|
|
114
|
+
if (frame.janus === "error") pending.reject(new Error(frame.error?.reason || "Janus error"));
|
|
115
|
+
else pending.resolve(frame);
|
|
116
|
+
}
|
|
117
|
+
const sender = frame.sender != null ? String(frame.sender) : null;
|
|
118
|
+
if (sender) {
|
|
119
|
+
for (const listener of this.handleListeners.get(sender) || []) {
|
|
120
|
+
try { listener(frame); } catch (error) { console.error("Janus handle listener:", error.message); }
|
|
121
|
+
}
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
_onClose(error) {
|
|
126
|
+
if (this.keepalive) clearInterval(this.keepalive);
|
|
127
|
+
this.keepalive = null;
|
|
128
|
+
for (const pending of this.pending.values()) {
|
|
129
|
+
clearTimeout(pending.timer);
|
|
130
|
+
pending.reject(error);
|
|
131
|
+
}
|
|
132
|
+
this.pending.clear();
|
|
133
|
+
}
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
module.exports = { JanusClient };
|
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
|
|
3
|
+
const { canonicalForChannel } = require("../../../core/identity");
|
|
4
|
+
const { KazeeCallSession } = require("./session");
|
|
5
|
+
|
|
6
|
+
const DEFAULT_JANUS_URL = "wss://janus.kazee.africa/ws";
|
|
7
|
+
const DEFAULT_ICE_SERVERS = [
|
|
8
|
+
{ urls: "stun:coturn.kazee.africa:3478" },
|
|
9
|
+
{ urls: "stun:stun.l.google.com:19302" },
|
|
10
|
+
];
|
|
11
|
+
|
|
12
|
+
class KazeeCallManager {
|
|
13
|
+
constructor({ adapter, SessionImpl = KazeeCallSession, janusUrl = process.env.KAZEE_JANUS_URL || DEFAULT_JANUS_URL,
|
|
14
|
+
iceServers = DEFAULT_ICE_SERVERS } = {}) {
|
|
15
|
+
this.adapter = adapter;
|
|
16
|
+
this.SessionImpl = SessionImpl;
|
|
17
|
+
this.janusUrl = janusUrl;
|
|
18
|
+
this.iceServers = iceServers;
|
|
19
|
+
this.session = null;
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
wire(socket) {
|
|
23
|
+
socket.on("call:incoming", (payload) => this.incoming(payload));
|
|
24
|
+
socket.on("call:ended", (payload) => this.ended(payload));
|
|
25
|
+
socket.on("call:rejected", (payload) => this.ended(payload));
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
async incoming(payload = {}) {
|
|
29
|
+
const callerId = String(payload.fromUserId || "");
|
|
30
|
+
const chatId = String(payload.chatId || "");
|
|
31
|
+
const roomId = Number(payload.roomId);
|
|
32
|
+
const callType = String(payload.callType || "");
|
|
33
|
+
const owner = String(this.adapter.ownerUserId || "");
|
|
34
|
+
const reject = (reason) => this.adapter._socket?.emit("call:reject", {
|
|
35
|
+
toUserId: callerId, callType, chatId, roomId, reason,
|
|
36
|
+
});
|
|
37
|
+
|
|
38
|
+
if (!callerId || !chatId || !Number.isSafeInteger(roomId)) return reject("invalid_call");
|
|
39
|
+
if (callerId !== owner) return reject("not_allowed");
|
|
40
|
+
if (callType !== "audio") return reject("unsupported_media");
|
|
41
|
+
if (this.session) return reject("busy");
|
|
42
|
+
|
|
43
|
+
const session = new this.SessionImpl({
|
|
44
|
+
roomId, chatId, callerId, botUserId: this.adapter.botUserId,
|
|
45
|
+
janusUrl: this.janusUrl, iceServers: this.iceServers,
|
|
46
|
+
onTranscript: (text) => this._transcript(text, payload),
|
|
47
|
+
onEnded: () => { if (this.session === session) this.session = null; },
|
|
48
|
+
});
|
|
49
|
+
this.session = session;
|
|
50
|
+
try {
|
|
51
|
+
await session.start();
|
|
52
|
+
this.adapter._socket?.emit("call:accept", {
|
|
53
|
+
toUserId: callerId, callType: "audio", chatId, roomId,
|
|
54
|
+
});
|
|
55
|
+
} catch (error) {
|
|
56
|
+
console.error("Kazee call connect:", error.message);
|
|
57
|
+
this.session = null;
|
|
58
|
+
await session.close("connect-failed").catch(() => {});
|
|
59
|
+
reject("connection_failed");
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
_transcript(text, incoming) {
|
|
64
|
+
const session = this.session;
|
|
65
|
+
if (!session || !text) return;
|
|
66
|
+
const callAdapter = Object.create(this.adapter);
|
|
67
|
+
// Provider narration/previews are useful in chat but must never be spoken
|
|
68
|
+
// into a call. Only the settled, guarded final answer reaches send().
|
|
69
|
+
callAdapter.suppressStreamPreview = true;
|
|
70
|
+
callAdapter.send = async (_channelId, response) => {
|
|
71
|
+
await session.speak(response);
|
|
72
|
+
return true;
|
|
73
|
+
};
|
|
74
|
+
callAdapter.edit = async () => true;
|
|
75
|
+
callAdapter.typing = async () => {};
|
|
76
|
+
callAdapter.sendVoice = async (_channelId, voicePath) => {
|
|
77
|
+
console.warn(`Kazee call ignored file voice response ${voicePath}; streamed TTS is active`);
|
|
78
|
+
return true;
|
|
79
|
+
};
|
|
80
|
+
this.adapter._emit("message", {
|
|
81
|
+
adapter: callAdapter,
|
|
82
|
+
channelId: String(incoming.chatId),
|
|
83
|
+
canonicalUserId: canonicalForChannel("kazee", String(incoming.fromUserId)),
|
|
84
|
+
userId: String(incoming.fromUserId),
|
|
85
|
+
type: "text",
|
|
86
|
+
text,
|
|
87
|
+
messageId: `call-${incoming.roomId}-${Date.now()}`,
|
|
88
|
+
from: { id: String(incoming.fromUserId), name: incoming.fromUserName || "" },
|
|
89
|
+
raw: { ...incoming, source: "kazee-call", ephemeralAudio: true },
|
|
90
|
+
});
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
async ended(payload = {}) {
|
|
94
|
+
if (!this.session) return;
|
|
95
|
+
if (payload.roomId != null && Number(payload.roomId) !== this.session.roomId) return;
|
|
96
|
+
const session = this.session;
|
|
97
|
+
this.session = null;
|
|
98
|
+
await session.close("remote-ended");
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
async stop() {
|
|
102
|
+
if (!this.session) return;
|
|
103
|
+
const session = this.session;
|
|
104
|
+
this.session = null;
|
|
105
|
+
await session.close("adapter-stopped");
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
module.exports = { KazeeCallManager, DEFAULT_ICE_SERVERS, DEFAULT_JANUS_URL };
|
|
@@ -0,0 +1,158 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
|
|
3
|
+
const { MediaStreamTrack, RTCPeerConnection, RtpHeader, RtpPacket } = require("werift");
|
|
4
|
+
const { JanusClient } = require("./janus-client");
|
|
5
|
+
const { OpusAudioCodec, TurnDetector, WEBRTC_SAMPLES, downsample48To16, upsampleTo48 } = require("./audio");
|
|
6
|
+
const { SpeechStream, synthesizePcm } = require("./speech-client");
|
|
7
|
+
|
|
8
|
+
const wait = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
|
|
9
|
+
|
|
10
|
+
class KazeeCallSession {
|
|
11
|
+
constructor({ roomId, chatId, callerId, botUserId, janusUrl, iceServers = [], onTranscript, onEnded,
|
|
12
|
+
JanusClientImpl = JanusClient, PeerConnectionImpl = RTCPeerConnection, SpeechStreamImpl = SpeechStream } = {}) {
|
|
13
|
+
this.roomId = Number(roomId);
|
|
14
|
+
this.chatId = String(chatId);
|
|
15
|
+
this.callerId = String(callerId);
|
|
16
|
+
this.botUserId = String(botUserId);
|
|
17
|
+
this.janus = new JanusClientImpl({ url: janusUrl });
|
|
18
|
+
this.PeerConnectionImpl = PeerConnectionImpl;
|
|
19
|
+
this.SpeechStreamImpl = SpeechStreamImpl;
|
|
20
|
+
this.iceServers = iceServers;
|
|
21
|
+
this.onTranscript = onTranscript || (() => {});
|
|
22
|
+
this.onEnded = onEnded || (() => {});
|
|
23
|
+
this.codec = new OpusAudioCodec();
|
|
24
|
+
this.vad = new TurnDetector();
|
|
25
|
+
this.stt = new SpeechStreamImpl({ onTranscript: this.onTranscript });
|
|
26
|
+
this.publisher = null;
|
|
27
|
+
this.publisherHandle = null;
|
|
28
|
+
this.subscribers = new Map();
|
|
29
|
+
this.closed = false;
|
|
30
|
+
this.sequence = Math.floor(Math.random() * 65535);
|
|
31
|
+
this.timestamp = Math.floor(Math.random() * 0xffffffff);
|
|
32
|
+
this.ssrc = Math.floor(Math.random() * 0xffffffff);
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
async start() {
|
|
36
|
+
await this.janus.connect();
|
|
37
|
+
this.publisherHandle = await this.janus.attach();
|
|
38
|
+
this.janus.onHandle(this.publisherHandle, (frame) => this._onPublisherEvent(frame));
|
|
39
|
+
const joined = await this.janus.message(this.publisherHandle, {
|
|
40
|
+
request: "join", ptype: "publisher", room: this.roomId, display: `open-claudia:${this.botUserId}`,
|
|
41
|
+
});
|
|
42
|
+
await this._consumePublisherFrame(joined);
|
|
43
|
+
await this.stt.connect();
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
async _onPublisherEvent(frame) {
|
|
47
|
+
try { await this._consumePublisherFrame(frame); }
|
|
48
|
+
catch (error) { console.error("Kazee call publisher:", error.message); await this.close("janus-error"); }
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
async _consumePublisherFrame(frame) {
|
|
52
|
+
const data = frame?.plugindata?.data || {};
|
|
53
|
+
if (data.error) throw new Error(data.error);
|
|
54
|
+
for (const publisher of data.publishers || []) {
|
|
55
|
+
if (!this.subscribers.has(Number(publisher.id)) && !String(publisher.display || "").startsWith("open-claudia:")) {
|
|
56
|
+
await this._subscribe(Number(publisher.id));
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
if (data.videoroom === "joined" && !this.publisher) await this._publish();
|
|
60
|
+
const gone = Number(data.leaving || data.unpublished || 0);
|
|
61
|
+
if (gone && this.subscribers.has(gone)) await this._removeSubscriber(gone);
|
|
62
|
+
if (data.videoroom === "destroyed") await this.close("room-destroyed");
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
async _publish() {
|
|
66
|
+
if (this.publisher) return;
|
|
67
|
+
const pc = new this.PeerConnectionImpl({ iceServers: this.iceServers });
|
|
68
|
+
const track = new MediaStreamTrack({ kind: "audio" });
|
|
69
|
+
const sender = pc.addTrack(track);
|
|
70
|
+
this.publisher = { pc, track, sender };
|
|
71
|
+
const offer = await pc.createOffer();
|
|
72
|
+
await pc.setLocalDescription(offer);
|
|
73
|
+
const configured = await this.janus.message(this.publisherHandle, { request: "configure", audio: true, video: false }, pc.localDescription);
|
|
74
|
+
const answer = configured?.jsep;
|
|
75
|
+
if (!answer) throw new Error("Janus publisher configure did not return SDP");
|
|
76
|
+
await pc.setRemoteDescription(answer);
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
async _subscribe(feedId) {
|
|
80
|
+
const handle = await this.janus.attach();
|
|
81
|
+
const pc = new this.PeerConnectionImpl({ iceServers: this.iceServers });
|
|
82
|
+
const sub = { handle, pc };
|
|
83
|
+
this.subscribers.set(feedId, sub);
|
|
84
|
+
pc.onTrack.subscribe((track) => {
|
|
85
|
+
if (track.kind === "audio") track.onReceiveRtp.subscribe((packet) => this._onIncomingRtp(packet));
|
|
86
|
+
});
|
|
87
|
+
this.janus.onHandle(handle, (frame) => this._onSubscriberEvent(feedId, frame));
|
|
88
|
+
const attached = await this.janus.message(handle, { request: "join", ptype: "subscriber", room: this.roomId, feed: feedId });
|
|
89
|
+
await this._consumeSubscriberFrame(feedId, attached);
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
async _onSubscriberEvent(feedId, frame) {
|
|
93
|
+
try { await this._consumeSubscriberFrame(feedId, frame); }
|
|
94
|
+
catch (error) { console.error("Kazee call subscriber:", error.message); await this._removeSubscriber(feedId); }
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
async _consumeSubscriberFrame(feedId, frame) {
|
|
98
|
+
const sub = this.subscribers.get(feedId);
|
|
99
|
+
if (!sub) return;
|
|
100
|
+
const data = frame?.plugindata?.data || {};
|
|
101
|
+
if (data.error) throw new Error(data.error);
|
|
102
|
+
if (frame.jsep && !sub.started) {
|
|
103
|
+
sub.started = true;
|
|
104
|
+
await sub.pc.setRemoteDescription(frame.jsep);
|
|
105
|
+
const answer = await sub.pc.createAnswer();
|
|
106
|
+
await sub.pc.setLocalDescription(answer);
|
|
107
|
+
await this.janus.message(sub.handle, { request: "start", room: this.roomId }, sub.pc.localDescription);
|
|
108
|
+
}
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
_onIncomingRtp(packet) {
|
|
112
|
+
if (this.closed || !packet?.payload?.length) return;
|
|
113
|
+
let pcm;
|
|
114
|
+
try { pcm = this.codec.decode(packet.payload); } catch (_) { return; }
|
|
115
|
+
const event = this.vad.push(pcm);
|
|
116
|
+
if (this.vad.active || event?.type === "start") this.stt.audio(downsample48To16(pcm)).catch((e) => this.close(`stt:${e.message}`));
|
|
117
|
+
if (event?.type === "commit") this.stt.commit().catch((e) => this.close(`stt:${e.message}`));
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
async speak(text) {
|
|
121
|
+
if (this.closed || !String(text || "").trim()) return;
|
|
122
|
+
const synthesized = await synthesizePcm(String(text));
|
|
123
|
+
const pcm48 = upsampleTo48(synthesized.pcm, synthesized.sampleRate);
|
|
124
|
+
for (let offset = 0; offset < pcm48.length && !this.closed; offset += WEBRTC_SAMPLES * 2) {
|
|
125
|
+
let frame = pcm48.subarray(offset, offset + WEBRTC_SAMPLES * 2);
|
|
126
|
+
if (frame.length < WEBRTC_SAMPLES * 2) frame = Buffer.concat([frame, Buffer.alloc(WEBRTC_SAMPLES * 2 - frame.length)]);
|
|
127
|
+
const opus = this.codec.encode(frame);
|
|
128
|
+
const payloadType = this.publisher?.sender?.codec?.payloadType || 111;
|
|
129
|
+
this.publisher?.track?.writeRtp(new RtpPacket(new RtpHeader({
|
|
130
|
+
payloadType, sequenceNumber: this.sequence++ & 0xffff, timestamp: this.timestamp >>> 0, ssrc: this.ssrc,
|
|
131
|
+
}), opus));
|
|
132
|
+
this.timestamp = (this.timestamp + WEBRTC_SAMPLES) >>> 0;
|
|
133
|
+
await wait(20);
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
async _removeSubscriber(feedId) {
|
|
138
|
+
const sub = this.subscribers.get(feedId);
|
|
139
|
+
if (!sub) return;
|
|
140
|
+
this.subscribers.delete(feedId);
|
|
141
|
+
try { await sub.pc.close(); } catch (_) {}
|
|
142
|
+
try { await this.janus.detach(sub.handle); } catch (_) {}
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
async close(reason = "ended") {
|
|
146
|
+
if (this.closed) return;
|
|
147
|
+
this.closed = true;
|
|
148
|
+
this.stt.close();
|
|
149
|
+
for (const feed of [...this.subscribers.keys()]) await this._removeSubscriber(feed);
|
|
150
|
+
try { await this.publisher?.pc?.close(); } catch (_) {}
|
|
151
|
+
try { await this.janus.detach(this.publisherHandle); } catch (_) {}
|
|
152
|
+
try { await this.janus.close(); } catch (_) {}
|
|
153
|
+
this.codec.close();
|
|
154
|
+
this.onEnded(reason);
|
|
155
|
+
}
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
module.exports = { KazeeCallSession };
|
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
|
|
3
|
+
const WebSocket = require("ws");
|
|
4
|
+
const { SPEECH_SERVICE_URL } = require("../../../core/config");
|
|
5
|
+
const { speechBearerToken } = require("../../../core/media");
|
|
6
|
+
const ttsVoice = require("../../../core/tts-voice");
|
|
7
|
+
|
|
8
|
+
class SpeechStream {
|
|
9
|
+
constructor({ baseUrl = SPEECH_SERVICE_URL, token = speechBearerToken(), WebSocketImpl = WebSocket, onTranscript } = {}) {
|
|
10
|
+
this.baseUrl = String(baseUrl || "").replace(/\/$/, "");
|
|
11
|
+
this.token = token;
|
|
12
|
+
this.WebSocketImpl = WebSocketImpl;
|
|
13
|
+
this.onTranscript = onTranscript || (() => {});
|
|
14
|
+
this.socket = null;
|
|
15
|
+
this.ready = null;
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
async connect() {
|
|
19
|
+
if (this.ready) return this.ready;
|
|
20
|
+
if (!this.baseUrl || !this.token) throw new Error("Hosted speech service is not configured");
|
|
21
|
+
const url = this.baseUrl.replace(/^http/, "ws") + "/v1/audio/transcriptions/stream";
|
|
22
|
+
this.ready = new Promise((resolve, reject) => {
|
|
23
|
+
const ws = new this.WebSocketImpl(url, { headers: { Authorization: `Bearer ${this.token}` } });
|
|
24
|
+
this.socket = ws;
|
|
25
|
+
ws.once("open", () => {
|
|
26
|
+
ws.send(JSON.stringify({ sample_rate: 16000, language: "en" }));
|
|
27
|
+
resolve();
|
|
28
|
+
});
|
|
29
|
+
ws.once("error", reject);
|
|
30
|
+
ws.on("message", (raw) => this._onMessage(raw));
|
|
31
|
+
ws.on("close", () => { this.socket = null; this.ready = null; });
|
|
32
|
+
});
|
|
33
|
+
return this.ready;
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
async audio(pcm16k) {
|
|
37
|
+
await this.connect();
|
|
38
|
+
if (this.socket?.readyState === this.WebSocketImpl.OPEN) this.socket.send(pcm16k);
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
async commit() {
|
|
42
|
+
await this.connect();
|
|
43
|
+
this.socket.send(JSON.stringify({ type: "commit" }));
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
_onMessage(raw) {
|
|
47
|
+
let message;
|
|
48
|
+
try { message = JSON.parse(raw.toString()); } catch (_) { return; }
|
|
49
|
+
const text = String(message.text || message.transcript || "").trim();
|
|
50
|
+
if (text && (message.final === true || message.type === "final" || message.type === "transcription")) {
|
|
51
|
+
Promise.resolve(this.onTranscript(text)).catch((error) => console.error("Kazee call transcript:", error.message));
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
close() {
|
|
56
|
+
const ws = this.socket;
|
|
57
|
+
this.socket = null;
|
|
58
|
+
this.ready = null;
|
|
59
|
+
if (ws?.readyState === this.WebSocketImpl.OPEN) ws.send(JSON.stringify({ type: "close" }));
|
|
60
|
+
try { ws?.close(); } catch (_) {}
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
async function synthesizePcm(text, { baseUrl = SPEECH_SERVICE_URL, token = speechBearerToken(), fetchImpl = fetch } = {}) {
|
|
65
|
+
if (!baseUrl || !token) throw new Error("Hosted speech service is not configured");
|
|
66
|
+
const response = await fetchImpl(`${String(baseUrl).replace(/\/$/, "")}/v1/audio/speech`, {
|
|
67
|
+
method: "POST",
|
|
68
|
+
headers: { Authorization: `Bearer ${token}`, "Content-Type": "application/json" },
|
|
69
|
+
body: JSON.stringify({ input: text, voice: ttsVoice.getVoice(), response_format: "pcm" }),
|
|
70
|
+
signal: AbortSignal.timeout(60000),
|
|
71
|
+
});
|
|
72
|
+
if (!response.ok) throw new Error(`Hosted TTS failed: HTTP ${response.status}`);
|
|
73
|
+
const contentType = response.headers?.get?.("content-type") || "";
|
|
74
|
+
const rate = Number(contentType.match(/(?:^|;)\s*rate=(\d+)/i)?.[1] || 24000);
|
|
75
|
+
if (!Number.isFinite(rate) || rate < 8000 || rate > 48000) throw new Error(`Hosted TTS returned invalid PCM rate: ${rate}`);
|
|
76
|
+
return { pcm: Buffer.from(await response.arrayBuffer()), sampleRate: rate };
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
module.exports = { SpeechStream, synthesizePcm };
|
package/core/actions.js
CHANGED
|
@@ -416,6 +416,27 @@ async function handleAction(envelope) {
|
|
|
416
416
|
await send(`Tooling mode: ${m}${m === "relaxed" ? " ⚠️ deny-gate is log-only; ad-hoc scripting allowed" : " — tool-first enforced"}`);
|
|
417
417
|
return;
|
|
418
418
|
}
|
|
419
|
+
if (d.startsWith("vc:")) {
|
|
420
|
+
if (!isChatOwner(envelope.channelId)) return send("Owner only — the voice is shared across this bot.");
|
|
421
|
+
const ttsVoice = require("./tts-voice");
|
|
422
|
+
const rest = d.slice(3);
|
|
423
|
+
if (rest === "clone") {
|
|
424
|
+
state.pendingVoiceClone = { at: Date.now() };
|
|
425
|
+
return send("🎙️ Send me a voice note now — 5–15s of clear speech and I'll clone it as your voice. Send /voice to cancel.");
|
|
426
|
+
}
|
|
427
|
+
state.pendingVoiceClone = null;
|
|
428
|
+
if (rest === "default") {
|
|
429
|
+
ttsVoice.clearVoice();
|
|
430
|
+
return send(`🎙️ Voice reset to default (${ttsVoice.getVoiceLabel()}).`);
|
|
431
|
+
}
|
|
432
|
+
if (rest.startsWith("p:")) {
|
|
433
|
+
const preset = ttsVoice.presetByKey(rest.slice(2));
|
|
434
|
+
if (!preset) return send("That voice button is stale. Open /voice again.");
|
|
435
|
+
ttsVoice.setVoice(preset.id, preset.label);
|
|
436
|
+
return send(`🎙️ Voice set: ${preset.label}`);
|
|
437
|
+
}
|
|
438
|
+
return;
|
|
439
|
+
}
|
|
419
440
|
if (d.startsWith("apr:")) {
|
|
420
441
|
// Approval buttons. Owner-only: the exact payload was shown in the prompt;
|
|
421
442
|
// decide() is idempotent so double-taps can't flip it. Two shapes share
|
package/core/handlers.js
CHANGED
|
@@ -1322,6 +1322,49 @@ register({
|
|
|
1322
1322
|
},
|
|
1323
1323
|
});
|
|
1324
1324
|
|
|
1325
|
+
register({
|
|
1326
|
+
name: "voice", description: "Pick the Pocket TTS voice, or clone your own", args: "[<preset>|<hf://…>|clone|default]", ownerOnly: true,
|
|
1327
|
+
handler: async (env, { tail }) => {
|
|
1328
|
+
if (!authorized(env)) return;
|
|
1329
|
+
const ttsVoice = require("./tts-voice");
|
|
1330
|
+
const state = currentState();
|
|
1331
|
+
const arg = (tail || "").trim();
|
|
1332
|
+
if (arg) {
|
|
1333
|
+
const low = arg.toLowerCase();
|
|
1334
|
+
if (low === "clone") {
|
|
1335
|
+
state.pendingVoiceClone = { at: Date.now() };
|
|
1336
|
+
return send("🎙️ Send me a voice note now — 5–15s of clear speech and I'll clone it as your voice. Send /voice to cancel.");
|
|
1337
|
+
}
|
|
1338
|
+
state.pendingVoiceClone = null;
|
|
1339
|
+
if (low === "default" || low === "reset") {
|
|
1340
|
+
ttsVoice.clearVoice();
|
|
1341
|
+
return send(`🎙️ Voice reset to default (${ttsVoice.getVoiceLabel()}).`);
|
|
1342
|
+
}
|
|
1343
|
+
const preset = ttsVoice.presetByKey(low);
|
|
1344
|
+
if (preset) {
|
|
1345
|
+
ttsVoice.setVoice(preset.id, preset.label);
|
|
1346
|
+
return send(`🎙️ Voice set: ${preset.label}`);
|
|
1347
|
+
}
|
|
1348
|
+
// Anything else is treated as a raw voice reference pocket-tts understands:
|
|
1349
|
+
// a built-in shorthand, an hf:// path, an http(s) URL, or a stored id.
|
|
1350
|
+
ttsVoice.setVoice(arg, null);
|
|
1351
|
+
return send(`🎙️ Voice set: ${arg}`);
|
|
1352
|
+
}
|
|
1353
|
+
state.pendingVoiceClone = null;
|
|
1354
|
+
const rows = ttsVoice.presets().map((p) => [{ text: p.label, callback_data: `vc:p:${p.key}` }]);
|
|
1355
|
+
rows.push([
|
|
1356
|
+
{ text: "🎙️ Clone from a clip", callback_data: "vc:clone" },
|
|
1357
|
+
{ text: "↩︎ Default", callback_data: "vc:default" },
|
|
1358
|
+
]);
|
|
1359
|
+
send(
|
|
1360
|
+
`🎙️ Voice: ${ttsVoice.getVoiceLabel()}\n\n` +
|
|
1361
|
+
"Pick a voice — it takes effect immediately, no restart. Or tap Clone from a clip (or send /voice clone) to use your own recorded voice.\n\n" +
|
|
1362
|
+
"For any other voice: /voice hf://kyutai/tts-voices/<folder>/<clip>.wav",
|
|
1363
|
+
{ keyboard: { inline_keyboard: rows } },
|
|
1364
|
+
);
|
|
1365
|
+
},
|
|
1366
|
+
});
|
|
1367
|
+
|
|
1325
1368
|
register({
|
|
1326
1369
|
name: "budget", description: "Set max spend for next task", args: "[$N]",
|
|
1327
1370
|
handler: async (env, { tail }) => {
|
package/core/media.js
CHANGED
|
@@ -4,7 +4,8 @@
|
|
|
4
4
|
const fs = require("fs");
|
|
5
5
|
const path = require("path");
|
|
6
6
|
const { execSync } = require("child_process");
|
|
7
|
-
const { WHISPER_CLI, WHISPER_MODEL, FFMPEG, TEMP_DIR, ELEVENLABS_API_KEY, ELEVENLABS_VOICE_ID, ELEVENLABS_MODEL, SPEECH_SERVICE_URL, SPEECH_API_KEY, AGENTSPACE_POD_TOKEN,
|
|
7
|
+
const { WHISPER_CLI, WHISPER_MODEL, FFMPEG, TEMP_DIR, ELEVENLABS_API_KEY, ELEVENLABS_VOICE_ID, ELEVENLABS_MODEL, SPEECH_SERVICE_URL, SPEECH_API_KEY, AGENTSPACE_POD_TOKEN, TTS_PROVIDER } = require("./config");
|
|
8
|
+
const ttsVoice = require("./tts-voice");
|
|
8
9
|
|
|
9
10
|
const TTS_CMD = process.platform === "darwin" ? "say" : null;
|
|
10
11
|
|
|
@@ -108,7 +109,7 @@ async function pocketToVoice(clean) {
|
|
|
108
109
|
const res = await fetch(`${SPEECH_SERVICE_URL.replace(/\/$/, "")}/v1/audio/speech`, {
|
|
109
110
|
method: "POST",
|
|
110
111
|
headers: { Authorization: `Bearer ${token}`, "Content-Type": "application/json" },
|
|
111
|
-
body: JSON.stringify({ input: clean, voice:
|
|
112
|
+
body: JSON.stringify({ input: clean, voice: ttsVoice.getVoice(), response_format: "wav" }),
|
|
112
113
|
signal: AbortSignal.timeout(60000),
|
|
113
114
|
});
|
|
114
115
|
if (!res.ok) {
|
|
@@ -129,6 +130,52 @@ async function pocketToVoice(clean) {
|
|
|
129
130
|
}
|
|
130
131
|
}
|
|
131
132
|
|
|
133
|
+
// Upload a reference clip to the speech gateway for voice cloning. The gateway
|
|
134
|
+
// stores the clip on the TTS pod's model-cache volume and returns a voice id
|
|
135
|
+
// usable as the `voice` field on later /v1/audio/speech calls; pocket-tts clones
|
|
136
|
+
// from it. Returns the id string, or null on any failure.
|
|
137
|
+
async function cloneVoiceFromClip(clipPath) {
|
|
138
|
+
const token = speechBearerToken();
|
|
139
|
+
if (!SPEECH_SERVICE_URL || !token || !clipPath) return null;
|
|
140
|
+
// Telegram voice notes arrive as ogg/opus, which pocket-tts can't decode as a
|
|
141
|
+
// reference prompt — transcode to 24kHz mono wav (the model's native rate)
|
|
142
|
+
// first. Fall back to the raw clip only if ffmpeg is unavailable.
|
|
143
|
+
let wavPath = null;
|
|
144
|
+
let uploadPath = clipPath;
|
|
145
|
+
if (FFMPEG) {
|
|
146
|
+
try {
|
|
147
|
+
wavPath = path.join(TEMP_DIR, `voice-clone-${Date.now()}-${Math.random().toString(36).slice(2, 6)}.wav`);
|
|
148
|
+
execSync(`"${FFMPEG}" -i "${clipPath}" -ar 24000 -ac 1 -y "${wavPath}" 2>/dev/null`, { timeout: 30000 });
|
|
149
|
+
uploadPath = wavPath;
|
|
150
|
+
} catch (e) {
|
|
151
|
+
console.error("Voice clone transcode error:", e.message);
|
|
152
|
+
wavPath = null;
|
|
153
|
+
uploadPath = clipPath;
|
|
154
|
+
}
|
|
155
|
+
}
|
|
156
|
+
try {
|
|
157
|
+
const body = new FormData();
|
|
158
|
+
body.append("file", new Blob([fs.readFileSync(uploadPath)]), path.basename(uploadPath));
|
|
159
|
+
const res = await fetch(`${SPEECH_SERVICE_URL.replace(/\/$/, "")}/v1/voices`, {
|
|
160
|
+
method: "POST",
|
|
161
|
+
headers: { Authorization: `Bearer ${token}` },
|
|
162
|
+
body,
|
|
163
|
+
signal: AbortSignal.timeout(120000),
|
|
164
|
+
});
|
|
165
|
+
if (!res.ok) {
|
|
166
|
+
console.error(`Voice clone upload failed: HTTP ${res.status}`);
|
|
167
|
+
return null;
|
|
168
|
+
}
|
|
169
|
+
const payload = await res.json().catch(() => ({}));
|
|
170
|
+
return String(payload.voice_id || payload.id || "").trim() || null;
|
|
171
|
+
} catch (e) {
|
|
172
|
+
console.error("Voice clone error:", e.message);
|
|
173
|
+
return null;
|
|
174
|
+
} finally {
|
|
175
|
+
if (wavPath) { try { fs.unlinkSync(wavPath); } catch (e) {} }
|
|
176
|
+
}
|
|
177
|
+
}
|
|
178
|
+
|
|
132
179
|
// True when at least one textToVoice route can produce audio. Mirrors the
|
|
133
180
|
// pocket -> ElevenLabs -> say fallback chain; local Whisper is deliberately
|
|
134
181
|
// irrelevant here (hosted pods speak via the gateway without it).
|
|
@@ -203,4 +250,4 @@ async function synthSentenceMp3(text) {
|
|
|
203
250
|
}
|
|
204
251
|
}
|
|
205
252
|
|
|
206
|
-
module.exports = { transcribeAudio, transcribeAudioLocal, textToVoice, splitSentences, synthSentenceMp3, pocketToVoice, speechBearerToken, canSpeak, TTS_CMD };
|
|
253
|
+
module.exports = { transcribeAudio, transcribeAudioLocal, textToVoice, splitSentences, synthSentenceMp3, pocketToVoice, cloneVoiceFromClip, speechBearerToken, canSpeak, TTS_CMD };
|
package/core/router.js
CHANGED
|
@@ -13,7 +13,7 @@ const introFlow = require("./intro-flow");
|
|
|
13
13
|
const { send, deleteMessage } = require("./io");
|
|
14
14
|
const { currentState, saveState, autoLinkOwnerChannel } = require("./state");
|
|
15
15
|
const { admitRunContext, runAgent } = require("./runner");
|
|
16
|
-
const { transcribeAudio } = require("./media");
|
|
16
|
+
const { transcribeAudio, cloneVoiceFromClip } = require("./media");
|
|
17
17
|
const { vault } = require("./vault-store");
|
|
18
18
|
const { isOnboarded, handleOnboarding } = require("./onboarding");
|
|
19
19
|
const {
|
|
@@ -185,6 +185,13 @@ async function handleVoice(envelope) {
|
|
|
185
185
|
if (media.size && media.size > MAX_VOICE_SIZE) {
|
|
186
186
|
return send(`Voice note too large (${Math.round(media.size / 1024 / 1024)}MB). Max: ${MAX_VOICE_SIZE / 1024 / 1024}MB`);
|
|
187
187
|
}
|
|
188
|
+
// /voice clone armed this channel: the next clip is a cloning sample, not a
|
|
189
|
+
// prompt. Consume the flag (fresh only) before the normal transcribe path.
|
|
190
|
+
if (state.pendingVoiceClone) {
|
|
191
|
+
const fresh = Date.now() - state.pendingVoiceClone.at < 5 * 60 * 1000;
|
|
192
|
+
state.pendingVoiceClone = null;
|
|
193
|
+
if (fresh) return captureVoiceClone(envelope, media);
|
|
194
|
+
}
|
|
188
195
|
const runContext = admitRunContext("", state.currentSession.dir, envelope.messageId, {
|
|
189
196
|
purpose: "foreground",
|
|
190
197
|
attachments: [{ type: "voice", mimeType: media.mimeType || "audio/ogg", fileName: media.fileName || null }],
|
|
@@ -216,6 +223,11 @@ async function handleAudio(envelope) {
|
|
|
216
223
|
try {
|
|
217
224
|
const media = envelope.media?.[0];
|
|
218
225
|
if (!media) return;
|
|
226
|
+
if (state.pendingVoiceClone) {
|
|
227
|
+
const fresh = Date.now() - state.pendingVoiceClone.at < 5 * 60 * 1000;
|
|
228
|
+
state.pendingVoiceClone = null;
|
|
229
|
+
if (fresh) return captureVoiceClone(envelope, media);
|
|
230
|
+
}
|
|
219
231
|
const runContext = admitRunContext("", state.currentSession.dir, envelope.messageId, {
|
|
220
232
|
purpose: "foreground",
|
|
221
233
|
attachments: [{ type: "audio", mimeType: media.mimeType || null, fileName: media.fileName || null }],
|
|
@@ -233,6 +245,26 @@ async function handleAudio(envelope) {
|
|
|
233
245
|
} catch (err) { await send(`Audio failed: ${err.message}`); }
|
|
234
246
|
}
|
|
235
247
|
|
|
248
|
+
// A clip the owner sent right after /voice clone. Upload it to the speech
|
|
249
|
+
// gateway (which stores it on the TTS pod and returns a voice id), then set that
|
|
250
|
+
// id as the active voice. Never transcribed or sent to the agent.
|
|
251
|
+
async function captureVoiceClone(envelope, media) {
|
|
252
|
+
try {
|
|
253
|
+
envelope.adapter.typing(envelope.channelId).catch(() => {});
|
|
254
|
+
await send("🎙️ Got it — cloning your voice…");
|
|
255
|
+
const clipPath = await envelope.adapter.downloadMedia(media);
|
|
256
|
+
const voiceId = await cloneVoiceFromClip(clipPath);
|
|
257
|
+
try { fs.unlinkSync(clipPath); } catch (e) {}
|
|
258
|
+
if (!voiceId) {
|
|
259
|
+
return send("Couldn't process that clip for cloning. Send a clear 5–15s recording and try /voice clone again.");
|
|
260
|
+
}
|
|
261
|
+
require("./tts-voice").setVoice(voiceId, "your cloned voice");
|
|
262
|
+
return send("✅ Cloned. I'll speak in that voice from now on — change it anytime with /voice.");
|
|
263
|
+
} catch (err) {
|
|
264
|
+
await send(`Voice clone failed: ${err.message}`);
|
|
265
|
+
}
|
|
266
|
+
}
|
|
267
|
+
|
|
236
268
|
async function handlePhoto(envelope) {
|
|
237
269
|
if (isDuplicate(envelope)) return;
|
|
238
270
|
const state = currentState();
|
package/core/runner.js
CHANGED
|
@@ -1765,7 +1765,9 @@ const STREAM_PREVIEW_FOOTER = "\n\n⏳ working — /stop to cancel";
|
|
|
1765
1765
|
// the narration as it happens instead of silence. finalizeStreamPreview
|
|
1766
1766
|
// settles the message when the run ends.
|
|
1767
1767
|
function createStreamPreview({ state, store }) {
|
|
1768
|
-
let enabled = !!store
|
|
1768
|
+
let enabled = !!store
|
|
1769
|
+
&& (store?.adapter?.type || "") !== "voice"
|
|
1770
|
+
&& store?.adapter?.suppressStreamPreview !== true;
|
|
1769
1771
|
if (enabled) {
|
|
1770
1772
|
// Interim text is NOT vetted by guardOutboundReply (only the final answer
|
|
1771
1773
|
// is), so it must never reach a guarded external speaker. Fail closed if
|
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
|
|
3
|
+
// Live-read, persisted Pocket TTS voice selection.
|
|
4
|
+
//
|
|
5
|
+
// The active voice overrides the POCKET_TTS_VOICE env default and is read at
|
|
6
|
+
// synth time (media.js), so /voice takes effect immediately with no restart.
|
|
7
|
+
// The choice survives restarts via a small JSON file in the bot config dir;
|
|
8
|
+
// env stays the deploy-time default/floor and is used only when no override is
|
|
9
|
+
// set. A cloned-clip voice is just another voice string (a server-issued id).
|
|
10
|
+
|
|
11
|
+
const fs = require("fs");
|
|
12
|
+
const path = require("path");
|
|
13
|
+
const { CONFIG_DIR, POCKET_TTS_VOICE } = require("./config");
|
|
14
|
+
const { atomicWriteFileSync } = require("./fsutil");
|
|
15
|
+
|
|
16
|
+
const STORE = path.join(CONFIG_DIR, "tts-voice.json");
|
|
17
|
+
|
|
18
|
+
// Curated shortlist for the /voice picker. `id` is exactly what pocket-tts's
|
|
19
|
+
// get_state_for_audio_prompt resolves: the built-in "alba" shorthand or an
|
|
20
|
+
// hf:// reference into the kyutai/tts-voices repo. Every path here was verified
|
|
21
|
+
// against that repo's file tree, so the buttons always resolve.
|
|
22
|
+
const PRESETS = [
|
|
23
|
+
{ key: "alba", label: "Alba (default)", id: "alba" },
|
|
24
|
+
{ key: "announcer", label: "Alba · announcer", id: "hf://kyutai/tts-voices/alba-mackenna/announcer.wav" },
|
|
25
|
+
{ key: "merchant", label: "Alba · merchant", id: "hf://kyutai/tts-voices/alba-mackenna/merchant.wav" },
|
|
26
|
+
{ key: "expresso", label: "Expresso", id: "hf://kyutai/tts-voices/expresso/ex01-ex02_default_001_channel1_168s.wav" },
|
|
27
|
+
{ key: "enunciated", label: "Expresso · clear", id: "hf://kyutai/tts-voices/expresso/ex01-ex02_enunciated_001_channel1_432s.wav" },
|
|
28
|
+
{ key: "vctk-p225", label: "VCTK p225", id: "hf://kyutai/tts-voices/vctk/p225_023.wav" },
|
|
29
|
+
{ key: "vctk-p226", label: "VCTK p226", id: "hf://kyutai/tts-voices/vctk/p226_023.wav" },
|
|
30
|
+
];
|
|
31
|
+
|
|
32
|
+
// undefined = not loaded yet; null = loaded, no override (use env default);
|
|
33
|
+
// { voice, label } = an active override.
|
|
34
|
+
let cache;
|
|
35
|
+
|
|
36
|
+
function defaultVoice() {
|
|
37
|
+
return (POCKET_TTS_VOICE && String(POCKET_TTS_VOICE).trim()) || "alba";
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
function load() {
|
|
41
|
+
if (cache !== undefined) return cache;
|
|
42
|
+
try {
|
|
43
|
+
const raw = JSON.parse(fs.readFileSync(STORE, "utf-8"));
|
|
44
|
+
if (raw && typeof raw.voice === "string" && raw.voice.trim()) {
|
|
45
|
+
cache = { voice: raw.voice.trim(), label: typeof raw.label === "string" && raw.label ? raw.label : null };
|
|
46
|
+
return cache;
|
|
47
|
+
}
|
|
48
|
+
} catch (e) { /* no store yet, or malformed — fall back to default */ }
|
|
49
|
+
cache = null;
|
|
50
|
+
return cache;
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
// The active voice string handed to the speech gateway.
|
|
54
|
+
function getVoice() {
|
|
55
|
+
const s = load();
|
|
56
|
+
return (s && s.voice) || defaultVoice();
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
// Human-friendly label for the active voice (preset label, custom note, or id).
|
|
60
|
+
function getVoiceLabel() {
|
|
61
|
+
const s = load();
|
|
62
|
+
if (s) return s.label || s.voice;
|
|
63
|
+
const dv = defaultVoice();
|
|
64
|
+
const p = PRESETS.find((x) => x.id === dv || x.key === dv);
|
|
65
|
+
return p ? p.label : dv;
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
// True when the active voice is an explicit override (not the env default).
|
|
69
|
+
function isOverridden() {
|
|
70
|
+
return !!load();
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
function setVoice(voice, label = null) {
|
|
74
|
+
const v = String(voice || "").trim();
|
|
75
|
+
if (!v) throw new Error("voice is required");
|
|
76
|
+
cache = { voice: v, label: label ? String(label).trim() : null };
|
|
77
|
+
atomicWriteFileSync(STORE, JSON.stringify(cache, null, 2), { backup: false, mode: 0o600 });
|
|
78
|
+
return cache;
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
// Drop the override and fall back to the env/config default.
|
|
82
|
+
function clearVoice() {
|
|
83
|
+
cache = null;
|
|
84
|
+
try { fs.unlinkSync(STORE); } catch (e) { /* already absent */ }
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
function presets() { return PRESETS.slice(); }
|
|
88
|
+
function presetByKey(key) { return PRESETS.find((p) => p.key === String(key || "").toLowerCase()) || null; }
|
|
89
|
+
|
|
90
|
+
// Test seam: forget the in-memory cache so the next read re-consults disk.
|
|
91
|
+
function _resetCache() { cache = undefined; }
|
|
92
|
+
|
|
93
|
+
module.exports = {
|
|
94
|
+
getVoice, getVoiceLabel, isOverridden,
|
|
95
|
+
setVoice, clearVoice,
|
|
96
|
+
presets, presetByKey, defaultVoice,
|
|
97
|
+
STORE, _resetCache,
|
|
98
|
+
};
|
package/package.json
CHANGED
|
@@ -1,12 +1,13 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@inetafrica/open-claudia",
|
|
3
|
-
"version": "3.1.
|
|
3
|
+
"version": "3.1.13",
|
|
4
4
|
"description": "An always-on, provider-agnostic coding-agent harness for Claude Code and OpenAI Codex via chat",
|
|
5
5
|
"main": "bot.js",
|
|
6
6
|
"bin": {
|
|
7
7
|
"open-claudia": "./bin/cli.js"
|
|
8
8
|
},
|
|
9
9
|
"scripts": {
|
|
10
|
+
"test:kazee-call": "node test-kazee-call-audio.js && node test-kazee-call-janus.js && node test-kazee-call-webrtc.js && node test-kazee-call-speech.js && node test-kazee-call-manager.js",
|
|
10
11
|
"setup": "node setup.js",
|
|
11
12
|
"start": "node bot.js",
|
|
12
13
|
"pretest": "node test-provider-fixture.js && node test-provider-core-prompt.js && node test-turn-observer.js && node test-provider-prompt-parity.js && node test-provider-env-isolation.js && node test-provider-resume-parity.js && node test-provider-bootstrap.js && node test-provider-registry.js && node test-provider-stream-decoder.js && node test-provider-events.js && node test-provider-claude.js && node test-provider-codex.js && node test-provider-runner.js && node test-provider-refusal-recovery.js && node test-provider-run-lifecycle.js && node test-provider-gateway.js && node test-provider-side-chat.js && node test-single-runtime.js && node test-provider-migration-backup.js && node test-provider-state-migration.js && node test-provider-session-history.js && node test-provider-session-commands.js && node test-provider-compaction.js && node test-provider-scheduler.js && node test-credential-capture.js && node test-cursor-removal.js && node test-utility-provider-policy.js && node test-provider-subagent.js && node test-provider-recall.js && node test-provider-pack-review.js && node test-memory-mutation-queue.js && node test-provider-dream.js && node test-provider-enforcer.js && node test-provider-tool-hooks.js && node test-provider-boot-matrix.js && node test-provider-capabilities.js && node test-provider-language.js && node test-provider-coupling-audit.js && node test-provider-parity-e2e.js && node test-kazee-channel-health.js && node test-kazee-media-caption.js && node test-prompt-budget.js && node test-agency-ledger.js && node test-agency-deliberate.js && node test-guardrail.js && node test-spaces-comment-content.js && node test-boot-health.js && node test-lexical-floor.js && node test-safe-mode.js && node test-logout.js",
|
|
@@ -134,10 +135,15 @@
|
|
|
134
135
|
"engines": {
|
|
135
136
|
"node": ">=24.0.0"
|
|
136
137
|
},
|
|
138
|
+
"overrides": {
|
|
139
|
+
"ws": "^8.21.1"
|
|
140
|
+
},
|
|
137
141
|
"dependencies": {
|
|
138
142
|
"node-cron": "^4.2.1",
|
|
139
143
|
"node-telegram-bot-api": "^0.67.0",
|
|
144
|
+
"opusscript": "^0.1.1",
|
|
140
145
|
"socket.io-client": "^4.7.5",
|
|
141
|
-
"
|
|
146
|
+
"werift": "^0.23.0",
|
|
147
|
+
"ws": "^8.21.1"
|
|
142
148
|
}
|
|
143
149
|
}
|
|
@@ -195,7 +195,19 @@ assert.match(pkg.description, /provider-agnostic coding-agent harness/i);
|
|
|
195
195
|
// bot process advertised voice as enabled. Synthesis moves into the credentialed
|
|
196
196
|
// bot process behind a loopback "say" kind; the CLI now sends only text; the R8
|
|
197
197
|
// external-contact egress guard is preserved. Plus this version-pin bump.
|
|
198
|
-
|
|
198
|
+
// 3.1.11 was cut for the /voice picker + voice cloning feature but its build
|
|
199
|
+
// failed on this very version pin (still 3.1.10): the package.json bump landed
|
|
200
|
+
// without the matching pin update, so the docker stage was skipped and no image
|
|
201
|
+
// was published. Never retag a failed version — re-cut as +1.
|
|
202
|
+
// 3.1.12 approved by Sumeet 2026-07-21: re-cut of 3.1.11. Owner-gated /voice
|
|
203
|
+
// preset picker over the Pocket TTS catalog (live-read, persists across restart,
|
|
204
|
+
// Custom hf:// escape hatch) plus voice cloning — send a clip and it's transcoded
|
|
205
|
+
// (ogg->24kHz mono wav) and registered via the speech gateway /v1/voices as the
|
|
206
|
+
// active stored:<digest> voice. Plus this version-pin bump.
|
|
207
|
+
// 3.1.13 approved by Sumeet 2026-07-22 ("work through till all your plan is
|
|
208
|
+
// completed"): owner-only incoming Kazee audio calls bridge Janus VideoRoom
|
|
209
|
+
// RTP to hosted streaming STT, the normal agent turn, and streamed Pocket TTS.
|
|
210
|
+
assert.strictEqual(pkg.version, "3.1.13", "release version must remain unchanged without explicit approval");
|
|
199
211
|
for (const keyword of ["claude", "codex", "coding-agent", "provider-agnostic"]) {
|
|
200
212
|
assert.ok(pkg.keywords.includes(keyword), `package keyword missing: ${keyword}`);
|
|
201
213
|
}
|