@inetafrica/open-claudia 3.5.47 → 3.5.58
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +62 -10
- package/bin/cli.js +10 -3
- package/bot.js +11 -0
- package/channels/kazee/adapter.js +220 -23
- package/channels/kazee/call/camera.js +233 -0
- package/channels/kazee/call/job-steps.js +81 -0
- package/channels/kazee/call/manager.js +1056 -27
- package/channels/kazee/call/miss-capture.js +82 -0
- package/channels/kazee/call/quick-voice.js +610 -0
- package/channels/kazee/call/session.js +787 -36
- package/channels/kazee/call/thread-context.js +100 -0
- package/channels/kazee/catch-up.js +275 -0
- package/core/actions.js +13 -4
- package/core/handlers.js +14 -16
- package/core/identity.js +15 -1
- package/core/loopback.js +4 -1
- package/core/media.js +39 -1
- package/core/run-context.js +27 -3
- package/core/runner.js +70 -1
- package/core/state.js +6 -2
- package/core/switchboard/brain-link.js +161 -0
- package/core/switchboard/call-host.js +298 -0
- package/core/switchboard/handoff.js +80 -0
- package/core/switchboard/ipc.js +130 -0
- package/core/switchboard/supervisor.js +225 -0
- package/core/system-prompt.js +44 -4
- package/core/transcripts.js +39 -0
- package/core/tts-voice.js +62 -4
- package/core/turn-journal.js +23 -1
- package/package.json +2 -2
- package/test-call-speech.js +7 -0
- package/test-fixtures/switchboard-fake-brain.js +20 -0
- package/test-kazee-call-manager.js +22 -4
- package/test-kazee-call-session.js +345 -8
- package/test-provider-compaction.js +8 -0
- package/test-provider-language.js +17 -9
- package/test-provider-prompt-parity.js +1 -0
- package/test-provider-runner.js +2 -0
- package/test-run-lock.js +24 -0
|
@@ -5,6 +5,9 @@ const { CallLeaseHeartbeat } = require("./lease");
|
|
|
5
5
|
const { publicCallIdentity, requestRoomAccess, supportedIceServers } = require("./access");
|
|
6
6
|
const { canonicalForChannel } = require("../../../core/identity");
|
|
7
7
|
const { KazeeCallSession } = require("./session");
|
|
8
|
+
const quickVoice = require("./quick-voice");
|
|
9
|
+
const threadContext = require("./thread-context");
|
|
10
|
+
const { stepForTool } = require("./job-steps");
|
|
8
11
|
|
|
9
12
|
const DEFAULT_JANUS_URL = "wss://janus.kazee.africa/ws";
|
|
10
13
|
const DEFAULT_ICE_SERVERS = [
|
|
@@ -14,6 +17,145 @@ const DEFAULT_ICE_SERVERS = [
|
|
|
14
17
|
// FCM push TTL is 60s and chat-central treats ringing logs as fresh for 90s.
|
|
15
18
|
// This is the outbound invitation deadline; terminal server events end it early.
|
|
16
19
|
const RING_TIMEOUT_MS = 60_000;
|
|
20
|
+
const qlog = (msg) => console.log(`${new Date().toISOString()} Kazee call quick: ${msg}`);
|
|
21
|
+
// The quick line must be ready this soon after the words, or it is skipped:
|
|
22
|
+
// by then the full agent is close to speaking itself.
|
|
23
|
+
const QUICK_LINE_MAX_MS = 3500;
|
|
24
|
+
// While the full agent works: a "still on it" line after this much quiet,
|
|
25
|
+
// at most a few per turn, never after the answer.
|
|
26
|
+
const QUICK_FILLER_QUIET_MS = 8000;
|
|
27
|
+
const QUICK_FILLER_MAX = 3;
|
|
28
|
+
const QUICK_TICK_MS = 500;
|
|
29
|
+
// The phone keeps ringing while the quick voice writes an opening line from
|
|
30
|
+
// the recent chat (~1.8 s, measured 11 Oct); past this, the fixed greeting.
|
|
31
|
+
const QUICK_GREETING_MAX_MS = 3500;
|
|
32
|
+
// A thread call reads the thread first (its title from the server, its last
|
|
33
|
+
// brief, more of its turns), so its opening line gets longer to be written.
|
|
34
|
+
const QUICK_THREAD_GREETING_MAX_MS = 6000;
|
|
35
|
+
// A held answer waits at most this long for its lead-in; a plain one after.
|
|
36
|
+
// Haiku 4.5 wrote them in 0.8-1.7 s on 11 Oct, and most are written while
|
|
37
|
+
// the line before them still plays.
|
|
38
|
+
const QUICK_LEAD_IN_MAX_MS = 1500;
|
|
39
|
+
// The quick voice's first sentence for an answer it says (_relay) must be
|
|
40
|
+
// ready this soon, or the answer is said as written. Its short lines took
|
|
41
|
+
// 1.6-2.5 s on Opus 5.5 (11 Oct 18:21 call).
|
|
42
|
+
const QUICK_RELAY_MAX_MS = 6000;
|
|
43
|
+
// When the quick voice says the answers, nothing else will speak first, so
|
|
44
|
+
// its holding line is said even when this late; past it, the "still on it"
|
|
45
|
+
// line takes over (fillerQuietMs). 11 Oct 18:58: a line ready at 3.8 s was
|
|
46
|
+
// skipped under the 3.5 s limit, and the caller heard nothing for 14 s.
|
|
47
|
+
const QUICK_RELAY_LINE_MAX_MS = 8000;
|
|
48
|
+
// Words that already say which question they answer need no lead-in.
|
|
49
|
+
const LEAD_IN_ALREADY = /^(?:back to|on (?:the|your)|about (?:the|your)|as for|and on|now,? on)\b/i;
|
|
50
|
+
// Words at which a message said while work runs is never only a check-in
|
|
51
|
+
// (_routeBusy): "You there?" is 2, "Hey, are you still there? I can't hear
|
|
52
|
+
// anything" 9, the 11 Oct complaint that hid a request 70.
|
|
53
|
+
const LONG_UTTERANCE_WORDS = 20;
|
|
54
|
+
// After the goodbye of a caller who wants the call ended while work is open:
|
|
55
|
+
// this long with nothing playing, then the call ends (_hangUpWhenDone).
|
|
56
|
+
const HANG_UP_QUIET_MS = 2000;
|
|
57
|
+
const countWords = (text) => String(text || "").trim().split(/\s+/).filter(Boolean).length;
|
|
58
|
+
|
|
59
|
+
// Runs fn in the call's chat context, as the full agent's turns do. A call
|
|
60
|
+
// placed inside a thread runs in that thread's conversation.
|
|
61
|
+
function inChat({ adapter, chatId, userId, threadId = null }, fn) {
|
|
62
|
+
const { runInChat } = require("../../../core/context");
|
|
63
|
+
const { conversationKeyFor, roomKeyFor } = require("../../../core/identity");
|
|
64
|
+
const transport = adapter?.type || "kazee";
|
|
65
|
+
const thread = threadId ? String(threadId) : null;
|
|
66
|
+
return runInChat({
|
|
67
|
+
adapter, channelId: String(chatId), userId: String(userId), transport, threadId: thread,
|
|
68
|
+
canonicalUserId: canonicalForChannel(transport, String(userId)),
|
|
69
|
+
conversationKey: conversationKeyFor(transport, String(chatId), thread),
|
|
70
|
+
roomKey: roomKeyFor(transport, String(chatId)),
|
|
71
|
+
}, fn);
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
// A thread id as Kazee sends it (a Mongo ObjectId), else none.
|
|
75
|
+
function threadIdOf(value) {
|
|
76
|
+
const id = String(value || "").trim();
|
|
77
|
+
return /^[a-f0-9]{24}$/i.test(id) ? id : null;
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
// The provider the call's chat runs on: the quick voice uses its CLI and
|
|
81
|
+
// credentials, as the full agent's turns do.
|
|
82
|
+
function chatProvider(target) {
|
|
83
|
+
const id = inChat(target, () => {
|
|
84
|
+
const state = require("../../../core/state");
|
|
85
|
+
return state.getActiveProvider(state.currentState());
|
|
86
|
+
});
|
|
87
|
+
return id ? require("../../../core/providers").registry.getProvider(id) : null;
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
// The end of the call's chat, for the opening line. A chat turn files its
|
|
91
|
+
// transcript under the conversation's own project (run-context's
|
|
92
|
+
// projectSnapshot), not the shared workspace session, so read it from there.
|
|
93
|
+
function recentChat(target, limit = quickVoice.GREETING_ENTRIES) {
|
|
94
|
+
return inChat(target, () => {
|
|
95
|
+
const current = require("../../../core/state").currentState();
|
|
96
|
+
const transcripts = require("../../../core/transcripts");
|
|
97
|
+
const project = require("../../../core/run-context").projectSnapshot(current, current.currentSession?.dir || null, undefined);
|
|
98
|
+
return transcripts.recentTranscriptEntries({ limit }, transcripts.transcriptStateForRun({ project }, current));
|
|
99
|
+
});
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
// The newest archived brief of the conversation target names (its thread's
|
|
103
|
+
// own, for a thread call): thread-context.js newestBrief.
|
|
104
|
+
function conversationBrief(target) {
|
|
105
|
+
return inChat(target, () => {
|
|
106
|
+
const current = require("../../../core/state").currentState();
|
|
107
|
+
const transcripts = require("../../../core/transcripts");
|
|
108
|
+
const project = require("../../../core/run-context").projectSnapshot(current, current.currentSession?.dir || null, undefined);
|
|
109
|
+
const info = transcripts.transcriptProjectInfo(transcripts.transcriptStateForRun({ project }, current));
|
|
110
|
+
return threadContext.newestBrief(info?.transcriptPath);
|
|
111
|
+
});
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
// What a call knows of its chat when it starts. A plain call: the end of the
|
|
115
|
+
// chat, for the opening line. A thread call (thread-context.js): the thread's
|
|
116
|
+
// title, its last brief and more of its turns, for the opening line, plus the
|
|
117
|
+
// note every full-agent turn on the call carries.
|
|
118
|
+
async function greetingContext(target) {
|
|
119
|
+
const threadId = threadIdOf(target?.threadId);
|
|
120
|
+
if (!threadId) return recentChat(target);
|
|
121
|
+
const scoped = { ...target, threadId };
|
|
122
|
+
let entries = [];
|
|
123
|
+
try { entries = await recentChat(scoped, threadContext.THREAD_ENTRIES); } catch (_) {}
|
|
124
|
+
let brief = null;
|
|
125
|
+
try { brief = conversationBrief(scoped); } catch (_) {}
|
|
126
|
+
const title = await threadContext.threadTitle(target.adapter, target.chatId, threadId);
|
|
127
|
+
return {
|
|
128
|
+
entries,
|
|
129
|
+
thread: {
|
|
130
|
+
id: threadId,
|
|
131
|
+
title,
|
|
132
|
+
briefAt: brief?.at || null,
|
|
133
|
+
brief: brief ? threadContext.briefHead(brief.text, threadContext.BRIEF_QUICK_CHARS) : "",
|
|
134
|
+
agentNote: threadContext.agentNote({ title, brief }),
|
|
135
|
+
},
|
|
136
|
+
};
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
// Work asked for on a call that then ended goes on in the call's chat: a
|
|
140
|
+
// wake-up due now, so it runs as an ordinary chat turn (a full answer in chat
|
|
141
|
+
// form, not phone brevity) and survives a restart. Calls are owner-only.
|
|
142
|
+
// The scheduler takes the job's project, session and model/effort from the
|
|
143
|
+
// current chat, so the job is made inside the call's chat (11 Oct: made
|
|
144
|
+
// outside it, the job got the default "tickets" session and failed).
|
|
145
|
+
function carryOnInChat({ adapter, chatId, userId, prompt, threadId = null }) {
|
|
146
|
+
const transport = adapter?.type || "kazee";
|
|
147
|
+
const canonicalUserId = canonicalForChannel(transport, String(userId));
|
|
148
|
+
const owner = String(userId) === String(adapter?.ownerUserId || "");
|
|
149
|
+
// A thread call's work carries on in its thread.
|
|
150
|
+
const thread = threadIdOf(threadId);
|
|
151
|
+
return inChat({ adapter, chatId, userId, threadId: thread }, () => require("../../../core/scheduler").addJob({
|
|
152
|
+
kind: "wakeup", adapter: adapter?.id || null, adapterType: transport,
|
|
153
|
+
channelId: String(chatId), threadId: thread, canonicalUserId, userId: String(userId),
|
|
154
|
+
createdBy: { canonicalUserId, userId: String(userId), channelId: String(chatId), adapterType: transport, owner, name: null, at: new Date().toISOString() },
|
|
155
|
+
prompt, label: "Carry on from the call", source: "call", fireAt: Date.now(),
|
|
156
|
+
}));
|
|
157
|
+
}
|
|
158
|
+
|
|
17
159
|
const OUTGOING_GREETING = "Hey, it's Claudia. Can you hear me okay?";
|
|
18
160
|
// Inbound must greet too: callers get silence until they speak first, which
|
|
19
161
|
// reads as a dead call (live test 2026-08-08 — media was up, caller hung up).
|
|
@@ -21,7 +163,9 @@ const INCOMING_GREETING = "Hey, it's Claudia — you're through. Can you hear me
|
|
|
21
163
|
|
|
22
164
|
class KazeeCallManager {
|
|
23
165
|
constructor({ adapter, SessionImpl = KazeeCallSession, janusUrl = process.env.KAZEE_JANUS_URL || DEFAULT_JANUS_URL,
|
|
24
|
-
iceServers = DEFAULT_ICE_SERVERS, deviceId = randomUUID()
|
|
166
|
+
iceServers = DEFAULT_ICE_SERVERS, deviceId = randomUUID(), resolveQuickProvider = chatProvider, quickSpawn = undefined,
|
|
167
|
+
quickSettings = quickVoice.quickVoiceSettings, quickTiming = {}, resolveGreetingContext = greetingContext, carryOn = carryOnInChat,
|
|
168
|
+
dispatchJob = null } = {}) {
|
|
25
169
|
this.adapter = adapter;
|
|
26
170
|
this.SessionImpl = SessionImpl;
|
|
27
171
|
this.janusUrl = janusUrl;
|
|
@@ -30,6 +174,15 @@ class KazeeCallManager {
|
|
|
30
174
|
this.deviceId = deviceId;
|
|
31
175
|
this._pending = null;
|
|
32
176
|
this._ringTimer = null;
|
|
177
|
+
this.resolveQuickProvider = resolveQuickProvider;
|
|
178
|
+
this.quickSpawn = quickSpawn;
|
|
179
|
+
this.quickSettings = quickSettings;
|
|
180
|
+
this.quickTiming = { lineMaxMs: QUICK_LINE_MAX_MS, fillerQuietMs: QUICK_FILLER_QUIET_MS, fillerMax: QUICK_FILLER_MAX, tickMs: QUICK_TICK_MS, greetingMaxMs: QUICK_GREETING_MAX_MS, threadGreetingMaxMs: QUICK_THREAD_GREETING_MAX_MS, leadInMaxMs: QUICK_LEAD_IN_MAX_MS, relayMaxMs: QUICK_RELAY_MAX_MS, relayLineMaxMs: QUICK_RELAY_LINE_MAX_MS, ...quickTiming };
|
|
181
|
+
this.resolveGreetingContext = resolveGreetingContext;
|
|
182
|
+
this.carryOn = carryOn;
|
|
183
|
+
// Where a full-agent job goes: this process's router, or (when the
|
|
184
|
+
// switchboard holds the call) the brain across the link.
|
|
185
|
+
this.dispatchJob = dispatchJob || ((envelope) => this.adapter._emit("message", envelope));
|
|
33
186
|
}
|
|
34
187
|
|
|
35
188
|
wire(socket) {
|
|
@@ -86,9 +239,13 @@ class KazeeCallManager {
|
|
|
86
239
|
}
|
|
87
240
|
|
|
88
241
|
_sessionEnded(session, reason) {
|
|
242
|
+
// Their line dropped and never came back (session.js _dropTick): the work
|
|
243
|
+
// carries on in the chat, and the chat says why the call ended.
|
|
244
|
+
if (reason === "connection-lost") this._lineLost(session);
|
|
89
245
|
// Several terminal paths clear this.session before close() calls back.
|
|
90
246
|
// Invalidate the closed call's work even when it no longer owns the slot.
|
|
91
|
-
session?._voiceTurn
|
|
247
|
+
for (const job of [...(session?._jobs || []), session?._voiceTurn].filter(Boolean)) job.controller.abort("call-ended");
|
|
248
|
+
this._stopQuickVoice(session);
|
|
92
249
|
if (this.session !== session) return;
|
|
93
250
|
this._detachSession(session);
|
|
94
251
|
if (this.adapter._socket?.connected) this.adapter._socket.emit("call:end", {
|
|
@@ -97,7 +254,8 @@ class KazeeCallManager {
|
|
|
97
254
|
}
|
|
98
255
|
|
|
99
256
|
_detachSession(session) {
|
|
100
|
-
session?._voiceTurn
|
|
257
|
+
for (const job of [...(session?._jobs || []), session?._voiceTurn].filter(Boolean)) job.controller.abort("call-ended");
|
|
258
|
+
this._stopQuickVoice(session);
|
|
101
259
|
if (this.session === session) this.session = null;
|
|
102
260
|
}
|
|
103
261
|
|
|
@@ -132,7 +290,9 @@ class KazeeCallManager {
|
|
|
132
290
|
if (!callerId || !chatId || !Number.isSafeInteger(roomId) || roomId <= 0) return reject("invalid_call");
|
|
133
291
|
if (payload.roomAdmissionRequired !== undefined && typeof payload.roomAdmissionRequired !== 'boolean') return reject("invalid_call");
|
|
134
292
|
if (callerId !== String(this.adapter.ownerUserId || "")) return reject("not_allowed");
|
|
135
|
-
|
|
293
|
+
// A video call is answered too: the caller's camera becomes stills the
|
|
294
|
+
// full agent can look at (camera.js); the bot itself sends only its voice.
|
|
295
|
+
if (callType !== "audio" && callType !== "video") return reject("unsupported_media");
|
|
136
296
|
if (this.session || this._pending) return reject("busy");
|
|
137
297
|
const pending = { identity: { ...identity, chatId }, controller: new AbortController() };
|
|
138
298
|
this._pending = pending;
|
|
@@ -150,17 +310,29 @@ class KazeeCallManager {
|
|
|
150
310
|
session = new this.SessionImpl({ roomId, chatId, callerId, botUserId: this.adapter.botUserId,
|
|
151
311
|
...this._sessionOptions(config, access, boundIdentity, pending.controller.signal), greeting: INCOMING_GREETING,
|
|
152
312
|
onTranscript: (text, meta) => this._transcript(text, { ...payload, ...boundIdentity }, meta, session),
|
|
313
|
+
onEarlyWords: (text, meta) => this._earlyWords(text, meta, session),
|
|
314
|
+
onUtteranceDropped: (utteranceId) => this._dropEarly(session, utteranceId, "no turn"),
|
|
153
315
|
onSpeechStart: () => this._speechStart(session),
|
|
316
|
+
onSpeechNoise: () => this._speechNoise(session),
|
|
154
317
|
onMediaReady: () => this._mediaReady(session),
|
|
155
318
|
onEnded: reason => { session?._leaseHeartbeat?.close(); pending.controller.abort(); this._sessionEnded(session, reason); },
|
|
156
319
|
});
|
|
157
320
|
session.callIdentity = boundIdentity;
|
|
158
321
|
session.chatId = chatId;
|
|
322
|
+
// Rung from inside a thread (chat-central passes the checked root).
|
|
323
|
+
session.threadId = threadIdOf(payload.threadRootId);
|
|
159
324
|
this.session = session;
|
|
325
|
+
this._callContext(session, chatId, callerId);
|
|
326
|
+
this._startQuickVoice(session, chatId, callerId);
|
|
327
|
+
const greeting = this._prepareGreeting(session, chatId, callerId);
|
|
160
328
|
this._manageLease(session, access);
|
|
161
329
|
await session.start();
|
|
162
330
|
if (this.session !== session || session.closed || pending.controller.signal.aborted) return;
|
|
163
|
-
|
|
331
|
+
// Answer once the opening line is ready, so the caller hears ringing,
|
|
332
|
+
// not silence, while it is written.
|
|
333
|
+
if (greeting) await greeting;
|
|
334
|
+
if (this.session !== session || session.closed || pending.controller.signal.aborted) return;
|
|
335
|
+
this.adapter._socket?.emit("call:accept", { ...boundIdentity, toUserId: callerId, callType, chatId, roomId });
|
|
164
336
|
session._accepted = true;
|
|
165
337
|
if (session._mediaReady) session._leaseHeartbeat?.activate();
|
|
166
338
|
} catch (_) {
|
|
@@ -176,9 +348,11 @@ class KazeeCallManager {
|
|
|
176
348
|
// the callee (socket + FCM/VoIP push). The callee's publisher appearing in
|
|
177
349
|
// the room is what completes the media leg — the existing auto-subscribe in
|
|
178
350
|
// the session handles that, so acceptance needs no extra media work here.
|
|
179
|
-
async outgoing({ chatId, toUserId } = {}) {
|
|
351
|
+
async outgoing({ chatId, toUserId, reason, threadId } = {}) {
|
|
180
352
|
const target = String(toUserId || this.adapter.ownerUserId || "");
|
|
181
353
|
const chat = String(chatId || "");
|
|
354
|
+
// Placed from a thread turn: the call belongs to that thread.
|
|
355
|
+
const thread = threadIdOf(threadId);
|
|
182
356
|
if (!target || !chat) throw new Error("missing chatId/toUserId");
|
|
183
357
|
if (target !== String(this.adapter.ownerUserId || "")) throw new Error("Calls are restricted to the owner");
|
|
184
358
|
if (this.session || this._pending) throw new Error("busy: a call session is already active");
|
|
@@ -197,20 +371,29 @@ class KazeeCallManager {
|
|
|
197
371
|
session = new this.SessionImpl({ roomId, chatId: chat, callerId: target, botUserId: this.adapter.botUserId,
|
|
198
372
|
...this._sessionOptions(config, access, identity, pending.controller.signal), asCaller: true, greeting: OUTGOING_GREETING,
|
|
199
373
|
onTranscript: (text, meta) => this._transcript(text, remote, meta, session),
|
|
374
|
+
onEarlyWords: (text, meta) => this._earlyWords(text, meta, session),
|
|
375
|
+
onUtteranceDropped: (utteranceId) => this._dropEarly(session, utteranceId, "no turn"),
|
|
200
376
|
onSpeechStart: () => this._speechStart(session),
|
|
377
|
+
onSpeechNoise: () => this._speechNoise(session),
|
|
201
378
|
onMediaReady: () => this._mediaReady(session),
|
|
202
379
|
onEnded: reason => { session?._leaseHeartbeat?.close(); pending.controller.abort(); this._clearRingTimer(); this._sessionEnded(session, reason); },
|
|
203
380
|
});
|
|
204
381
|
session.callIdentity = identity;
|
|
205
382
|
session.chatId = chat;
|
|
383
|
+
session.threadId = thread;
|
|
206
384
|
this.session = session;
|
|
385
|
+
this._callContext(session, chat, target);
|
|
386
|
+
this._startQuickVoice(session, chat, target);
|
|
387
|
+
// Written while the phone rings; the session speaks it once the callee
|
|
388
|
+
// picks up (no accept to hold on this side).
|
|
389
|
+
this._prepareGreeting(session, chat, target, { outgoing: true, reason });
|
|
207
390
|
this._manageLease(session, access);
|
|
208
391
|
await session.start();
|
|
209
392
|
if (this.session !== session || session.closed || pending.controller.signal.aborted) throw new Error("Call cancelled");
|
|
210
|
-
this.adapter._socket.emit("call:invite", { ...identity, toUserId: target, callType: "audio", chatId: chat });
|
|
393
|
+
this.adapter._socket.emit("call:invite", { ...identity, toUserId: target, callType: "audio", chatId: chat, ...(thread ? { threadRootId: thread } : {}) });
|
|
211
394
|
this._ringTimer = setTimeout(() => { this._ringTimer = null; this._ringTimeout(target, chat, roomId).catch(() => console.error("Kazee call ring timeout failed")); }, RING_TIMEOUT_MS);
|
|
212
395
|
this._ringTimer.unref?.();
|
|
213
|
-
return { ...identity, chatId: chat, toUserId: target };
|
|
396
|
+
return { ...identity, chatId: chat, toUserId: target, ...(thread ? { threadId: thread } : {}) };
|
|
214
397
|
} catch (error) {
|
|
215
398
|
this._detachSession(session);
|
|
216
399
|
if (session) await session.close("connect-failed").catch(() => {});
|
|
@@ -242,11 +425,395 @@ class KazeeCallManager {
|
|
|
242
425
|
}
|
|
243
426
|
|
|
244
427
|
_speechStart(session) {
|
|
245
|
-
|
|
428
|
+
// Hold every job's speech while the caller talks; what they say next
|
|
429
|
+
// decides whether it is let through, waits, or is dropped (_routeBusy).
|
|
430
|
+
for (const job of session?._jobs || []) job.muted = true;
|
|
246
431
|
// Sound alone is not a new request. Cancel queued work only once a new
|
|
247
432
|
// complete transcript exists; a cough must not discard an accepted ask.
|
|
248
433
|
}
|
|
249
434
|
|
|
435
|
+
// The sound that muted the turn had no words in it: not a new request, so
|
|
436
|
+
// the answer being worked on may speak again (11 Oct: wind while walking
|
|
437
|
+
// muted a lookup's answer for good, and the caller heard nothing).
|
|
438
|
+
_speechNoise(session) {
|
|
439
|
+
if (this.session !== session || session.closed) return;
|
|
440
|
+
if ((session._jobs || []).some((job) => job.muted && !job.controller.signal.aborted)) {
|
|
441
|
+
this._releaseJobs(session);
|
|
442
|
+
console.log(`${new Date().toISOString()} Kazee call noise: muted turn resumed`);
|
|
443
|
+
}
|
|
444
|
+
// A sound with no words cut an answer (or the greeting) off: say the rest.
|
|
445
|
+
this._resumeCut(session, "noise");
|
|
446
|
+
}
|
|
447
|
+
|
|
448
|
+
// Say again the words a caller cut off by accident (session.js _bargeIn):
|
|
449
|
+
// after a noise, a check-in, "say again", or words that need no answer.
|
|
450
|
+
// plain: the quick voice already led in ("Sorry, as I was saying.").
|
|
451
|
+
// Only a real new question drops them (_dropCut).
|
|
452
|
+
_resumeCut(session, why, { plain = false } = {}) {
|
|
453
|
+
if (this.session !== session || session.closed) return false;
|
|
454
|
+
const cut = session.takeCutSpeech?.();
|
|
455
|
+
if (!cut?.text) return false;
|
|
456
|
+
const text = plain || cut.fromStart ? cut.text : `Sorry, as I was saying. ${cut.text}`;
|
|
457
|
+
qlog(`resume cut (${why}) chars=${cut.text.length}${cut.fromStart ? " from start" : ""}`);
|
|
458
|
+
this._rememberSpeech(session, "assistant", cut.text);
|
|
459
|
+
session.speak(text, { resumable: true }).catch(() => {});
|
|
460
|
+
return true;
|
|
461
|
+
}
|
|
462
|
+
|
|
463
|
+
_dropCut(session, why) {
|
|
464
|
+
const cut = session.takeCutSpeech?.();
|
|
465
|
+
if (cut?.text) qlog(`cut dropped (${why}) chars=${cut.text.length}`);
|
|
466
|
+
}
|
|
467
|
+
|
|
468
|
+
// The caller asked to end the call ([[BYE]]). Right after the goodbye, or
|
|
469
|
+
// once the work they asked for is done and its answer said ([[JOB]]
|
|
470
|
+
// [[BYE]]). 11 Oct: "Hanging up now" and "Okay, talk soon" were said
|
|
471
|
+
// twice and the call stayed up until they hung up themselves.
|
|
472
|
+
async _hangUp(session, why) {
|
|
473
|
+
if (this.session !== session || session.closed) return;
|
|
474
|
+
session._hangUpPending = false;
|
|
475
|
+
qlog(`hang up (${why})`);
|
|
476
|
+
await session.close("goodbye").catch(() => {});
|
|
477
|
+
}
|
|
478
|
+
|
|
479
|
+
// Wanting the call ended while work is open ([[JOB]] [[BYE]], or [[BYE]]
|
|
480
|
+
// / [[ADD]] [[BYE]] while Claudia works): once the goodbye has played and
|
|
481
|
+
// the line is quiet for a moment, the call ends and the work carries on in
|
|
482
|
+
// the chat (_carryOn). 11 Oct 15:13: waiting for the work first kept the
|
|
483
|
+
// caller on the line 64 s ("You still didn't hang up").
|
|
484
|
+
_hangUpWhenDone(session) {
|
|
485
|
+
if (session._hangUpPending) return;
|
|
486
|
+
session._hangUpPending = true;
|
|
487
|
+
qlog("hang up after the goodbye, work carries on in the chat");
|
|
488
|
+
(async () => {
|
|
489
|
+
// Quiet for a moment, not just between two sentences of a line.
|
|
490
|
+
let quietSince = 0;
|
|
491
|
+
while (session._hangUpPending && this.session === session && !session.closed) {
|
|
492
|
+
await new Promise((resolve) => { const timer = setTimeout(resolve, this.quickTiming.tickMs); timer.unref?.(); });
|
|
493
|
+
const busy = session.speechBusy?.() || session.callerTalking?.();
|
|
494
|
+
if (busy) { quietSince = 0; continue; }
|
|
495
|
+
quietSince ||= Date.now();
|
|
496
|
+
if (Date.now() - quietSince < HANG_UP_QUIET_MS || !session._hangUpPending) continue;
|
|
497
|
+
this._carryOn(session);
|
|
498
|
+
await this._hangUp(session, "work carries on in the chat");
|
|
499
|
+
}
|
|
500
|
+
})().catch((error) => qlog(`hang up failed: ${error.message}`));
|
|
501
|
+
}
|
|
502
|
+
|
|
503
|
+
// The call is ending with work still open. Each job not done yet is stopped
|
|
504
|
+
// and asked again as a chat turn, so it runs once, in full, and answers in
|
|
505
|
+
// chat form. A running one too: a call run keeps lookups short and offers
|
|
506
|
+
// longer work "after the call", so left to finish it only promised (11 Oct
|
|
507
|
+
// 17:05: "Okay, just fix it and then call me back" got "I'm on it" in the
|
|
508
|
+
// chat and nothing ran). An answer still held back is posted as it is.
|
|
509
|
+
_carryOn(session, { dropped = false } = {}) {
|
|
510
|
+
const asked = [];
|
|
511
|
+
for (const job of this._openJobs(session)) {
|
|
512
|
+
if (job.done) {
|
|
513
|
+
const held = job.pending.splice(0).join(" ").trim();
|
|
514
|
+
if (held) this._postToChat(session, held);
|
|
515
|
+
} else {
|
|
516
|
+
job.started = job.running;
|
|
517
|
+
this._dropJob(session, job, "cancelled");
|
|
518
|
+
asked.push(job);
|
|
519
|
+
}
|
|
520
|
+
}
|
|
521
|
+
if (!asked.length) return 0;
|
|
522
|
+
qlog(`carried on in the chat: asked=${asked.length} running=${asked.filter((job) => job.started).length}`);
|
|
523
|
+
const name = String(asked[0]?.incoming?.fromUserName || "").trim() || "The caller";
|
|
524
|
+
const call = (session._voiceHistory || []).slice(-6)
|
|
525
|
+
.map((entry) => `${entry.role === "user" ? name : "Claudia"}: ${entry.text.slice(-600)}`).join("\n");
|
|
526
|
+
for (const job of asked) {
|
|
527
|
+
const prompt = [
|
|
528
|
+
dropped
|
|
529
|
+
? `[Carried on from a call] On the Kazee call that just ended, ${name} asked for this, then their line dropped and did not come back, so the call ended and the work carries on here. Do it now and report the result in this chat. If they asked to be called back when it is done, place the call once it is ready.`
|
|
530
|
+
: `[Carried on from a call] On the Kazee call that just ended, ${name} asked for this and wanted the call ended, so it was hung up and the work carries on here. Do it now and report the result in this chat. If they asked to be called back when it is done, place the call once it is ready.`,
|
|
531
|
+
`Asked on the call: "${job.question}"`,
|
|
532
|
+
job.started ? "Work on it had started during the call and was stopped; anything it already changed is in place." : "",
|
|
533
|
+
call ? `The call's last words (most recent last):\n${call}` : "",
|
|
534
|
+
].filter(Boolean).join("\n");
|
|
535
|
+
try { this.carryOn({ adapter: this.adapter, chatId: session.chatId, userId: job.incoming?.fromUserId || session.callerId, prompt, ...(session.threadId ? { threadId: session.threadId } : {}) }); }
|
|
536
|
+
catch (error) { qlog(`carry on failed: ${error.message}`); }
|
|
537
|
+
}
|
|
538
|
+
const short = (text) => (text.length > 120 ? `${text.slice(0, 117)}...` : text);
|
|
539
|
+
const lead = dropped ? "Your line dropped and didn't come back, so I ended the call." : "Call ended.";
|
|
540
|
+
this._postToChat(session, `${lead} Carrying on here: ${asked.map((job) => `"${short(job.question)}"`).join("; ")}`);
|
|
541
|
+
return asked.length;
|
|
542
|
+
}
|
|
543
|
+
|
|
544
|
+
_lineLost(session) {
|
|
545
|
+
const carried = this._carryOn(session, { dropped: true });
|
|
546
|
+
if (carried) return;
|
|
547
|
+
const heard = String(session.lastHeardText?.() || "").trim();
|
|
548
|
+
const last = heard ? ` The last thing I heard was: "${heard.length > 160 ? `${heard.slice(0, 157)}...` : heard}"` : "";
|
|
549
|
+
this._postToChat(session, `Your line dropped and didn't come back within 30 seconds, so I ended the call.${last} Call me back when you can.`);
|
|
550
|
+
}
|
|
551
|
+
|
|
552
|
+
_postToChat(session, text) {
|
|
553
|
+
const chatId = String(session?.chatId || "");
|
|
554
|
+
if (!chatId || typeof this.adapter?.send !== "function") return Promise.resolve(false);
|
|
555
|
+
// A thread call's notes go to its thread.
|
|
556
|
+
const opts = session?.threadId ? [{ threadId: session.threadId }] : [];
|
|
557
|
+
return Promise.resolve(this.adapter.send(chatId, text, ...opts)).catch((error) => { qlog(`chat post failed: ${error.message}`); return false; });
|
|
558
|
+
}
|
|
559
|
+
|
|
560
|
+
_keepCall(session, why) {
|
|
561
|
+
if (!session?._hangUpPending) return;
|
|
562
|
+
session._hangUpPending = false;
|
|
563
|
+
qlog(`hang up cancelled (${why})`);
|
|
564
|
+
}
|
|
565
|
+
|
|
566
|
+
// ── Jobs: the full agent's work on a call ────────────────────────
|
|
567
|
+
//
|
|
568
|
+
// Each request the full agent works on is a job: the running one first,
|
|
569
|
+
// then any the caller added while it ran (they run in order in the call's
|
|
570
|
+
// lane). A job's speech is held, never dropped, while the caller talks, and
|
|
571
|
+
// goes out once the quick voice has decided what their words meant. Only a
|
|
572
|
+
// change of mind (REPLACE) or a call end stops a job.
|
|
573
|
+
|
|
574
|
+
// Work the caller's next words bear on: running or queued jobs, and a
|
|
575
|
+
// finished one whose answer is still held (it came in while they talked).
|
|
576
|
+
_openJobs(session) {
|
|
577
|
+
return (session?._jobs || []).filter((job) => !job.controller.signal.aborted && (!job.done || job.pending.length));
|
|
578
|
+
}
|
|
579
|
+
|
|
580
|
+
_releaseJobs(session) {
|
|
581
|
+
// Done jobs too: an answer that arrived while the caller talked waits here.
|
|
582
|
+
for (const job of (session?._jobs || []).filter((other) => !other.controller.signal.aborted)) {
|
|
583
|
+
job.muted = false;
|
|
584
|
+
job.flush?.().catch?.(() => {});
|
|
585
|
+
}
|
|
586
|
+
}
|
|
587
|
+
|
|
588
|
+
|
|
589
|
+
// What the quick voice is told about the work (quick-voice.js workNote).
|
|
590
|
+
_board(session, { except = null } = {}) {
|
|
591
|
+
const open = this._openJobs(session).filter((job) => job !== except);
|
|
592
|
+
const current = open.find((job) => job.done) || open.find((job) => job.running) || open[0] || null;
|
|
593
|
+
return {
|
|
594
|
+
current: current ? { question: current.question, seconds: (Date.now() - (current.runningSince || current.startedAt)) / 1000, steps: current.steps, said: current.said, ready: current.done } : null,
|
|
595
|
+
next: open.filter((job) => job !== current && !job.done).map((job) => job.question),
|
|
596
|
+
// An answer still waiting to be spoken is not "earlier": the quick
|
|
597
|
+
// voice must not tell it first.
|
|
598
|
+
earlier: (session?._answered || []).filter((item) => !item.job?.pending.length).slice(-2).map(({ question, answer }) => ({ question, answer })),
|
|
599
|
+
cut: session?.cutSpeech?.()?.text || "",
|
|
600
|
+
};
|
|
601
|
+
}
|
|
602
|
+
|
|
603
|
+
// ── Quick voice ───────────────────────────────────────────────────
|
|
604
|
+
|
|
605
|
+
_startQuickVoice(session, chatId, userId) {
|
|
606
|
+
let settings;
|
|
607
|
+
try { settings = this.quickSettings(); } catch (_) { return; }
|
|
608
|
+
if (!settings?.enabled) return;
|
|
609
|
+
let provider = null;
|
|
610
|
+
try { provider = this.resolveQuickProvider({ adapter: this.adapter, chatId, userId }); } catch (_) {}
|
|
611
|
+
if (!quickVoice.QuickVoiceProcess.supported(provider)) {
|
|
612
|
+
qlog(`off: provider ${provider?.id || "none"} has no warm stream`);
|
|
613
|
+
return;
|
|
614
|
+
}
|
|
615
|
+
const relay = settings.relay === true;
|
|
616
|
+
const quick = quickVoice.track(new quickVoice.QuickVoiceProcess({
|
|
617
|
+
provider, settings, spawnImpl: this.quickSpawn, log: qlog,
|
|
618
|
+
systemPrompt: quickVoice.buildQuickPrompt({ ownerName: quickVoice.ownerDisplayName(), relay }),
|
|
619
|
+
}));
|
|
620
|
+
session._quick = quick;
|
|
621
|
+
session._quickRelay = relay;
|
|
622
|
+
session._quickEarly = settings.early === true;
|
|
623
|
+
// The quick voice says the first line itself; the session's delayed cue
|
|
624
|
+
// would only double it.
|
|
625
|
+
session.acknowledgement = async () => null;
|
|
626
|
+
try { quick.start(); } catch (error) {
|
|
627
|
+
qlog(`start failed: ${error.message}`);
|
|
628
|
+
session._quick = null;
|
|
629
|
+
quick.stop().catch?.(() => {});
|
|
630
|
+
}
|
|
631
|
+
}
|
|
632
|
+
|
|
633
|
+
// While the phone rings, the quick voice reads the end of the chat and
|
|
634
|
+
// writes an opening line that picks up where things were left ("Hey Sumeet,
|
|
635
|
+
// still on the calls update? What's up?"). Nothing to go on, a failure or
|
|
636
|
+
// past greetingMaxMs: the fixed greeting, at once.
|
|
637
|
+
_prepareGreeting(session, chatId, userId, note = {}) {
|
|
638
|
+
const quick = session?._quick;
|
|
639
|
+
if (!quick?.ready()) return null;
|
|
640
|
+
const started = Date.now();
|
|
641
|
+
let expired = false;
|
|
642
|
+
let generation = null;
|
|
643
|
+
let timer = null;
|
|
644
|
+
const deadline = new Promise((resolve) => {
|
|
645
|
+
timer = setTimeout(() => { expired = true; resolve(null); }, session.threadId ? this.quickTiming.threadGreetingMaxMs : this.quickTiming.greetingMaxMs);
|
|
646
|
+
timer.unref?.();
|
|
647
|
+
});
|
|
648
|
+
const write = (async () => {
|
|
649
|
+
const message = quickVoice.greetingMessage(await this._callContext(session, chatId, userId), Date.now(), note);
|
|
650
|
+
if (!message || expired || session._quick !== quick) return "";
|
|
651
|
+
const asked = quick.say(message);
|
|
652
|
+
generation = asked.generation;
|
|
653
|
+
return quickVoice.spokenLine(await asked.text);
|
|
654
|
+
})().catch(() => "");
|
|
655
|
+
const line = Promise.race([write, deadline]).then((text) => {
|
|
656
|
+
clearTimeout(timer);
|
|
657
|
+
const ready = Date.now() - started;
|
|
658
|
+
if (!text) {
|
|
659
|
+
if (text === null) quick.cancel(generation);
|
|
660
|
+
qlog(`greeting ${text === null ? "late" : "skipped"} ready=${ready}ms, fixed greeting used`);
|
|
661
|
+
return "";
|
|
662
|
+
}
|
|
663
|
+
qlog(`greeting ready=${ready}ms chars=${text.length}`);
|
|
664
|
+
this._rememberSpeech(session, "assistant", text);
|
|
665
|
+
return text;
|
|
666
|
+
});
|
|
667
|
+
session.greetingLine = line;
|
|
668
|
+
return line;
|
|
669
|
+
}
|
|
670
|
+
|
|
671
|
+
// What the call knows of its chat, asked once per call (greetingContext):
|
|
672
|
+
// the opening line reads it, and a thread call keeps the thread's note for
|
|
673
|
+
// every full-agent turn (_startJob).
|
|
674
|
+
_callContext(session, chatId, userId) {
|
|
675
|
+
if (!session._context) {
|
|
676
|
+
const target = { adapter: this.adapter, chatId, userId, ...(session.threadId ? { threadId: session.threadId } : {}) };
|
|
677
|
+
session._context = Promise.resolve()
|
|
678
|
+
.then(() => this.resolveGreetingContext(target))
|
|
679
|
+
.then((context) => {
|
|
680
|
+
const thread = context && !Array.isArray(context) ? context.thread : null;
|
|
681
|
+
if (thread) {
|
|
682
|
+
session._threadNote = String(thread.agentNote || "");
|
|
683
|
+
qlog(`thread call: thread=${String(session.threadId).slice(-6)} title=${thread.title ? "yes" : "no"} brief=${thread.brief ? "yes" : "no"} turns=${Array.isArray(context.entries) ? context.entries.length : 0}`);
|
|
684
|
+
}
|
|
685
|
+
return context;
|
|
686
|
+
})
|
|
687
|
+
.catch(() => []);
|
|
688
|
+
}
|
|
689
|
+
return session._context;
|
|
690
|
+
}
|
|
691
|
+
|
|
692
|
+
_stopQuickVoice(session) {
|
|
693
|
+
const quick = session?._quick;
|
|
694
|
+
if (!quick) return;
|
|
695
|
+
session._quick = null;
|
|
696
|
+
quick.stop().catch?.(() => {});
|
|
697
|
+
}
|
|
698
|
+
|
|
699
|
+
// One utterance: a short fitting line as soon as the quick voice has it,
|
|
700
|
+
// then a "still on it" line whenever the full agent leaves the caller in
|
|
701
|
+
// silence, until its answer arrives. Never over the caller, never after
|
|
702
|
+
// the answer, never once a newer utterance took over.
|
|
703
|
+
_quickTurn(session, turn, text, { firstLine = true, early = null } = {}) {
|
|
704
|
+
const quick = session._quick;
|
|
705
|
+
const timing = this.quickTiming;
|
|
706
|
+
const open = () => !turn.done && !turn.controller.signal.aborted &&
|
|
707
|
+
!session.closed && this.session === session && (session._jobs || []).includes(turn) && session._quick === quick;
|
|
708
|
+
const live = () => !turn.muted && open();
|
|
709
|
+
const sayLine = async (message, label, { maxMs = Infinity, unlessSpoke = false, decide = false, early: started = null } = {}) => {
|
|
710
|
+
const asked = Date.now();
|
|
711
|
+
const { generation, text: reply } = started?.asked || quick.say(message);
|
|
712
|
+
const cancel = () => quick.cancel(generation);
|
|
713
|
+
turn.controller.signal.addEventListener("abort", cancel, { once: true });
|
|
714
|
+
let raw = "";
|
|
715
|
+
try { raw = await reply; }
|
|
716
|
+
finally { turn.controller.signal.removeEventListener("abort", cancel); }
|
|
717
|
+
const parsed = decide ? quickVoice.parseQuickReply(raw) : { action: null, line: quickVoice.spokenLine(raw) };
|
|
718
|
+
// Nothing was cut off: REPEAT can only mean it handled the words alone.
|
|
719
|
+
const cutText = session.cutSpeech?.()?.text || "";
|
|
720
|
+
if (parsed.action === "REPEAT" && !cutText) parsed.action = "SELF";
|
|
721
|
+
if (parsed.action === "REPEAT") parsed.line = quickVoice.repeatLead(parsed.line, cutText);
|
|
722
|
+
const promised = decide && parsed.action === "SELF" && quickVoice.promisesWork(parsed.line);
|
|
723
|
+
if (promised) parsed.action = "JOB";
|
|
724
|
+
const { line } = parsed;
|
|
725
|
+
const ready = Date.now() - asked;
|
|
726
|
+
// Not meant for Claudia (the TV, someone else in the room): no work, no
|
|
727
|
+
// line, and what their words cut off is said again (11 Oct 17:51: words
|
|
728
|
+
// stop her, then the quick voice decides whether they were for her).
|
|
729
|
+
// Late, or the full agent already answering: the job stands.
|
|
730
|
+
if (decide && parsed.action === "IGNORE" && !turn.spoke && open() && ready <= maxMs) {
|
|
731
|
+
qlog(`route: IGNORE ready=${ready}ms, job dropped heard=${JSON.stringify(quickVoice.scrubLine(text).slice(0, 80))}`);
|
|
732
|
+
this._dropJob(session, turn, "ignored");
|
|
733
|
+
this._resumeCut(session, "not for me");
|
|
734
|
+
return false;
|
|
735
|
+
}
|
|
736
|
+
// The quick voice handled it alone ("Are you there?"): the full agent's
|
|
737
|
+
// job for it is dropped, unless it has already started answering.
|
|
738
|
+
const self = decide && ["SELF", "REPEAT", "BYE"].includes(parsed.action) && !turn.spoke && open() && ready <= maxMs;
|
|
739
|
+
if (self) {
|
|
740
|
+
qlog(`route: ${parsed.action} ready=${ready}ms, job dropped heard=${JSON.stringify(quickVoice.scrubLine(text).slice(0, 80))} line=${JSON.stringify(line || "")}`);
|
|
741
|
+
this._dropJob(session, turn, "self");
|
|
742
|
+
if (parsed.action !== "BYE") this._keepCall(session, "caller talked on");
|
|
743
|
+
} else if (decide) {
|
|
744
|
+
this._dropCut(session, "new question");
|
|
745
|
+
// The job runs: logged too, so a call review sees every decision.
|
|
746
|
+
const said = promised ? " (said SELF, promised work)" : parsed.action && parsed.action !== "JOB" ? ` (said ${parsed.action})` : parsed.action ? "" : " (said no tag)";
|
|
747
|
+
qlog(`route: JOB${said}${parsed.bye ? " then BYE" : ""} ready=${ready}ms heard=${JSON.stringify(quickVoice.scrubLine(text).slice(0, 80))} line=${JSON.stringify(line || "")}`);
|
|
748
|
+
if (parsed.bye) this._hangUpWhenDone(session);
|
|
749
|
+
else this._keepCall(session, "new work");
|
|
750
|
+
}
|
|
751
|
+
// Handled alone: after [[REPEAT]] (a check-in, "say again") the cut-off
|
|
752
|
+
// words go out after the lead-in. After anything else the caller moved
|
|
753
|
+
// on, as the quick voice was told, so they are dropped (11 Oct 17:45: an
|
|
754
|
+
// answer said again after a new line on the same thing, "why you
|
|
755
|
+
// repeating yourself").
|
|
756
|
+
const resume = () => {
|
|
757
|
+
if (self && parsed.action === "REPEAT") this._resumeCut(session, "repeat", { plain: true });
|
|
758
|
+
else if (self) this._dropCut(session, parsed.action === "BYE" ? "goodbye" : "moved on");
|
|
759
|
+
};
|
|
760
|
+
// Handled alone, the line IS the answer: a sound that stops it, or the
|
|
761
|
+
// caller still talking when it is ready, keeps it for after the sound
|
|
762
|
+
// instead of losing it (11 Oct 17:31: "Do we try it" and "Are you
|
|
763
|
+
// ready" both went unanswered). Earlier cut-off words wait behind it,
|
|
764
|
+
// not spoken over the caller.
|
|
765
|
+
const keep = self && parsed.action !== "BYE";
|
|
766
|
+
const talking = session.callerTalking?.();
|
|
767
|
+
if (!line || !(self || live()) || ready > maxMs || (unlessSpoke && turn.spoke) || talking) {
|
|
768
|
+
if (keep && line && talking && session.keepUnsaid) {
|
|
769
|
+
session.keepUnsaid(line);
|
|
770
|
+
qlog(`${label} kept for later (caller talking) ready=${ready}ms chars=${line.length}`);
|
|
771
|
+
return false;
|
|
772
|
+
}
|
|
773
|
+
qlog(`${label} skipped ready=${ready}ms${talking ? " (caller talking)" : ""}`);
|
|
774
|
+
resume();
|
|
775
|
+
return false;
|
|
776
|
+
}
|
|
777
|
+
qlog(`${label} ready=${ready}ms chars=${line.length}`);
|
|
778
|
+
const spoken = await session.speak(line, keep ? { resumable: true } : undefined);
|
|
779
|
+
if (spoken !== false && (self || live())) this._rememberSpeech(session, "assistant", line);
|
|
780
|
+
if (spoken === false && keep) {
|
|
781
|
+
qlog(`${label} cut, kept for later chars=${line.length}`);
|
|
782
|
+
return false;
|
|
783
|
+
}
|
|
784
|
+
resume();
|
|
785
|
+
// Cut off mid-goodbye: they are still talking, so the call stays up.
|
|
786
|
+
if (self && parsed.action === "BYE" && spoken !== false && !session.callerTalking?.()) await this._hangUp(session, "goodbye");
|
|
787
|
+
return spoken !== false && (self || live());
|
|
788
|
+
};
|
|
789
|
+
(async () => {
|
|
790
|
+
const lineMaxMs = this._relaying(session) ? timing.relayLineMaxMs : timing.lineMaxMs;
|
|
791
|
+
if (firstLine) await sayLine(quickVoice.utteranceMessage(text, this._board(session, { except: turn })), "line", { maxMs: lineMaxMs, unlessSpoke: true, decide: true, early });
|
|
792
|
+
// A job the caller added waits its turn: no "still on it" for it until
|
|
793
|
+
// the full agent is actually on it.
|
|
794
|
+
while (open() && !turn.running && this._openJobs(session)[0] !== turn) {
|
|
795
|
+
await new Promise((resolve) => { const timer = setTimeout(resolve, timing.tickMs); timer.unref?.(); });
|
|
796
|
+
}
|
|
797
|
+
let fillers = 0;
|
|
798
|
+
let lastFillerAt = Date.now();
|
|
799
|
+
// While the job is open, not while it is unmuted: a cough mutes it for
|
|
800
|
+
// a moment, and the fillers must come back after (11 Oct: one noise
|
|
801
|
+
// ended them for good, and the caller sat through 28 s of silence).
|
|
802
|
+
while (fillers < timing.fillerMax && open()) {
|
|
803
|
+
await new Promise((resolve) => { const timer = setTimeout(resolve, timing.tickMs); timer.unref?.(); });
|
|
804
|
+
if (!open()) break;
|
|
805
|
+
// Not while it writes an answer of hers to say: that is next.
|
|
806
|
+
if (!live() || session.speechBusy?.() || session.callerTalking?.() || session._relayBusy) continue;
|
|
807
|
+
const quietSince = Math.max(turn.runningSince || turn.startedAt, lastFillerAt, session._lastSpokeAt || 0);
|
|
808
|
+
if (Date.now() - quietSince < timing.fillerQuietMs) continue;
|
|
809
|
+
fillers += 1;
|
|
810
|
+
lastFillerAt = Date.now();
|
|
811
|
+
await sayLine(quickVoice.fillerMessage(text, (Date.now() - (turn.runningSince || turn.startedAt)) / 1000, this._board(session)), `filler ${fillers}`);
|
|
812
|
+
lastFillerAt = Date.now();
|
|
813
|
+
}
|
|
814
|
+
})().catch((error) => qlog(`turn failed: ${error.message}`));
|
|
815
|
+
}
|
|
816
|
+
|
|
250
817
|
_rememberSpeech(session, role, text) {
|
|
251
818
|
const history = session._voiceHistory ||= [];
|
|
252
819
|
if (role === "assistant" && history.at(-1)?.role === "assistant") {
|
|
@@ -255,19 +822,407 @@ class KazeeCallManager {
|
|
|
255
822
|
while (history.length > 8 || history.reduce((size, entry) => size + entry.text.length, 0) > 12000) history.shift();
|
|
256
823
|
}
|
|
257
824
|
|
|
825
|
+
// Take back the last words remembered for the assistant when they were
|
|
826
|
+
// never said. Consecutive lines share one entry, so only their tail goes.
|
|
827
|
+
_forgetSpeech(session, text) {
|
|
828
|
+
const history = session._voiceHistory || [];
|
|
829
|
+
const last = history.at(-1);
|
|
830
|
+
const words = String(text).slice(-6000);
|
|
831
|
+
if (last?.role !== "assistant") return;
|
|
832
|
+
if (last.text === words) history.pop();
|
|
833
|
+
else if (last.text.endsWith(` ${words}`)) last.text = last.text.slice(0, -(words.length + 1));
|
|
834
|
+
}
|
|
835
|
+
|
|
836
|
+
// A job's written answer, no longer waiting to be said: these words of it
|
|
837
|
+
// (when given), or all of it.
|
|
838
|
+
_settleUnsaid(session, job, texts = null) {
|
|
839
|
+
if (!session?._unsaid?.length) return;
|
|
840
|
+
const left = [...(texts || [])];
|
|
841
|
+
session._unsaid = session._unsaid.filter((entry) => {
|
|
842
|
+
if (entry.job !== job) return true;
|
|
843
|
+
if (!texts) return false;
|
|
844
|
+
const at = left.indexOf(entry.text);
|
|
845
|
+
if (at < 0) return true;
|
|
846
|
+
left.splice(at, 1);
|
|
847
|
+
return false;
|
|
848
|
+
});
|
|
849
|
+
}
|
|
850
|
+
|
|
258
851
|
_transcript(text, incoming, meta = {}, sourceSession = this.session) {
|
|
259
852
|
const session = this.session;
|
|
260
853
|
if (!session || session.closed || sourceSession !== session || !text) return;
|
|
261
854
|
// In a group room, tell the provider who is talking; 1:1 stays unprefixed.
|
|
262
855
|
const speaker = String(meta.display || "").trim();
|
|
263
856
|
const attributed = (meta.participants || 0) > 1 && speaker ? `${speaker}: ${text}` : text;
|
|
264
|
-
|
|
265
|
-
|
|
857
|
+
const early = this._takeEarly(session, attributed, meta);
|
|
858
|
+
// Work is running and the quick voice is up: it decides what these words
|
|
859
|
+
// mean for that work. Otherwise they start a job.
|
|
860
|
+
if (this._openJobs(session).length && session._quick?.ready()) return this._routeBusy(session, attributed, incoming, meta, early);
|
|
861
|
+
// No quick voice to decide: the newest words take over, as before.
|
|
862
|
+
if (!session._quick?.ready()) {
|
|
863
|
+
session._voiceTurn?.controller.abort("superseded");
|
|
864
|
+
for (const job of this._openJobs(session)) job.controller.abort("superseded");
|
|
865
|
+
}
|
|
866
|
+
return this._startJob(session, attributed, incoming, meta, { early });
|
|
867
|
+
}
|
|
868
|
+
|
|
869
|
+
// The words heard when the caller stopped (session.js _commitTurn), about
|
|
870
|
+
// 0.7 s before the recogniser's final (median 688 ms over 273 finals, 11
|
|
871
|
+
// Oct). The quick voice starts on them now, with the message the final
|
|
872
|
+
// would send; _transcript uses that reply only if the final has the same
|
|
873
|
+
// words and the work has not changed. The full agent waits for the final.
|
|
874
|
+
// One at a time, so the quick voice hears the caller in order.
|
|
875
|
+
_earlyWords(text, meta = {}, sourceSession = this.session) {
|
|
876
|
+
const session = this.session;
|
|
877
|
+
if (!session || session.closed || sourceSession !== session || !text || !meta.utteranceId) return;
|
|
878
|
+
if (!session._quickEarly || !session._quick?.ready() || session._early) return;
|
|
879
|
+
const speaker = String(meta.display || "").trim();
|
|
880
|
+
const attributed = (meta.participants || 0) > 1 && speaker ? `${speaker}: ${text}` : text;
|
|
881
|
+
session._early = {
|
|
882
|
+
utteranceId: meta.utteranceId, text: attributed, quick: session._quick, jobs: this._openJobs(session), at: Date.now(),
|
|
883
|
+
asked: session._quick.say(quickVoice.utteranceMessage(attributed, this._board(session))),
|
|
884
|
+
};
|
|
885
|
+
}
|
|
886
|
+
|
|
887
|
+
// The early reply for the final's words, or null. A final with other words
|
|
888
|
+
// (or other work by now) stops it, and the quick voice is told it was
|
|
889
|
+
// never said. Another utterance's final leaves it for its own.
|
|
890
|
+
_takeEarly(session, attributed, meta = {}) {
|
|
891
|
+
const early = session._early;
|
|
892
|
+
if (!early || early.utteranceId !== meta.utteranceId) return null;
|
|
893
|
+
session._early = null;
|
|
894
|
+
const jobs = this._openJobs(session);
|
|
895
|
+
const why = early.quick !== session._quick || !session._quick?.ready() ? "quick voice changed"
|
|
896
|
+
: !quickVoice.sameWords(early.text, attributed) ? "other words"
|
|
897
|
+
: jobs.length !== early.jobs.length || jobs.some((job, i) => job !== early.jobs[i]) ? "work changed" : "";
|
|
898
|
+
if (why) {
|
|
899
|
+
early.quick.retract(early.asked.generation);
|
|
900
|
+
qlog(`early start dropped (${why})`);
|
|
901
|
+
return null;
|
|
902
|
+
}
|
|
903
|
+
qlog(`early start used ${Date.now() - early.at}ms ahead`);
|
|
904
|
+
return early;
|
|
905
|
+
}
|
|
906
|
+
|
|
907
|
+
_dropEarly(session, utteranceId, reason) {
|
|
908
|
+
const early = session?._early;
|
|
909
|
+
if (!early || early.utteranceId !== utteranceId) return;
|
|
910
|
+
session._early = null;
|
|
911
|
+
early.quick.retract(early.asked.generation);
|
|
912
|
+
qlog(`early start dropped (${reason})`);
|
|
913
|
+
}
|
|
914
|
+
|
|
915
|
+
// Words said while a job runs. The quick voice tags them (quick-voice.js):
|
|
916
|
+
// IGNORE (not for Claudia), ANSWER (a check-in or status, answered from the
|
|
917
|
+
// job board), ADD (another request, run after the current one) or REPLACE
|
|
918
|
+
// (a change of mind: the current work is stopped). No usable tag in time:
|
|
919
|
+
// ADD, so nothing is lost and nothing is stopped by a guess.
|
|
920
|
+
async _routeBusy(session, text, incoming, meta, early = null) {
|
|
921
|
+
const quick = session._quick;
|
|
922
|
+
this._rememberSpeech(session, "user", text);
|
|
923
|
+
const asked = Date.now();
|
|
924
|
+
const { generation, text: reply } = early?.asked || quick.say(quickVoice.utteranceMessage(text, this._board(session)));
|
|
925
|
+
let timer = null;
|
|
926
|
+
const late = new Promise((resolve) => { timer = setTimeout(() => resolve(null), this.quickTiming.lineMaxMs); timer.unref?.(); });
|
|
927
|
+
const raw = await Promise.race([reply.catch(() => ""), late]);
|
|
928
|
+
clearTimeout(timer);
|
|
929
|
+
if (raw === null) quick.cancel(generation);
|
|
930
|
+
if (this.session !== session || session.closed) return;
|
|
931
|
+
const parsed = quickVoice.parseQuickReply(raw || "");
|
|
932
|
+
const cutText = session.cutSpeech?.()?.text || "";
|
|
933
|
+
if (parsed.action === "REPEAT" && !cutText) parsed.action = "ANSWER";
|
|
934
|
+
if (parsed.action === "REPEAT") parsed.line = quickVoice.repeatLead(parsed.line, cutText);
|
|
935
|
+
let action = parsed.action === "SELF" ? "ANSWER" : (["IGNORE", "ANSWER", "ADD", "REPLACE", "REPEAT", "BYE"].includes(parsed.action) ? parsed.action : "ADD");
|
|
936
|
+
// A long message is never only a check-in: what it asks would be lost,
|
|
937
|
+
// as ANSWER starts no work (11 Oct: "say apple at some point" inside a
|
|
938
|
+
// 70-word complaint was answered "Got it" and never done).
|
|
939
|
+
const long = action === "ANSWER" && countWords(text) >= LONG_UTTERANCE_WORDS;
|
|
940
|
+
if (long) action = "ADD";
|
|
941
|
+
const ready = Date.now() - asked;
|
|
942
|
+
qlog(`route: ${action}${parsed.action && parsed.action !== action ? ` (said ${parsed.action || "no tag"}${long ? ", long" : ""})` : ""}${parsed.bye && action !== "BYE" ? " then BYE" : ""}${raw === null ? " (late)" : ""} ready=${ready}ms heard=${JSON.stringify(quickVoice.scrubLine(text).slice(0, 80))} line=${JSON.stringify(parsed.line || "")}`);
|
|
943
|
+
// keep: the line is the whole answer (ANSWER), so the caller still
|
|
944
|
+
// talking, or a sound that stops it, keeps it for later, as in _quickTurn.
|
|
945
|
+
const sayLine = async ({ keep = false } = {}) => {
|
|
946
|
+
if (!parsed.line || raw === null) return false;
|
|
947
|
+
if (session.callerTalking?.()) {
|
|
948
|
+
if (!keep || !session.keepUnsaid) return "talking";
|
|
949
|
+
session.keepUnsaid(parsed.line);
|
|
950
|
+
qlog(`answer kept for later (caller talking) chars=${parsed.line.length}`);
|
|
951
|
+
return "kept";
|
|
952
|
+
}
|
|
953
|
+
const spoken = await session.speak(parsed.line, keep ? { resumable: true } : undefined);
|
|
954
|
+
if (spoken === false) {
|
|
955
|
+
if (!keep) return false;
|
|
956
|
+
qlog(`answer cut chars=${parsed.line.length}`);
|
|
957
|
+
return "kept";
|
|
958
|
+
}
|
|
959
|
+
this._rememberSpeech(session, "assistant", parsed.line);
|
|
960
|
+
return true;
|
|
961
|
+
};
|
|
962
|
+
if (action === "REPLACE" || action === "ADD") this._dropCut(session, "new question");
|
|
963
|
+
// Wanting the call ended waits for the work to be done and said.
|
|
964
|
+
if (parsed.bye) this._hangUpWhenDone(session);
|
|
965
|
+
else if (action === "REPLACE" || action === "ADD") this._keepCall(session, "new work");
|
|
966
|
+
if (action === "BYE") {
|
|
967
|
+
await sayLine();
|
|
968
|
+
this._dropCut(session, "goodbye");
|
|
969
|
+
this._releaseJobs(session);
|
|
970
|
+
return;
|
|
971
|
+
}
|
|
972
|
+
if (action === "REPLACE") {
|
|
973
|
+
for (const job of this._openJobs(session)) this._dropJob(session, job, "replaced");
|
|
974
|
+
this._moveTalk(session, { question: text, placeholder: true });
|
|
975
|
+
await sayLine();
|
|
976
|
+
return this._startJob(session, text, incoming, meta, { remembered: true, firstLine: false });
|
|
977
|
+
}
|
|
978
|
+
if (action === "ADD") {
|
|
979
|
+
// The talk is now on the new question: a held answer that goes out
|
|
980
|
+
// next opens with a few words naming its own topic (_leadIn).
|
|
981
|
+
this._moveTalk(session, { question: text, placeholder: true });
|
|
982
|
+
const line = sayLine();
|
|
983
|
+
this._releaseJobs(session);
|
|
984
|
+
await line;
|
|
985
|
+
return this._startJob(session, text, incoming, meta, { remembered: true, firstLine: false });
|
|
986
|
+
}
|
|
987
|
+
const said = action === "ANSWER" || action === "REPEAT" ? await sayLine({ keep: action === "ANSWER" }) : false;
|
|
988
|
+
// Their next words already started: the cut words wait (a sound with no
|
|
989
|
+
// words, or the next turn, decides), never spoken over them (11 Oct
|
|
990
|
+
// 18:24: a REPEAT replayed 545 characters while they talked).
|
|
991
|
+
if (said === "talking") qlog(`resume held (caller talking) chars=${session.cutSpeech?.()?.text.length || 0}`);
|
|
992
|
+
else if (said !== "kept") this._resumeCut(session, action.toLowerCase(), { plain: action === "REPEAT" });
|
|
993
|
+
this._releaseJobs(session);
|
|
994
|
+
}
|
|
995
|
+
|
|
996
|
+
// Stop a job for good: its run is stopped (runner stopCallRunWhenDropped),
|
|
997
|
+
// and what it held back is never spoken.
|
|
998
|
+
_dropJob(session, job, reason) {
|
|
999
|
+
job.pending = [];
|
|
1000
|
+
this._settleUnsaid(session, job);
|
|
1001
|
+
job.controller.abort(reason);
|
|
1002
|
+
session._jobs = (session._jobs || []).filter((other) => other !== job);
|
|
1003
|
+
}
|
|
1004
|
+
|
|
1005
|
+
// ── Lead-ins ─────────────────────────────────────────────────────
|
|
1006
|
+
//
|
|
1007
|
+
// session._talk is what the call is on now: the job last asked or heard.
|
|
1008
|
+
// When a job speaks after the talk moved on (another question was asked or
|
|
1009
|
+
// answered in between), its words open with a few naming its topic: "Back
|
|
1010
|
+
// to the mail backup." (11 Oct test: two answers ran together with no
|
|
1011
|
+
// lead-in, and the caller could not tell which question each was for.)
|
|
1012
|
+
_moveTalk(session, topic) {
|
|
1013
|
+
session._talk = topic;
|
|
1014
|
+
// Write each waiting job's lead-in now, so it is ready when its turn comes.
|
|
1015
|
+
// Not when the quick voice says the answers: it opens them itself (_relay).
|
|
1016
|
+
if (!session._quick?.ready() || this._relaying(session)) return;
|
|
1017
|
+
for (const job of this._openJobs(session)) {
|
|
1018
|
+
if (job !== topic && !job.leadIn) job.leadIn = this._writeLeadIn(session, job);
|
|
1019
|
+
}
|
|
1020
|
+
}
|
|
1021
|
+
|
|
1022
|
+
_writeLeadIn(session, job) {
|
|
1023
|
+
const quick = session._quick;
|
|
1024
|
+
const asked = Date.now();
|
|
1025
|
+
const { generation, text } = quick.say(quickVoice.leadInMessage(job.question));
|
|
1026
|
+
job.controller.signal.addEventListener("abort", () => quick.cancel(generation), { once: true });
|
|
1027
|
+
return text.then((raw) => {
|
|
1028
|
+
const line = quickVoice.spokenLine(String(raw || "").replace(/\[\[[^\]]*\]\]/g, " "));
|
|
1029
|
+
qlog(`lead-in ready=${Date.now() - asked}ms line=${JSON.stringify(line)}`);
|
|
1030
|
+
return line;
|
|
1031
|
+
}).catch(() => "");
|
|
1032
|
+
}
|
|
1033
|
+
|
|
1034
|
+
// The words a job opens with when the talk moved on: the quick voice's
|
|
1035
|
+
// lead-in if it is ready in time, else a plain one. None when the call is
|
|
1036
|
+
// still on this job, without a quick voice, or when the words already
|
|
1037
|
+
// name their topic ("On your voice question, ...").
|
|
1038
|
+
async _leadIn(session, job, response) {
|
|
1039
|
+
if (session._talk === job || !session._quick?.ready() || !job.question) return "";
|
|
1040
|
+
if (LEAD_IN_ALREADY.test(String(response || "").trim())) return "";
|
|
1041
|
+
// Kept on the job until it has spoken, so a talk move meanwhile does not
|
|
1042
|
+
// write it a second one.
|
|
1043
|
+
const pending = job.leadIn ||= this._writeLeadIn(session, job);
|
|
1044
|
+
let timer = null;
|
|
1045
|
+
const late = new Promise((resolve) => { timer = setTimeout(() => resolve(null), this.quickTiming.leadInMaxMs); timer.unref?.(); });
|
|
1046
|
+
const line = await Promise.race([pending, late]);
|
|
1047
|
+
clearTimeout(timer);
|
|
1048
|
+
return line || "On your earlier question.";
|
|
1049
|
+
}
|
|
1050
|
+
|
|
1051
|
+
// ── The quick voice says the answers ─────────────────────────────
|
|
1052
|
+
//
|
|
1053
|
+
// The call has one voice (11 Oct 17:47, the caller: "it should handle
|
|
1054
|
+
// everything"). The full agent's words go to the quick voice as text, and
|
|
1055
|
+
// it says them in its own words. Everything said on the call went through
|
|
1056
|
+
// it, so its own replies show what the caller already heard: it leaves that
|
|
1057
|
+
// out, or skips words that add nothing (11 Oct 18:28: a second answer
|
|
1058
|
+
// explained the setup again, "you just started re-explaining the same
|
|
1059
|
+
// thing"). Its first sentence is said while it writes the rest. Slow, or
|
|
1060
|
+
// gone: the words are said as written.
|
|
1061
|
+
|
|
1062
|
+
_relaying(session) {
|
|
1063
|
+
return !!session?._quickRelay && !!session._quick?.ready();
|
|
1064
|
+
}
|
|
1065
|
+
|
|
1066
|
+
// Resolves once what the quick voice made of the words has been said:
|
|
1067
|
+
// true when it was said (or rightly skipped), false when it was not.
|
|
1068
|
+
// One at a time per call, so each one sees what the one before said.
|
|
1069
|
+
_relay(session, turn, texts) {
|
|
1070
|
+
const words = texts.map((text) => String(text || "").trim()).filter(Boolean).join(" ");
|
|
1071
|
+
if (!words) return Promise.resolve(false);
|
|
1072
|
+
const queued = (session._relayChain || Promise.resolve()).then(() => this._relayNow(session, turn, words));
|
|
1073
|
+
// Worded: what was said is remembered now (_sayRelayed), or nothing new
|
|
1074
|
+
// was (SKIP). Either way the written words are no longer the call's.
|
|
1075
|
+
const settle = () => this._settleUnsaid(session, turn, texts);
|
|
1076
|
+
queued.then(settle, settle);
|
|
1077
|
+
session._relayChain = queued.catch(() => {});
|
|
1078
|
+
return queued.then((result) => result.played, () => false);
|
|
1079
|
+
}
|
|
1080
|
+
|
|
1081
|
+
async _relayNow(session, turn, words) {
|
|
1082
|
+
const quick = session._quick;
|
|
1083
|
+
const gone = () => turn.controller.signal.aborted || this.session !== session || session.closed;
|
|
1084
|
+
if (gone()) return { played: Promise.resolve(false) };
|
|
1085
|
+
if (!this._relaying(session)) return { played: this._speakAsWritten(session, turn, words) };
|
|
1086
|
+
const talk = session._talk;
|
|
1087
|
+
const movedOn = talk && talk !== turn && talk.question && talk.question !== turn.question ? talk.question : "";
|
|
1088
|
+
this._moveTalk(session, turn);
|
|
1089
|
+
const quietSeconds = session._lastSpokeAt ? (Date.now() - session._lastSpokeAt) / 1000 : null;
|
|
1090
|
+
const asked = Date.now();
|
|
1091
|
+
const parts = [];
|
|
1092
|
+
let first = "";
|
|
1093
|
+
let settled = false;
|
|
1094
|
+
session._relayBusy = (session._relayBusy || 0) + 1;
|
|
1095
|
+
try {
|
|
1096
|
+
const { generation, text } = quick.say(quickVoice.answerMessage({ question: turn.question, answer: words, done: turn.done, movedOn, quietSeconds }), {
|
|
1097
|
+
onText: (raw) => {
|
|
1098
|
+
if (first || settled || gone()) return;
|
|
1099
|
+
first = quickVoice.firstSentence(raw);
|
|
1100
|
+
if (!first) return;
|
|
1101
|
+
qlog(`relay first sentence ready=${Date.now() - asked}ms chars=${first.length}`);
|
|
1102
|
+
parts.push(this._sayRelayed(session, turn, first));
|
|
1103
|
+
},
|
|
1104
|
+
});
|
|
1105
|
+
const cancel = () => quick.cancel(generation);
|
|
1106
|
+
turn.controller.signal.addEventListener("abort", cancel, { once: true });
|
|
1107
|
+
let timer = null;
|
|
1108
|
+
const late = new Promise((resolve) => { timer = setTimeout(() => resolve(null), this.quickTiming.relayMaxMs); timer.unref?.(); });
|
|
1109
|
+
let raw = await Promise.race([text, late]);
|
|
1110
|
+
// Its first sentence went out in time: the rest is close behind.
|
|
1111
|
+
if (raw === null && first) raw = await text;
|
|
1112
|
+
clearTimeout(timer);
|
|
1113
|
+
settled = true;
|
|
1114
|
+
turn.controller.signal.removeEventListener("abort", cancel);
|
|
1115
|
+
if (gone()) return { played: Promise.resolve(false) };
|
|
1116
|
+
const ready = Date.now() - asked;
|
|
1117
|
+
if (raw === null) {
|
|
1118
|
+
quick.cancel(generation);
|
|
1119
|
+
qlog(`relay late ready=${ready}ms, said as written chars=${words.length}`);
|
|
1120
|
+
return { played: this._speakAsWritten(session, turn, words) };
|
|
1121
|
+
}
|
|
1122
|
+
const reply = quickVoice.relayText(raw);
|
|
1123
|
+
if (reply.skip && !first) {
|
|
1124
|
+
qlog(`relay: SKIP ready=${ready}ms, nothing new chars=${words.length}`);
|
|
1125
|
+
return { played: Promise.resolve(true) };
|
|
1126
|
+
}
|
|
1127
|
+
if (!reply.text && !first) {
|
|
1128
|
+
qlog(`relay failed ready=${ready}ms, said as written chars=${words.length}`);
|
|
1129
|
+
return { played: this._speakAsWritten(session, turn, words) };
|
|
1130
|
+
}
|
|
1131
|
+
let rest = reply.text;
|
|
1132
|
+
if (first) {
|
|
1133
|
+
const at = rest.indexOf(first);
|
|
1134
|
+
rest = at >= 0 ? rest.slice(at + first.length).trim() : "";
|
|
1135
|
+
}
|
|
1136
|
+
if (rest) parts.push(this._sayRelayed(session, turn, rest));
|
|
1137
|
+
qlog(`relay ready=${ready}ms chars=${words.length}->${first.length + rest.length}`);
|
|
1138
|
+
return { played: Promise.all(parts).then((results) => results.some(Boolean)) };
|
|
1139
|
+
} finally {
|
|
1140
|
+
session._relayBusy -= 1;
|
|
1141
|
+
}
|
|
1142
|
+
}
|
|
1143
|
+
|
|
1144
|
+
// One part of what the quick voice made of an answer. The caller talking
|
|
1145
|
+
// (their words muted the job): kept after the words they cut off, said
|
|
1146
|
+
// after a sound or a check-in, dropped by a new question (as cut words are).
|
|
1147
|
+
async _sayRelayed(session, turn, part) {
|
|
1148
|
+
if (turn.controller.signal.aborted || this.session !== session || session.closed) return false;
|
|
1149
|
+
if (turn.muted || session._hangUpPending || session.callerTalking?.()) {
|
|
1150
|
+
session.keepUnsaid?.(part, { after: true });
|
|
1151
|
+
qlog(`relay kept for later (caller talking) chars=${part.length}`);
|
|
1152
|
+
return false;
|
|
1153
|
+
}
|
|
1154
|
+
// Remembered when due, as in _speakAsWritten.
|
|
1155
|
+
this._rememberSpeech(session, "assistant", part);
|
|
1156
|
+
const spoken = await session.speak(part, { resumable: true });
|
|
1157
|
+
if (spoken !== false && !turn.controller.signal.aborted && this.session === session && !session.closed) {
|
|
1158
|
+
turn.spoke = true;
|
|
1159
|
+
turn.said.push(part);
|
|
1160
|
+
return true;
|
|
1161
|
+
}
|
|
1162
|
+
if (spoken === false) this._forgetSpeech(session, part);
|
|
1163
|
+
return false;
|
|
1164
|
+
}
|
|
1165
|
+
|
|
1166
|
+
// The full agent's words as it wrote them: no quick voice to say them, or
|
|
1167
|
+
// the quick voice was too slow. A lead-in first when the talk moved on.
|
|
1168
|
+
async _speakAsWritten(session, turn, response) {
|
|
1169
|
+
// Remembered when it is due to be said, not once heard: the run ends
|
|
1170
|
+
// without waiting for its answer to play (send below), and a job queued
|
|
1171
|
+
// behind it reads the call so far when its prompt is built.
|
|
1172
|
+
this._rememberSpeech(session, "assistant", response);
|
|
1173
|
+
this._settleUnsaid(session, turn, [response]);
|
|
1174
|
+
const lead = await this._leadIn(session, turn, response);
|
|
1175
|
+
if (turn.controller.signal.aborted || this.session !== session || session.closed) return false;
|
|
1176
|
+
if (session._talk !== turn) this._moveTalk(session, turn);
|
|
1177
|
+
turn.leadIn = null;
|
|
1178
|
+
const spoken = await session.speak(lead ? `${lead} ${response}` : response, { resumable: true });
|
|
1179
|
+
if (spoken !== false && !turn.controller.signal.aborted && this.session === session && !session.closed) {
|
|
1180
|
+
turn.spoke = true;
|
|
1181
|
+
turn.said.push(response);
|
|
1182
|
+
return true;
|
|
1183
|
+
}
|
|
1184
|
+
// Never said (cut off, or the call ended): not part of the call.
|
|
1185
|
+
if (spoken === false) this._forgetSpeech(session, response);
|
|
1186
|
+
return false;
|
|
1187
|
+
}
|
|
1188
|
+
|
|
1189
|
+
_startJob(session, attributed, incoming, meta = {}, { remembered = false, firstLine = true, early: earlyStart = null } = {}) {
|
|
1190
|
+
const speaker = String(meta.display || "").trim();
|
|
1191
|
+
const turn = {
|
|
1192
|
+
controller: new AbortController(), muted: false, done: false, spoke: false, startedAt: Date.now(),
|
|
1193
|
+
question: attributed, running: false, runningSince: 0, steps: [], said: [], pending: [], answer: "",
|
|
1194
|
+
incoming,
|
|
1195
|
+
};
|
|
1196
|
+
session._jobs = [...(session._jobs || []).filter((job) => !job.controller.signal.aborted && (!job.done || job.pending.length)), turn];
|
|
266
1197
|
session._voiceTurn = turn;
|
|
1198
|
+
// A new question is what the call is on. One the caller added while
|
|
1199
|
+
// work ran (_routeBusy) moved the talk when it was asked; it becomes
|
|
1200
|
+
// this job unless something was heard since.
|
|
1201
|
+
if (!remembered || (session._talk?.placeholder && session._talk.question === attributed)) this._moveTalk(session, turn);
|
|
267
1202
|
const callAdapter = Object.create(this.adapter);
|
|
268
1203
|
callAdapter.callHistory = (session._voiceHistory || []).map(entry => ({ ...entry }));
|
|
1204
|
+
if (remembered && callAdapter.callHistory.at(-1)?.role === "user" && callAdapter.callHistory.at(-1).text === attributed) callAdapter.callHistory.pop();
|
|
269
1205
|
callAdapter.callTurn = { signal: turn.controller.signal };
|
|
270
|
-
|
|
1206
|
+
// A thread call: every turn carries the thread's note (thread-context.js).
|
|
1207
|
+
if (session._threadNote) callAdapter.liveCallThreadNote = session._threadNote;
|
|
1208
|
+
if (!remembered) this._rememberSpeech(session, "user", attributed);
|
|
1209
|
+
// Read when the prompt is built, not when queued: a job the caller added
|
|
1210
|
+
// starts after the one before it, and must hear that job's answer.
|
|
1211
|
+
const ownEntry = [...(session._voiceHistory || [])].reverse().find((entry) => entry.role === "user" && entry.text === attributed);
|
|
1212
|
+
// An earlier job's answer is remembered when said, and the quick voice
|
|
1213
|
+
// takes a few seconds to word it (_relay): until then it is read from
|
|
1214
|
+
// what that job wrote. 12 Oct 07:20: a follow-up's prompt was built 1 s
|
|
1215
|
+
// after the first run ended, 3 s before its answer was said, and it
|
|
1216
|
+
// explained the audio routing all over again.
|
|
1217
|
+
callAdapter.liveCallHistory = () => {
|
|
1218
|
+
const history = (session._voiceHistory || []).filter((entry) => entry !== ownEntry).map((entry) => ({ ...entry }));
|
|
1219
|
+
for (const { job, text } of session._unsaid || []) {
|
|
1220
|
+
if (job === turn || job.controller.signal.aborted) continue;
|
|
1221
|
+
if (history.at(-1)?.role === "assistant") history.at(-1).text = `${history.at(-1).text} ${text}`;
|
|
1222
|
+
else history.push({ role: "assistant", text });
|
|
1223
|
+
}
|
|
1224
|
+
return history;
|
|
1225
|
+
};
|
|
271
1226
|
// Provider narration/previews are useful in chat but must never be spoken
|
|
272
1227
|
// into a call. send() receives settled answers; explicit sentence speech
|
|
273
1228
|
// enters separately through the call-turn lease and outbound reply guard.
|
|
@@ -277,29 +1232,89 @@ class KazeeCallManager {
|
|
|
277
1232
|
callAdapter.suppressChatVisibility = true;
|
|
278
1233
|
// Prompt builder switches to phone-call brevity when it sees this.
|
|
279
1234
|
callAdapter.liveCall = true;
|
|
1235
|
+
// The quick voice says the first line; the prompt then skips its own.
|
|
1236
|
+
callAdapter.quickVoice = !!session._quick?.ready();
|
|
1237
|
+
// ... and says this job's answer in its own words (_relay).
|
|
1238
|
+
callAdapter.quickRelay = callAdapter.quickVoice && this._relaying(session);
|
|
1239
|
+
// The caller's camera, read when the prompt is built (system-prompt.js).
|
|
1240
|
+
callAdapter.liveCallCamera = () => session.cameraStill?.() || null;
|
|
280
1241
|
const spokenSentences = [];
|
|
281
|
-
|
|
282
|
-
!session.closed && this.session === session && session.
|
|
1242
|
+
const open = () => !turn.controller.signal.aborted && !turn.done &&
|
|
1243
|
+
!session.closed && this.session === session && (session._jobs || []).includes(turn);
|
|
1244
|
+
// Held while the caller talks, never refused: the call-say CLI must not
|
|
1245
|
+
// see a held job as stale (11 Oct: "loopback 409 ... stale").
|
|
1246
|
+
callAdapter.isCallTurnActive = () => !turn.controller.signal.aborted &&
|
|
1247
|
+
!session.closed && this.session === session && (session._jobs || []).includes(turn);
|
|
283
1248
|
callAdapter.cancelCallSpeech = () => {
|
|
284
|
-
if (this.session === session && session.
|
|
1249
|
+
if (this.session === session && (session._jobs || []).includes(turn)) session.cancelSpeech();
|
|
285
1250
|
};
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
if (
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
1251
|
+
// The job board: the runner reports when the full agent starts on this
|
|
1252
|
+
// job and each tool it uses (core/runner.js onCallStage/onCallActivity).
|
|
1253
|
+
callAdapter.onCallStage = (stage) => {
|
|
1254
|
+
if (stage === "started" && !turn.running) { turn.running = true; turn.runningSince = Date.now(); }
|
|
1255
|
+
};
|
|
1256
|
+
callAdapter.onCallActivity = (event) => {
|
|
1257
|
+
const step = event?.type === "tool_start" ? stepForTool(event.name, event.detail) : null;
|
|
1258
|
+
if (step && turn.steps.at(-1) !== step) turn.steps.push(step);
|
|
1259
|
+
if (turn.steps.length > 8) turn.steps.shift();
|
|
1260
|
+
};
|
|
1261
|
+
const speakNow = (response) => (this._relaying(session)
|
|
1262
|
+
? this._relay(session, turn, [response])
|
|
1263
|
+
: this._speakAsWritten(session, turn, response));
|
|
1264
|
+
// A job speaks only while it is on the board: a finished job leaves it
|
|
1265
|
+
// once its last words are out, and never speaks again. After a goodbye
|
|
1266
|
+
// its words wait and go to the chat (_carryOn): 11 Oct 17:05 an answer
|
|
1267
|
+
// that came in after "Bye" was said for 15 s, and the call stayed up
|
|
1268
|
+
// until the caller spoke again.
|
|
1269
|
+
const speak = async (response) => {
|
|
1270
|
+
if (turn.controller.signal.aborted || session.closed || this.session !== session || !(session._jobs || []).includes(turn)) return false;
|
|
1271
|
+
if (turn.muted || session._hangUpPending) { turn.pending.push(response); return true; }
|
|
1272
|
+
return speakNow(response);
|
|
1273
|
+
};
|
|
1274
|
+
turn.flush = async () => {
|
|
1275
|
+
const free = () => turn.pending.length && !turn.muted && !session._hangUpPending && !turn.controller.signal.aborted;
|
|
1276
|
+
// Held words go to the quick voice together: one reply, said once.
|
|
1277
|
+
if (this._relaying(session) && free()) return this._relay(session, turn, turn.pending.splice(0));
|
|
1278
|
+
while (free()) await speakNow(turn.pending.shift());
|
|
294
1279
|
};
|
|
295
1280
|
callAdapter.speakSentence = async response => {
|
|
296
1281
|
const spoken = await speak(response);
|
|
297
1282
|
if (spoken) spokenSentences.push(response);
|
|
298
1283
|
return spoken;
|
|
299
1284
|
};
|
|
1285
|
+
// The answer handed over at the run's result event (core/runner.js
|
|
1286
|
+
// earlyCallAnswer), before its process exits: the same words delivered
|
|
1287
|
+
// after it are not said again, only anything past them.
|
|
1288
|
+
let early = "";
|
|
1289
|
+
const sameWords = (text) => String(text || "").trim().replace(/\s+/g, " ");
|
|
1290
|
+
callAdapter.earlyCallAnswer = async (response) => {
|
|
1291
|
+
if (early || !sameWords(response)) return false;
|
|
1292
|
+
early = sameWords(response);
|
|
1293
|
+
return deliver(response);
|
|
1294
|
+
};
|
|
300
1295
|
callAdapter.send = async (_channelId, response) => {
|
|
1296
|
+
const words = sameWords(response);
|
|
1297
|
+
if (early && (!words || early.includes(words))) return true;
|
|
1298
|
+
return deliver(early && words.startsWith(early) ? words.slice(early.length).trim() : response);
|
|
1299
|
+
};
|
|
1300
|
+
const deliver = async (response) => {
|
|
301
1301
|
const remainder = require("../../../core/call-speech").unspokenSuffix(response, spokenSentences);
|
|
302
|
-
|
|
1302
|
+
turn.answer = String(response || "");
|
|
1303
|
+
if (turn.answer && !turn.controller.signal.aborted) {
|
|
1304
|
+
session._answered = [...(session._answered || []), { question: turn.question, answer: turn.answer.slice(0, 400), job: turn }].slice(-4);
|
|
1305
|
+
}
|
|
1306
|
+
// Done, but kept on the board until the next job starts: a long reply
|
|
1307
|
+
// can arrive in more than one send, and held words still go out.
|
|
1308
|
+
turn.done = true;
|
|
1309
|
+
// Not awaited: the run is over once its answer is written, so a job
|
|
1310
|
+
// the caller added starts while this one is heard (session.speak plays
|
|
1311
|
+
// them in order). 11 Oct: a queued question waited 27 s for the answer
|
|
1312
|
+
// before it to finish playing before its run even began.
|
|
1313
|
+
if (remainder) {
|
|
1314
|
+
if (!turn.controller.signal.aborted) session._unsaid = [...(session._unsaid || []), { job: turn, text: remainder }].slice(-4);
|
|
1315
|
+
// Not said (off the board, or the call moved on): not the call's.
|
|
1316
|
+
speak(remainder).then((spoken) => { if (spoken === false) this._settleUnsaid(session, turn, [remainder]); }, () => this._settleUnsaid(session, turn, [remainder]));
|
|
1317
|
+
}
|
|
303
1318
|
return true;
|
|
304
1319
|
};
|
|
305
1320
|
callAdapter.edit = async () => true;
|
|
@@ -308,9 +1323,13 @@ class KazeeCallManager {
|
|
|
308
1323
|
console.warn(`Kazee call ignored file voice response ${voicePath}; streamed TTS is active`);
|
|
309
1324
|
return true;
|
|
310
1325
|
};
|
|
311
|
-
this.
|
|
1326
|
+
if (callAdapter.quickVoice) this._quickTurn(session, turn, attributed, { firstLine, early: earlyStart });
|
|
1327
|
+
else if (earlyStart) earlyStart.quick.retract(earlyStart.asked.generation);
|
|
1328
|
+
this.dispatchJob({
|
|
312
1329
|
adapter: callAdapter,
|
|
313
1330
|
channelId: String(incoming.chatId),
|
|
1331
|
+
// A thread call's turns run in the thread's conversation.
|
|
1332
|
+
...(session.threadId ? { threadId: session.threadId } : {}),
|
|
314
1333
|
canonicalUserId: canonicalForChannel("kazee", String(incoming.fromUserId)),
|
|
315
1334
|
userId: String(incoming.fromUserId),
|
|
316
1335
|
type: "text",
|
|
@@ -318,7 +1337,8 @@ class KazeeCallManager {
|
|
|
318
1337
|
messageId: `call-${incoming.roomId}-${Date.now()}`,
|
|
319
1338
|
from: { id: String(incoming.fromUserId), name: incoming.fromUserName || "" },
|
|
320
1339
|
raw: { ...incoming, source: "kazee-call", ephemeralAudio: true, speaker: speaker || undefined, feedId: meta.feedId },
|
|
321
|
-
});
|
|
1340
|
+
}, { turn, session });
|
|
1341
|
+
return turn;
|
|
322
1342
|
}
|
|
323
1343
|
|
|
324
1344
|
async ended(payload = {}) {
|
|
@@ -347,6 +1367,15 @@ class KazeeCallManager {
|
|
|
347
1367
|
this._detachSession(session);
|
|
348
1368
|
await session.close("adapter-stopped");
|
|
349
1369
|
}
|
|
1370
|
+
|
|
1371
|
+
// The switchboard is stopping for good (not a brain restart): end the call
|
|
1372
|
+
// so the caller's app hangs up now, not after chat-central's 20 s grace.
|
|
1373
|
+
async shutdown(reason = "switchboard-stopped") {
|
|
1374
|
+
this._pending?.controller.abort();
|
|
1375
|
+
this._clearRingTimer();
|
|
1376
|
+
if (!this.session) return;
|
|
1377
|
+
await this.session.close(reason);
|
|
1378
|
+
}
|
|
350
1379
|
}
|
|
351
1380
|
|
|
352
|
-
module.exports = { KazeeCallManager, DEFAULT_ICE_SERVERS, DEFAULT_JANUS_URL };
|
|
1381
|
+
module.exports = { KazeeCallManager, DEFAULT_ICE_SERVERS, DEFAULT_JANUS_URL, recentChat, greetingContext, carryOnInChat, threadIdOf };
|