conductor-remote 1.130.1 → 1.131.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/assets/{AutoModelSettings-DQAuzuHG.js → AutoModelSettings-bXS1_9Es.js} +1 -1
- package/dist/assets/{PierrePatch-CSMuVBjh.js → PierrePatch-BZvD6Gh8.js} +1 -1
- package/dist/assets/index-Dbu5DOed.css +1 -0
- package/dist/assets/{index-B4mWU6o0.js → index-x6nsv58D.js} +41 -41
- package/dist/index.html +2 -2
- package/dist/sw.js +1 -1
- package/dist-node/src/http/routes/voice.js +57 -0
- package/dist-node/src/http/services/voice.js +21 -2
- package/dist-node/src/routes.js +3 -0
- package/dist-node/src/shared.js +1 -0
- package/dist-node/src/voice/broker.js +162 -7
- package/dist-node/src/voice/history.js +7 -0
- package/dist-node/src/voice/preview.js +91 -5
- package/dist-node/src/voice/prompt.js +13 -8
- package/dist-node/src/voice/response.js +23 -0
- package/dist-node/src/voice/tools.js +90 -15
- package/dist-node/src/voice/webrtc.js +2 -1
- package/docs/voice-setup.md +10 -0
- package/package.json +1 -1
- package/dist/assets/index-DD0rU7nR.css +0 -1
package/dist/index.html
CHANGED
|
@@ -25,8 +25,8 @@
|
|
|
25
25
|
<title>Conductor Remote</title>
|
|
26
26
|
<!-- Runs before the module bundle so it can catch a stale shell that fails to boot. -->
|
|
27
27
|
<script src="/self-heal.js"></script>
|
|
28
|
-
<script type="module" crossorigin src="/assets/index-
|
|
29
|
-
<link rel="stylesheet" crossorigin href="/assets/index-
|
|
28
|
+
<script type="module" crossorigin src="/assets/index-x6nsv58D.js"></script>
|
|
29
|
+
<link rel="stylesheet" crossorigin href="/assets/index-Dbu5DOed.css">
|
|
30
30
|
<link rel="manifest" href="/manifest.webmanifest"></head>
|
|
31
31
|
<body>
|
|
32
32
|
<div id="root"></div>
|
package/dist/sw.js
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
if(!self.define){let e,s={};const i=(i,n)=>(i=new URL(i+".js",n).href,s[i]||new Promise(s=>{if("document"in self){const e=document.createElement("script");e.src=i,e.onload=s,document.head.appendChild(e)}else e=i,importScripts(i),s()}).then(()=>{let e=s[i];if(!e)throw new Error(`Module ${i} didn’t register its module`);return e}));self.define=(n,r)=>{const l=e||("document"in self?document.currentScript.src:"")||location.href;if(s[l])return;let o={};const t=e=>i(e,l),a={module:{uri:l},exports:o,require:t};s[l]=Promise.all(n.map(e=>a[e]||t(e))).then(e=>(r(...e),o))}}define(["./workbox-dcde9eb3"],function(e){"use strict";importScripts("/push-sw.js"),self.addEventListener("message",e=>{e.data&&"SKIP_WAITING"===e.data.type&&self.skipWaiting()}),e.clientsClaim(),e.precacheAndRoute([{url:"self-heal.js",revision:"49bd63adb25a09341f8d2610e8bd3c76"},{url:"push-sw.js",revision:"4e0b714c0ea53e85c167e6d8f5b767aa"},{url:"index.html",revision:"
|
|
1
|
+
if(!self.define){let e,s={};const i=(i,n)=>(i=new URL(i+".js",n).href,s[i]||new Promise(s=>{if("document"in self){const e=document.createElement("script");e.src=i,e.onload=s,document.head.appendChild(e)}else e=i,importScripts(i),s()}).then(()=>{let e=s[i];if(!e)throw new Error(`Module ${i} didn’t register its module`);return e}));self.define=(n,r)=>{const l=e||("document"in self?document.currentScript.src:"")||location.href;if(s[l])return;let o={};const t=e=>i(e,l),a={module:{uri:l},exports:o,require:t};s[l]=Promise.all(n.map(e=>a[e]||t(e))).then(e=>(r(...e),o))}}define(["./workbox-dcde9eb3"],function(e){"use strict";importScripts("/push-sw.js"),self.addEventListener("message",e=>{e.data&&"SKIP_WAITING"===e.data.type&&self.skipWaiting()}),e.clientsClaim(),e.precacheAndRoute([{url:"self-heal.js",revision:"49bd63adb25a09341f8d2610e8bd3c76"},{url:"push-sw.js",revision:"4e0b714c0ea53e85c167e6d8f5b767aa"},{url:"index.html",revision:"cf434f420a15be6e303151087025c1cf"},{url:"assets/workbox-window.prod.es5-BBnX5xw4.js",revision:null},{url:"assets/pierre-shiki-wasm-C8ETeZem.js",revision:null},{url:"assets/pierre-light-480U9XYS.js",revision:null},{url:"assets/pierre-dark-CyvmCCZW.js",revision:null},{url:"assets/index-x6nsv58D.js",revision:null},{url:"assets/index-Dbu5DOed.css",revision:null},{url:"assets/PierrePatch-BZvD6Gh8.js",revision:null},{url:"assets/AutoModelSettings-bXS1_9Es.js",revision:null},{url:"apple-touch-icon.png",revision:"1127bb396b4648add53dce3f22c92aee"},{url:"icon-192.png",revision:"c5e01ac58768627e18ee7b8b6a9239ef"},{url:"icon-512.png",revision:"a40638c55e310312457a621c9a0002c8"},{url:"icon-maskable-512.png",revision:"a9b0d962686287452216492cd2247499"},{url:"icon.svg",revision:"c1aee186821798733dd477e69a0ef243"},{url:"manifest.webmanifest",revision:"cf88fbc5755108a7fe0616fa160a8a15"}],{}),e.cleanupOutdatedCaches(),e.registerRoute(new e.NavigationRoute(e.createHandlerBoundToURL("/index.html"),{denylist:[/^\/api\//]})),e.registerRoute(({url:e})=>e.pathname.startsWith("/assets/diff-syntax/"),new e.CacheFirst({cacheName:"diff-syntax",plugins:[new e.ExpirationPlugin({maxEntries:32,maxAgeSeconds:31536e3})]}),"GET")});
|
|
@@ -11,6 +11,63 @@ export function createVoiceRoutes(services) {
|
|
|
11
11
|
const { json, voiceHistory, voiceConfig, voiceBroker, readBody, reads, voiceSafetyIdentifier } = services;
|
|
12
12
|
return async (req, res, url) => {
|
|
13
13
|
const { pathname } = url;
|
|
14
|
+
const draftsCall = routeParam(routes.voiceDrafts, req.method, pathname);
|
|
15
|
+
if (draftsCall)
|
|
16
|
+
return json(req, res, 200, { drafts: services.voicePreviews.list(draftsCall) });
|
|
17
|
+
const textCall = routeParam(routes.voiceCallText, req.method, pathname);
|
|
18
|
+
const draftCall = routeParam(routes.voiceDraftAction, req.method, pathname);
|
|
19
|
+
if (textCall || draftCall) {
|
|
20
|
+
const id = (textCall || draftCall);
|
|
21
|
+
if (!voiceBroker?.isBrowserCall(id))
|
|
22
|
+
return json(req, res, 409, {
|
|
23
|
+
error: 'This call is no longer connected. Saved drafts and receipts are still available.'
|
|
24
|
+
});
|
|
25
|
+
const raw = await readBody(req);
|
|
26
|
+
if (raw.length > 40_000)
|
|
27
|
+
return json(req, res, 413, { error: 'Draft is too large' });
|
|
28
|
+
const body = JSON.parse(raw || '{}');
|
|
29
|
+
if (textCall) {
|
|
30
|
+
if (typeof body.text !== 'string' || !body.text.trim())
|
|
31
|
+
return json(req, res, 400, { error: 'Text is required' });
|
|
32
|
+
return voiceBroker.sendText(id, body.text.trim())
|
|
33
|
+
? json(req, res, 200, { ok: true })
|
|
34
|
+
: json(req, res, 409, { error: 'Voice connection is unavailable' });
|
|
35
|
+
}
|
|
36
|
+
if (typeof body.token !== 'string')
|
|
37
|
+
return json(req, res, 400, { error: 'Draft token is required' });
|
|
38
|
+
const preview = services.voicePreviews.get(id, body.token);
|
|
39
|
+
if (!preview)
|
|
40
|
+
return json(req, res, 404, { error: 'Draft not found in this call' });
|
|
41
|
+
if (body.action === 'pause' || body.action === 'resume') {
|
|
42
|
+
return services.voicePreviews.pauseReview(id, body.token, body.action === 'pause')
|
|
43
|
+
? json(req, res, 200, { ok: true })
|
|
44
|
+
: json(req, res, 409, { error: 'This revision has already been used or replaced' });
|
|
45
|
+
}
|
|
46
|
+
if (body.action === 'present')
|
|
47
|
+
return json(req, res, 200, { ok: services.voicePreviews.present(id, body.token) });
|
|
48
|
+
if (body.action === 'edit') {
|
|
49
|
+
if (typeof body.text !== 'string' || (preview.kind === 'send_prompt' && !body.text.trim()))
|
|
50
|
+
return json(req, res, 400, { error: 'Draft text is required' });
|
|
51
|
+
const edited = services.voicePreviews.edit(id, body.token, body.text.trim());
|
|
52
|
+
if (!edited)
|
|
53
|
+
return json(req, res, 409, { error: 'This revision has already been used or replaced' });
|
|
54
|
+
// Reference data only; this update is never a new approval.
|
|
55
|
+
voiceBroker.inject(id, `The user edited or renewed the draft on screen. Previous approval is invalid. Current draft reference data: ${JSON.stringify(edited)}. Ask for fresh approval; do not read it aloud unless asked.`);
|
|
56
|
+
return json(req, res, 200, { draft: edited });
|
|
57
|
+
}
|
|
58
|
+
if (body.action === 'approve') {
|
|
59
|
+
const name = preview.kind === 'send_prompt' ? 'voice_send' : 'voice_create_workspace';
|
|
60
|
+
const args = preview.kind === 'send_prompt'
|
|
61
|
+
? { token: preview.token, session_id: preview.sessionId, text: preview.text }
|
|
62
|
+
: { token: preview.token, repo: preview.repo, prompt: preview.prompt };
|
|
63
|
+
const tool = services.voiceToolsForCall(id).find(tool => tool.name === name);
|
|
64
|
+
const result = JSON.parse(await tool.run(args));
|
|
65
|
+
if (result.status !== 'refused')
|
|
66
|
+
voiceBroker.inject(id, `The user approved draft ${preview.token} on screen; the action is queued. Do not request approval again or repeat the action. Its final receipt will be saved separately.`);
|
|
67
|
+
return json(req, res, result.status === 'refused' ? 409 : 200, result.status === 'refused' ? { error: result.spoken } : result);
|
|
68
|
+
}
|
|
69
|
+
return json(req, res, 400, { error: 'Unknown draft action' });
|
|
70
|
+
}
|
|
14
71
|
const diagnosticCall = routeParam(routes.voiceCallDiagnostics, req.method, pathname);
|
|
15
72
|
if (diagnosticCall) {
|
|
16
73
|
if (!voiceHistory.status(diagnosticCall))
|
|
@@ -25,6 +25,7 @@ export function createVoiceServices(services) {
|
|
|
25
25
|
// bearer means neither that bearer nor a device identifier leaves the Mac.
|
|
26
26
|
const voiceSafetyIdentifier = crypto.createHash('sha256').update(`conductor-remote:${cfg.token}`).digest('hex');
|
|
27
27
|
const voiceBoards = new Map();
|
|
28
|
+
const voiceSelections = new Map();
|
|
28
29
|
const voicePreviews = new PreviewStore(path.join(stateDir(), 'voice-previews.json'));
|
|
29
30
|
const voiceHistory = new VoiceHistory(path.join(stateDir(), 'voice-history.db'));
|
|
30
31
|
process.on('exit', () => voiceHistory.close());
|
|
@@ -41,7 +42,10 @@ export function createVoiceServices(services) {
|
|
|
41
42
|
stateFile: path.join(stateDir(), 'voice-calls.json'),
|
|
42
43
|
history: voiceHistory,
|
|
43
44
|
tools: callId => voiceToolsForCall(callId),
|
|
44
|
-
onClose: callId =>
|
|
45
|
+
onClose: callId => {
|
|
46
|
+
voiceBoards.delete(callId);
|
|
47
|
+
voiceSelections.delete(callId);
|
|
48
|
+
}
|
|
45
49
|
})
|
|
46
50
|
: null;
|
|
47
51
|
function voiceBoard(callId) {
|
|
@@ -120,11 +124,18 @@ export function createVoiceServices(services) {
|
|
|
120
124
|
};
|
|
121
125
|
}
|
|
122
126
|
function voiceToolsForCall(callId) {
|
|
127
|
+
let selection = voiceSelections.get(callId);
|
|
128
|
+
if (!selection) {
|
|
129
|
+
selection = { repo: null, confirmed: false };
|
|
130
|
+
voiceSelections.set(callId, selection);
|
|
131
|
+
}
|
|
123
132
|
return createVoiceTools({
|
|
124
133
|
callId,
|
|
134
|
+
selection,
|
|
125
135
|
board: voiceBoard(callId),
|
|
126
136
|
recall: new VoiceRecall({ history: voiceHistory, callId }),
|
|
127
137
|
previews: voicePreviews,
|
|
138
|
+
presentPreview: token => voiceBroker?.isBrowserCall(callId) ? voicePreviews.waitForPresentation(token) : Promise.resolve(false),
|
|
128
139
|
findSession: sessionId => reads.listSessionStates().find(state => state.sessionId === sessionId) ?? null,
|
|
129
140
|
listRepos: () => reads.listRepos().map(repo => ({ name: repo.name, defaultBranch: repo.default_branch })),
|
|
130
141
|
createWorkspace: createVoiceWorkspace,
|
|
@@ -146,5 +157,13 @@ export function createVoiceServices(services) {
|
|
|
146
157
|
mcpToken: () => voiceConfig.mcpToken,
|
|
147
158
|
log: line => console.warn(line)
|
|
148
159
|
});
|
|
149
|
-
return {
|
|
160
|
+
return {
|
|
161
|
+
voiceHistory,
|
|
162
|
+
voiceConfig,
|
|
163
|
+
voiceBroker,
|
|
164
|
+
voiceSafetyIdentifier,
|
|
165
|
+
voiceServer,
|
|
166
|
+
voicePreviews,
|
|
167
|
+
voiceToolsForCall
|
|
168
|
+
};
|
|
150
169
|
}
|
package/dist-node/src/routes.js
CHANGED
|
@@ -60,6 +60,9 @@ export const routes = {
|
|
|
60
60
|
voiceCall: flat('POST', '/api/voice/calls'),
|
|
61
61
|
/** Start the greeting only after the browser has installed OpenAI's SDP answer. */
|
|
62
62
|
voiceCallReady: param('POST', '/api/voice/calls/:callId/ready'),
|
|
63
|
+
voiceCallText: param('POST', '/api/voice/calls/:callId/text'),
|
|
64
|
+
voiceDrafts: param('GET', '/api/voice/calls/:callId/drafts'),
|
|
65
|
+
voiceDraftAction: param('POST', '/api/voice/calls/:callId/drafts'),
|
|
63
66
|
voiceCallDiagnostics: param('POST', '/api/voice/calls/:callId/diagnostics'),
|
|
64
67
|
/** End one PWA-owned call. SIP calls remain owned by their phone leg. */
|
|
65
68
|
voiceCallEnd: param('DELETE', '/api/voice/calls/:callId'),
|
package/dist-node/src/shared.js
CHANGED
|
@@ -125,7 +125,8 @@ export class VoiceBroker {
|
|
|
125
125
|
// Records written before WebRTC support were all SIP/MCP calls.
|
|
126
126
|
mode: call.mode === 'function' ? 'function' : 'mcp',
|
|
127
127
|
ready: call.ready ?? true,
|
|
128
|
-
greeted: call.greeted ?? false
|
|
128
|
+
greeted: call.greeted ?? false,
|
|
129
|
+
managedTurns: call.managedTurns === true
|
|
129
130
|
});
|
|
130
131
|
}
|
|
131
132
|
}
|
|
@@ -214,6 +215,7 @@ export class VoiceBroker {
|
|
|
214
215
|
callId,
|
|
215
216
|
socket,
|
|
216
217
|
mode: call.mode,
|
|
218
|
+
managedTurns: call.managedTurns === true,
|
|
217
219
|
ready: call.ready,
|
|
218
220
|
open: socket.readyState === 1,
|
|
219
221
|
toolsReady: call.mode === 'function',
|
|
@@ -222,6 +224,17 @@ export class VoiceBroker {
|
|
|
222
224
|
responseActive: false,
|
|
223
225
|
responseDone: false,
|
|
224
226
|
responseHadTool: false,
|
|
227
|
+
responseId: null,
|
|
228
|
+
inputEpoch: 0,
|
|
229
|
+
userSpeaking: false,
|
|
230
|
+
waitingForUserResponse: false,
|
|
231
|
+
userTurnPending: false,
|
|
232
|
+
finishedResponses: new Set(),
|
|
233
|
+
responseStartedAt: 0,
|
|
234
|
+
firstOutputAt: null,
|
|
235
|
+
playing: false,
|
|
236
|
+
typed: [],
|
|
237
|
+
toolEpoch: new Map(),
|
|
225
238
|
pending: [],
|
|
226
239
|
pendingTools: new Set(),
|
|
227
240
|
handledTools: new Set(),
|
|
@@ -260,7 +273,10 @@ export class VoiceBroker {
|
|
|
260
273
|
this.deps.onClose?.(callId);
|
|
261
274
|
}
|
|
262
275
|
createResponse(runtime) {
|
|
263
|
-
if (runtime.responseActive ||
|
|
276
|
+
if (runtime.responseActive ||
|
|
277
|
+
runtime.userSpeaking ||
|
|
278
|
+
runtime.waitingForUserResponse ||
|
|
279
|
+
runtime.socket.readyState !== 1)
|
|
264
280
|
return false;
|
|
265
281
|
runtime.responseActive = true;
|
|
266
282
|
runtime.responseDone = false;
|
|
@@ -274,7 +290,13 @@ export class VoiceBroker {
|
|
|
274
290
|
runtime.started = this.createResponse(runtime);
|
|
275
291
|
}
|
|
276
292
|
flush(runtime) {
|
|
277
|
-
if (!runtime.toolsReady ||
|
|
293
|
+
if (!runtime.toolsReady ||
|
|
294
|
+
!runtime.greeted ||
|
|
295
|
+
runtime.responseActive ||
|
|
296
|
+
runtime.userSpeaking ||
|
|
297
|
+
runtime.waitingForUserResponse ||
|
|
298
|
+
runtime.playing ||
|
|
299
|
+
runtime.socket.readyState !== 1)
|
|
278
300
|
return;
|
|
279
301
|
const text = runtime.pending.shift();
|
|
280
302
|
if (!text)
|
|
@@ -289,7 +311,9 @@ export class VoiceBroker {
|
|
|
289
311
|
}
|
|
290
312
|
/** OpenAI may finish the response before or after its tool calls. Both are a barrier. */
|
|
291
313
|
continueAfterTools(runtime) {
|
|
292
|
-
if (!runtime.responseDone || !runtime.responseHadTool || runtime.
|
|
314
|
+
if (!runtime.responseDone || !runtime.responseHadTool || runtime.userSpeaking || runtime.waitingForUserResponse)
|
|
315
|
+
return;
|
|
316
|
+
if ([...runtime.pendingTools].some(id => runtime.toolEpoch.get(id) === runtime.inputEpoch))
|
|
293
317
|
return;
|
|
294
318
|
runtime.responseDone = false;
|
|
295
319
|
runtime.responseHadTool = false;
|
|
@@ -299,13 +323,17 @@ export class VoiceBroker {
|
|
|
299
323
|
const item = (event.item ?? {});
|
|
300
324
|
const id = typeof event.item_id === 'string' ? event.item_id : typeof item.id === 'string' ? item.id : null;
|
|
301
325
|
const started = id ? runtime.toolStarted.get(id) : undefined;
|
|
326
|
+
const current = !id || runtime.toolEpoch.get(id) === runtime.inputEpoch;
|
|
302
327
|
if (id) {
|
|
303
328
|
runtime.toolStarted.delete(id);
|
|
304
329
|
runtime.pendingTools.delete(id);
|
|
330
|
+
runtime.toolEpoch.delete(id);
|
|
305
331
|
}
|
|
306
332
|
const name = typeof item.name === 'string' ? item.name : (id ?? 'unknown tool');
|
|
307
333
|
const latency = started === undefined ? 'unknown latency' : `${this.now() - started}ms`;
|
|
308
334
|
this.log(`[voice] ${callId} tool ${name}: ${latency}`);
|
|
335
|
+
if (!current)
|
|
336
|
+
return;
|
|
309
337
|
runtime.responseHadTool = true;
|
|
310
338
|
// Remote MCP results do not automatically continue, but OpenAI requires the
|
|
311
339
|
// original response *and every MCP call in it* to finish before the next one.
|
|
@@ -321,8 +349,28 @@ export class VoiceBroker {
|
|
|
321
349
|
if (!invocationId || !name || runtime.handledTools.has(invocationId))
|
|
322
350
|
return;
|
|
323
351
|
runtime.handledTools.add(invocationId);
|
|
352
|
+
if (runtime.userSpeaking ||
|
|
353
|
+
runtime.waitingForUserResponse ||
|
|
354
|
+
(typeof event.response_id === 'string' &&
|
|
355
|
+
(runtime.finishedResponses.has(event.response_id) ||
|
|
356
|
+
(runtime.responseId && event.response_id !== runtime.responseId)))) {
|
|
357
|
+
this.send(runtime, {
|
|
358
|
+
type: 'conversation.item.create',
|
|
359
|
+
item: {
|
|
360
|
+
type: 'function_call_output',
|
|
361
|
+
call_id: invocationId,
|
|
362
|
+
output: JSON.stringify({
|
|
363
|
+
status: 'interrupted',
|
|
364
|
+
spoken: 'The call was interrupted before this tool started. No action was executed.'
|
|
365
|
+
})
|
|
366
|
+
}
|
|
367
|
+
});
|
|
368
|
+
return;
|
|
369
|
+
}
|
|
324
370
|
runtime.pendingTools.add(invocationId);
|
|
325
371
|
runtime.toolStarted.set(invocationId, this.now());
|
|
372
|
+
const inputEpoch = runtime.inputEpoch;
|
|
373
|
+
runtime.toolEpoch.set(invocationId, inputEpoch);
|
|
326
374
|
runtime.responseHadTool = true;
|
|
327
375
|
void (async () => {
|
|
328
376
|
let output;
|
|
@@ -348,9 +396,11 @@ export class VoiceBroker {
|
|
|
348
396
|
const started = runtime.toolStarted.get(invocationId);
|
|
349
397
|
runtime.toolStarted.delete(invocationId);
|
|
350
398
|
runtime.pendingTools.delete(invocationId);
|
|
399
|
+
runtime.toolEpoch.delete(invocationId);
|
|
351
400
|
const latency = started === undefined ? 'unknown latency' : `${this.now() - started}ms`;
|
|
352
401
|
this.log(`[voice] ${callId} tool ${name}: ${latency}`);
|
|
353
|
-
|
|
402
|
+
if (inputEpoch === runtime.inputEpoch)
|
|
403
|
+
this.continueAfterTools(runtime);
|
|
354
404
|
})();
|
|
355
405
|
}
|
|
356
406
|
logUsage(callId, usage) {
|
|
@@ -376,6 +426,38 @@ export class VoiceBroker {
|
|
|
376
426
|
if (diagnostic)
|
|
377
427
|
this.log(`[voice-diagnostics] ${JSON.stringify({ callId, source: 'server', receivedAt: this.now(), ...diagnostic })}`);
|
|
378
428
|
const type = event.type;
|
|
429
|
+
if (typeof type === 'string' &&
|
|
430
|
+
/^response\.(output_audio_transcript|audio_transcript|output_text)\.delta$/.test(type) &&
|
|
431
|
+
runtime.firstOutputAt === null)
|
|
432
|
+
runtime.firstOutputAt = this.now();
|
|
433
|
+
if (type === 'input_audio_buffer.speech_started') {
|
|
434
|
+
runtime.inputEpoch++;
|
|
435
|
+
runtime.userSpeaking = true;
|
|
436
|
+
runtime.waitingForUserResponse = true;
|
|
437
|
+
runtime.responseDone = false;
|
|
438
|
+
runtime.responseHadTool = false;
|
|
439
|
+
return;
|
|
440
|
+
}
|
|
441
|
+
if (type === 'input_audio_buffer.speech_stopped') {
|
|
442
|
+
runtime.userSpeaking = false;
|
|
443
|
+
// Wait for committed input, not transcription (which can arrive much later).
|
|
444
|
+
return;
|
|
445
|
+
}
|
|
446
|
+
if (type === 'input_audio_buffer.committed' && runtime.managedTurns) {
|
|
447
|
+
runtime.waitingForUserResponse = false;
|
|
448
|
+
runtime.userTurnPending = true;
|
|
449
|
+
this.respondToUser(runtime);
|
|
450
|
+
return;
|
|
451
|
+
}
|
|
452
|
+
if (type === 'output_audio_buffer.started') {
|
|
453
|
+
runtime.playing = true;
|
|
454
|
+
return;
|
|
455
|
+
}
|
|
456
|
+
if (type === 'output_audio_buffer.stopped' || type === 'output_audio_buffer.cleared') {
|
|
457
|
+
runtime.playing = false;
|
|
458
|
+
this.flush(runtime);
|
|
459
|
+
return;
|
|
460
|
+
}
|
|
379
461
|
if (type === 'mcp_list_tools.completed') {
|
|
380
462
|
if (runtime.mode !== 'mcp')
|
|
381
463
|
return;
|
|
@@ -392,12 +474,22 @@ export class VoiceBroker {
|
|
|
392
474
|
// Server VAD can start a response without this sideband having sent
|
|
393
475
|
// `response.create`; broker nudges must not collide with it.
|
|
394
476
|
runtime.responseActive = true;
|
|
477
|
+
runtime.responseId = event.response?.id ?? null;
|
|
478
|
+
runtime.responseStartedAt = this.now();
|
|
479
|
+
runtime.firstOutputAt = null;
|
|
480
|
+
runtime.responseDone = false;
|
|
481
|
+
runtime.responseHadTool = false;
|
|
482
|
+
if (!runtime.managedTurns)
|
|
483
|
+
runtime.waitingForUserResponse = false;
|
|
484
|
+
else if (runtime.userSpeaking || runtime.waitingForUserResponse)
|
|
485
|
+
this.send(runtime, { type: 'response.cancel' });
|
|
395
486
|
return;
|
|
396
487
|
}
|
|
397
488
|
if (type === 'response.mcp_call.in_progress' && typeof event.item_id === 'string') {
|
|
398
489
|
runtime.responseHadTool = true;
|
|
399
490
|
runtime.pendingTools.add(event.item_id);
|
|
400
491
|
runtime.toolStarted.set(event.item_id, this.now());
|
|
492
|
+
runtime.toolEpoch.set(event.item_id, runtime.inputEpoch);
|
|
401
493
|
return;
|
|
402
494
|
}
|
|
403
495
|
if (type === 'response.function_call_arguments.done') {
|
|
@@ -418,11 +510,36 @@ export class VoiceBroker {
|
|
|
418
510
|
return;
|
|
419
511
|
}
|
|
420
512
|
if (type === 'response.done') {
|
|
421
|
-
runtime.responseActive = false;
|
|
422
|
-
runtime.responseDone = true;
|
|
423
513
|
const response = (event.response ?? {});
|
|
424
514
|
if (response.usage)
|
|
425
515
|
this.logUsage(callId, response.usage);
|
|
516
|
+
// Keep terminal outcomes even for an empty/cancelled answer, without logging prompts or errors containing secrets.
|
|
517
|
+
const safe = (value) => typeof value === 'string' ? value.replace(/[^a-zA-Z0-9_.-]/g, '_').slice(0, 100) : 'none';
|
|
518
|
+
this.log(`[voice] ${callId} response ${safe(response.id)} status=${safe(response.status)} reason=${safe(response.status_details?.reason)} code=${safe(response.status_details?.error?.code ?? response.status_details?.error?.type)} first_output_ms=${runtime.firstOutputAt === null ? 'none' : runtime.firstOutputAt - runtime.responseStartedAt}`);
|
|
519
|
+
if (response.id && runtime.finishedResponses.has(response.id))
|
|
520
|
+
return;
|
|
521
|
+
if (response.id) {
|
|
522
|
+
runtime.finishedResponses.add(response.id);
|
|
523
|
+
if (runtime.finishedResponses.size > 256)
|
|
524
|
+
runtime.finishedResponses.delete(runtime.finishedResponses.values().next().value);
|
|
525
|
+
}
|
|
526
|
+
if (response.id && runtime.responseId && response.id !== runtime.responseId)
|
|
527
|
+
return;
|
|
528
|
+
runtime.responseActive = false;
|
|
529
|
+
runtime.responseId = null;
|
|
530
|
+
runtime.responseDone = !response.status || response.status === 'completed';
|
|
531
|
+
if (runtime.typed.length) {
|
|
532
|
+
this.flushTyped(runtime);
|
|
533
|
+
return;
|
|
534
|
+
}
|
|
535
|
+
if (runtime.userTurnPending) {
|
|
536
|
+
this.respondToUser(runtime);
|
|
537
|
+
return;
|
|
538
|
+
}
|
|
539
|
+
if (!runtime.responseDone || runtime.waitingForUserResponse) {
|
|
540
|
+
runtime.responseHadTool = false;
|
|
541
|
+
return;
|
|
542
|
+
}
|
|
426
543
|
for (const item of response.output ?? []) {
|
|
427
544
|
if (item.type === 'mcp_call')
|
|
428
545
|
runtime.responseHadTool = true;
|
|
@@ -445,12 +562,50 @@ export class VoiceBroker {
|
|
|
445
562
|
this.flush(runtime);
|
|
446
563
|
}
|
|
447
564
|
}
|
|
565
|
+
flushTyped(runtime) {
|
|
566
|
+
if (runtime.responseActive || !runtime.typed.length)
|
|
567
|
+
return;
|
|
568
|
+
for (const text of runtime.typed.splice(0)) {
|
|
569
|
+
this.send(runtime, {
|
|
570
|
+
type: 'conversation.item.create',
|
|
571
|
+
item: { type: 'message', role: 'user', content: [{ type: 'input_text', text }] }
|
|
572
|
+
});
|
|
573
|
+
}
|
|
574
|
+
this.createResponse(runtime);
|
|
575
|
+
}
|
|
576
|
+
respondToUser(runtime) {
|
|
577
|
+
if (runtime.userTurnPending && this.createResponse(runtime))
|
|
578
|
+
runtime.userTurnPending = false;
|
|
579
|
+
}
|
|
580
|
+
/** All typed turns share the sideband scheduler with tools and background receipts. */
|
|
581
|
+
sendText(callId, text) {
|
|
582
|
+
const runtime = this.runtimes.get(callId);
|
|
583
|
+
if (runtime?.mode !== 'function' || !runtime.ready || runtime.socket.readyState !== 1)
|
|
584
|
+
return false;
|
|
585
|
+
runtime.inputEpoch++;
|
|
586
|
+
runtime.userSpeaking = false;
|
|
587
|
+
runtime.waitingForUserResponse = false;
|
|
588
|
+
runtime.userTurnPending = false;
|
|
589
|
+
runtime.responseDone = false;
|
|
590
|
+
runtime.responseHadTool = false;
|
|
591
|
+
runtime.typed.push(text);
|
|
592
|
+
this.send(runtime, { type: 'input_audio_buffer.clear' });
|
|
593
|
+
if (runtime.responseActive)
|
|
594
|
+
this.send(runtime, { type: 'response.cancel' });
|
|
595
|
+
this.send(runtime, { type: 'output_audio_buffer.clear' });
|
|
596
|
+
this.flushTyped(runtime);
|
|
597
|
+
return true;
|
|
598
|
+
}
|
|
599
|
+
isBrowserCall(callId) {
|
|
600
|
+
return this.calls.get(callId)?.mode === 'function';
|
|
601
|
+
}
|
|
448
602
|
/** Attach the relay sideband before the browser receives its SDP answer. */
|
|
449
603
|
registerWebRtc(callId, options = {}) {
|
|
450
604
|
this.calls.set(callId, {
|
|
451
605
|
callId,
|
|
452
606
|
acceptedAt: this.now(),
|
|
453
607
|
mode: 'function',
|
|
608
|
+
managedTurns: true,
|
|
454
609
|
ready: false,
|
|
455
610
|
greeted: false
|
|
456
611
|
});
|
|
@@ -4,6 +4,7 @@ import path from 'node:path';
|
|
|
4
4
|
import { DatabaseSync } from 'node:sqlite';
|
|
5
5
|
import { matchQuery } from "../search/coordinator.js";
|
|
6
6
|
import { HIT_CLOSE, HIT_OPEN } from "../shared.js";
|
|
7
|
+
import { voiceResponseOutcome } from "./response.js";
|
|
7
8
|
export const MAX_VOICE_SEARCH_CHARS = 500;
|
|
8
9
|
function object(value) {
|
|
9
10
|
return value && typeof value === 'object' && !Array.isArray(value) ? value : null;
|
|
@@ -310,6 +311,12 @@ export class VoiceHistory {
|
|
|
310
311
|
}
|
|
311
312
|
else if (type === 'response.done') {
|
|
312
313
|
const response = object(event.response);
|
|
314
|
+
const call = this.summary(callId);
|
|
315
|
+
if (call)
|
|
316
|
+
this.saveSummary({
|
|
317
|
+
...call,
|
|
318
|
+
responseOutcomes: [...(call.responseOutcomes ?? []), voiceResponseOutcome(response)].slice(-100)
|
|
319
|
+
});
|
|
313
320
|
if (Array.isArray(response?.output))
|
|
314
321
|
for (const item of response.output) {
|
|
315
322
|
this.item(callId, item, undefined, true);
|
|
@@ -7,6 +7,7 @@ export class PreviewStore {
|
|
|
7
7
|
file;
|
|
8
8
|
now;
|
|
9
9
|
previews = null;
|
|
10
|
+
presentations = new Map();
|
|
10
11
|
constructor(file, deps = {}) {
|
|
11
12
|
this.file = file;
|
|
12
13
|
this.now = deps.now ?? Date.now;
|
|
@@ -19,6 +20,19 @@ export class PreviewStore {
|
|
|
19
20
|
this.previews = Array.isArray(parsed)
|
|
20
21
|
? parsed.map(preview => preview.kind ? preview : { ...preview, kind: 'send_prompt' })
|
|
21
22
|
: [];
|
|
23
|
+
// A previous process may have dispatched before it lost the receipt.
|
|
24
|
+
let recovered = false;
|
|
25
|
+
for (const preview of this.previews) {
|
|
26
|
+
if (preview.status === 'claimed' && (!preview.outcome || preview.outcome.state === 'running')) {
|
|
27
|
+
preview.outcome = {
|
|
28
|
+
state: 'unknown',
|
|
29
|
+
message: 'The relay restarted before saving the receipt. Check the destination before retrying.'
|
|
30
|
+
};
|
|
31
|
+
recovered = true;
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
if (recovered)
|
|
35
|
+
this.write();
|
|
22
36
|
}
|
|
23
37
|
catch {
|
|
24
38
|
this.previews = [];
|
|
@@ -27,7 +41,9 @@ export class PreviewStore {
|
|
|
27
41
|
}
|
|
28
42
|
write() {
|
|
29
43
|
fs.mkdirSync(path.dirname(this.file), { recursive: true });
|
|
30
|
-
|
|
44
|
+
const temporary = `${this.file}.tmp`;
|
|
45
|
+
fs.writeFileSync(temporary, `${JSON.stringify(this.read(), null, 2)}\n`, { mode: 0o600 });
|
|
46
|
+
fs.renameSync(temporary, this.file);
|
|
31
47
|
fs.chmodSync(this.file, 0o600);
|
|
32
48
|
}
|
|
33
49
|
create(input) {
|
|
@@ -42,8 +58,8 @@ export class PreviewStore {
|
|
|
42
58
|
};
|
|
43
59
|
// Keep enough history to return an explicit `expired` refusal until the next
|
|
44
60
|
// preview, then bound this append-only credential file as calls accumulate.
|
|
45
|
-
this.
|
|
46
|
-
this.
|
|
61
|
+
this.retire(input.callId, createdAt);
|
|
62
|
+
this.read().push(preview);
|
|
47
63
|
this.write();
|
|
48
64
|
return preview;
|
|
49
65
|
}
|
|
@@ -57,11 +73,78 @@ export class PreviewStore {
|
|
|
57
73
|
expiresAt: createdAt + PREVIEW_TTL_MS,
|
|
58
74
|
status: 'ready'
|
|
59
75
|
};
|
|
60
|
-
this.
|
|
61
|
-
this.
|
|
76
|
+
this.retire(input.callId, createdAt);
|
|
77
|
+
this.read().push(preview);
|
|
62
78
|
this.write();
|
|
63
79
|
return preview;
|
|
64
80
|
}
|
|
81
|
+
retire(callId, now) {
|
|
82
|
+
// Expiry revokes approval, not the review content or its receipt. Keep 30 days.
|
|
83
|
+
this.previews = this.read().filter(candidate => candidate.createdAt >= now - 30 * 86_400_000);
|
|
84
|
+
for (const preview of this.previews) {
|
|
85
|
+
if (preview.callId === callId && preview.status === 'ready')
|
|
86
|
+
preview.status = 'superseded';
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
list(callId) {
|
|
90
|
+
return this.read()
|
|
91
|
+
.filter(preview => preview.callId === callId)
|
|
92
|
+
.map(preview => structuredClone(preview));
|
|
93
|
+
}
|
|
94
|
+
get(callId, token) {
|
|
95
|
+
return this.list(callId).find(preview => preview.token === token);
|
|
96
|
+
}
|
|
97
|
+
present(callId, token) {
|
|
98
|
+
const preview = this.read().find(candidate => candidate.callId === callId && candidate.token === token);
|
|
99
|
+
if (preview?.status !== 'ready')
|
|
100
|
+
return false;
|
|
101
|
+
preview.presented = true;
|
|
102
|
+
this.write();
|
|
103
|
+
this.presentations.get(token)?.(true);
|
|
104
|
+
return true;
|
|
105
|
+
}
|
|
106
|
+
waitForPresentation(token, timeoutMs = 1800) {
|
|
107
|
+
if (this.read().find(preview => preview.token === token)?.presented)
|
|
108
|
+
return Promise.resolve(true);
|
|
109
|
+
return new Promise(resolve => {
|
|
110
|
+
const finish = (shown) => {
|
|
111
|
+
clearTimeout(timer);
|
|
112
|
+
this.presentations.delete(token);
|
|
113
|
+
resolve(shown);
|
|
114
|
+
};
|
|
115
|
+
const timer = setTimeout(() => finish(false), timeoutMs);
|
|
116
|
+
this.presentations.set(token, finish);
|
|
117
|
+
});
|
|
118
|
+
}
|
|
119
|
+
settle(token, outcome) {
|
|
120
|
+
const preview = this.read().find(candidate => candidate.token === token);
|
|
121
|
+
if (preview?.status !== 'claimed')
|
|
122
|
+
return;
|
|
123
|
+
preview.outcome = outcome;
|
|
124
|
+
this.write();
|
|
125
|
+
}
|
|
126
|
+
edit(callId, token, text) {
|
|
127
|
+
const preview = this.get(callId, token);
|
|
128
|
+
if (preview?.status !== 'ready')
|
|
129
|
+
return null;
|
|
130
|
+
return preview.kind === 'send_prompt'
|
|
131
|
+
? this.create({
|
|
132
|
+
callId,
|
|
133
|
+
workspaceId: preview.workspaceId,
|
|
134
|
+
sessionId: preview.sessionId,
|
|
135
|
+
targetLabel: preview.targetLabel,
|
|
136
|
+
text
|
|
137
|
+
})
|
|
138
|
+
: this.createWorkspace({ callId, repo: preview.repo, prompt: text });
|
|
139
|
+
}
|
|
140
|
+
pauseReview(callId, token, paused) {
|
|
141
|
+
const preview = this.read().find(preview => preview.callId === callId && preview.token === token);
|
|
142
|
+
if (preview?.status !== 'ready')
|
|
143
|
+
return false;
|
|
144
|
+
preview.reviewPaused = paused;
|
|
145
|
+
this.write();
|
|
146
|
+
return true;
|
|
147
|
+
}
|
|
65
148
|
available(token, callId) {
|
|
66
149
|
const preview = this.read().find(candidate => candidate.token === token);
|
|
67
150
|
if (!preview)
|
|
@@ -72,10 +155,13 @@ export class PreviewStore {
|
|
|
72
155
|
return { ok: false, reason: 'already-used' };
|
|
73
156
|
if (preview.callId !== callId)
|
|
74
157
|
return { ok: false, reason: 'foreign-call' };
|
|
158
|
+
if (preview.reviewPaused)
|
|
159
|
+
return { ok: false, reason: 'editing' };
|
|
75
160
|
return { ok: true, preview };
|
|
76
161
|
}
|
|
77
162
|
markClaimed(preview) {
|
|
78
163
|
preview.status = 'claimed';
|
|
164
|
+
preview.outcome = { state: 'running' };
|
|
79
165
|
this.write();
|
|
80
166
|
return { ...preview };
|
|
81
167
|
}
|