@flowingspring/dsh-voco 0.2.0 → 0.2.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +55 -30
- package/README.zh.md +55 -30
- package/lib/client.js +52 -26
- package/lib/client.js.map +1 -1
- package/lib/plugins/voice-assistant.js +171 -57
- package/lib/plugins/voice-local.js +30 -14
- package/package.json +40 -27
|
@@ -13,25 +13,21 @@ const VOICE_MESSAGE_SECTION_ORDER = 118;
|
|
|
13
13
|
* Install `send_voice_message` and its guidance into one delegated task Agent scope.
|
|
14
14
|
* @param agentCtx - task-Agent context receiving the tool and guidance.
|
|
15
15
|
* @param send - binding-owned message receiver.
|
|
16
|
+
* @param resolveDelegationId - resolves the active delegation without exposing its id to the model or transcript.
|
|
16
17
|
* @returns disposer that revokes both registrations.
|
|
17
18
|
*/
|
|
18
|
-
function installVoiceMessageTool(agentCtx, send) {
|
|
19
|
+
function installVoiceMessageTool(agentCtx, send, resolveDelegationId) {
|
|
19
20
|
const disposeSection = agentCtx.systemPrompt.section({
|
|
20
21
|
name: "tool:send-voice-message",
|
|
21
22
|
order: VOICE_MESSAGE_SECTION_ORDER,
|
|
22
|
-
text: "
|
|
23
|
+
text: "This Agent handles tasks delegated by the realtime voice assistant. While a realtime voice delegation is active, use send_voice_message to keep the voice assistant informed. The tool is already bound to the exact active delegation; never ask for, invent, or expose an internal delegation id. Report structured events with type and detail. Use progress for meaningful progress, warning or error for important conditions, question when user input is required, and result exactly once before a successful final turn ends. Put the complete factual content in detail; the Voice layer independently rewrites every reported event against the user request. Keep detail complete enough to support an accurate rewrite. The full report remains in the task UI. The voice assistant does not automatically see your transcript, tool output, or reasoning. A result is held until the turn actually succeeds. A question leaves the task waiting for a user reply. Reporting never ends your turn. Do not use this tool when no realtime voice delegation is active."
|
|
23
24
|
});
|
|
24
25
|
let disposeTool;
|
|
25
26
|
try {
|
|
26
27
|
disposeTool = agentCtx.tools.register(defineTool({
|
|
27
28
|
name: "send_voice_message",
|
|
28
|
-
description: "Send a user-facing status or final result to the realtime voice assistant for the exact active delegation using type and complete factual detail. Progress, warnings, errors, and questions may repeat when meaningful. result may be called once. The Voice layer independently rewrites detail into conversational speech, so this tool accepts no Agent-authored speech field. COMPLETE is held until the Agent turn succeeds; it does not finish the turn. The voice assistant does not otherwise see this Agent transcript or tool output.",
|
|
29
|
+
description: "Send a user-facing status or final result to the realtime voice assistant for the exact active delegation using type and complete factual detail. The delegation is bound automatically; no delegation id is accepted. Progress, warnings, errors, and questions may repeat when meaningful. result may be called once. The Voice layer independently rewrites detail into conversational speech, so this tool accepts no Agent-authored speech field. COMPLETE is held until the Agent turn succeeds; it does not finish the turn. The voice assistant does not otherwise see this Agent transcript or tool output.",
|
|
29
30
|
parameters: {
|
|
30
|
-
delegation_id: {
|
|
31
|
-
type: "string",
|
|
32
|
-
required: true,
|
|
33
|
-
description: "Exact delegation_id from the realtime_delegation request envelope."
|
|
34
|
-
},
|
|
35
31
|
type: {
|
|
36
32
|
type: "string",
|
|
37
33
|
enum: [
|
|
@@ -85,8 +81,10 @@ function installVoiceMessageTool(agentCtx, send) {
|
|
|
85
81
|
const type = args.type ?? (args.channel === "COMPLETE" ? "result" : "progress");
|
|
86
82
|
const detail = args.detail.trim();
|
|
87
83
|
if (detail === "") throw new Error("send_voice_message detail must be non-empty");
|
|
84
|
+
const delegationId = resolveDelegationId();
|
|
85
|
+
if (delegationId === void 0) throw new Error("send_voice_message has no active voice delegation");
|
|
88
86
|
return Promise.resolve(send({
|
|
89
|
-
delegationId
|
|
87
|
+
delegationId,
|
|
90
88
|
channel: type === "result" ? "COMPLETE" : "STATUS",
|
|
91
89
|
type,
|
|
92
90
|
detail
|
|
@@ -154,6 +152,9 @@ try {
|
|
|
154
152
|
if (hostSession.KNOWN_SESSION_EVENT_TYPES !== void 0) registerVoiceSessionEventTypes(hostSession.KNOWN_SESSION_EVENT_TYPES);
|
|
155
153
|
}
|
|
156
154
|
} catch {}
|
|
155
|
+
function sameModelSelection(left, right) {
|
|
156
|
+
return left.provider === right.provider && left.model === right.model && left.reasoningEffort === right.reasoningEffort;
|
|
157
|
+
}
|
|
157
158
|
const REWRITE_SYSTEM_PROMPT = `你是语音模式下的自然回复编辑器。你的任务是把后台处理结果改写成准确、自然、适合直接朗读的中文回复。
|
|
158
159
|
|
|
159
160
|
必须遵守:
|
|
@@ -249,7 +250,13 @@ function apply(ctx, config = {}) {
|
|
|
249
250
|
const voiceId = binding.voiceSessionId;
|
|
250
251
|
if (voiceId === void 0 || !binding.voiceAttached || text.trim() === "") return;
|
|
251
252
|
if (ctx.voice.appendSpeechText(voiceId, text)) {
|
|
252
|
-
if (flush)
|
|
253
|
+
if (flush) {
|
|
254
|
+
debugVoiceLatency("tts-request", {
|
|
255
|
+
taskId,
|
|
256
|
+
textLength: text.length
|
|
257
|
+
});
|
|
258
|
+
ctx.voice.requestResponse(voiceId, { kind: "automatic" });
|
|
259
|
+
}
|
|
253
260
|
return;
|
|
254
261
|
}
|
|
255
262
|
append(binding, {
|
|
@@ -279,6 +286,14 @@ function apply(ctx, config = {}) {
|
|
|
279
286
|
const abort = new AbortController();
|
|
280
287
|
binding.rewriteAbort = abort;
|
|
281
288
|
const selection = ctx.agentDefaultModel.currentSelection();
|
|
289
|
+
debugVoiceLatency("rewrite-start", {
|
|
290
|
+
taskId,
|
|
291
|
+
eventType,
|
|
292
|
+
provider: selection.provider,
|
|
293
|
+
model: selection.model,
|
|
294
|
+
selectedReasoningEffort: selection.reasoningEffort,
|
|
295
|
+
originalLength: original.length
|
|
296
|
+
});
|
|
282
297
|
const prompt = [
|
|
283
298
|
"根据用户原话和处理结果,生成一条可以直接显示并朗读的最终回复。",
|
|
284
299
|
REWRITE_EVENT_INSTRUCTIONS[eventType],
|
|
@@ -309,13 +324,13 @@ function apply(ctx, config = {}) {
|
|
|
309
324
|
for await (const chunk of llm.stream({
|
|
310
325
|
provider: selection.provider,
|
|
311
326
|
model: selection.model,
|
|
312
|
-
...selection.reasoningEffort === void 0 ? {} : { reasoningEffort: selection.reasoningEffort },
|
|
313
327
|
messages: [message],
|
|
314
328
|
system: REWRITE_SYSTEM_PROMPT,
|
|
315
329
|
signal: abort.signal
|
|
316
330
|
})) {
|
|
317
331
|
if (generation !== binding.rewriteGeneration || abort.signal.aborted) return;
|
|
318
332
|
if (chunk.type !== "text-delta") continue;
|
|
333
|
+
if (rewritten === "") debugVoiceLatency("rewrite-first-text", { taskId });
|
|
319
334
|
rewritten += chunk.text;
|
|
320
335
|
pending += chunk.text;
|
|
321
336
|
const split = speechFragments(pending, false);
|
|
@@ -329,6 +344,10 @@ function apply(ctx, config = {}) {
|
|
|
329
344
|
if (rewritten.trim() === "" && pending.trim() === "") speakFragment(binding, taskId, fallbackEventSpeech(eventType, original));
|
|
330
345
|
} catch (error) {
|
|
331
346
|
if (!abort.signal.aborted && generation === binding.rewriteGeneration) {
|
|
347
|
+
debugVoiceLatency("rewrite-error", {
|
|
348
|
+
taskId,
|
|
349
|
+
error: String(error)
|
|
350
|
+
});
|
|
332
351
|
ctx.logger.warn(error instanceof Error ? error : new Error(String(error)));
|
|
333
352
|
speakFragment(binding, taskId, fallbackEventSpeech(eventType, original));
|
|
334
353
|
}
|
|
@@ -429,12 +448,40 @@ function apply(ctx, config = {}) {
|
|
|
429
448
|
binding.agent = handle.agent;
|
|
430
449
|
return handle.agent;
|
|
431
450
|
};
|
|
432
|
-
const
|
|
451
|
+
const loggedSourceSelection = (binding) => {
|
|
452
|
+
const event = ctx.sessions.get(binding.sessionId)?.events.findLast((candidate) => candidate.type === "request/header");
|
|
453
|
+
if (event?.type !== "request/header") return void 0;
|
|
454
|
+
const config = event.data.header.config;
|
|
455
|
+
return {
|
|
456
|
+
selection: {
|
|
457
|
+
provider: config.provider,
|
|
458
|
+
model: config.model,
|
|
459
|
+
...config.reasoningEffort === void 0 ? {} : { reasoningEffort: config.reasoningEffort }
|
|
460
|
+
},
|
|
461
|
+
seq: event.seq
|
|
462
|
+
};
|
|
463
|
+
};
|
|
464
|
+
const taskModelSelection = (binding) => {
|
|
465
|
+
const defaults = ctx.agentDefaultModel.currentSelection();
|
|
466
|
+
const logged = loggedSourceSelection(binding);
|
|
467
|
+
const current = binding.continuousTaskAgent?.selection;
|
|
468
|
+
if (current === void 0) return {
|
|
469
|
+
selection: logged?.selection ?? defaults,
|
|
470
|
+
sourceHeaderSeq: logged?.seq
|
|
471
|
+
};
|
|
472
|
+
if (logged !== void 0 && logged.seq !== binding.continuousTaskAgent?.sourceHeaderSeq) return {
|
|
473
|
+
selection: logged.selection,
|
|
474
|
+
sourceHeaderSeq: logged.seq
|
|
475
|
+
};
|
|
476
|
+
return {
|
|
477
|
+
selection: sameModelSelection(defaults, current) ? current : defaults,
|
|
478
|
+
sourceHeaderSeq: logged?.seq
|
|
479
|
+
};
|
|
480
|
+
};
|
|
481
|
+
const createTaskAgent = async (binding, taskSessionId, selection) => {
|
|
433
482
|
const sourceAgent = await ensureAgent(binding);
|
|
434
483
|
const sourceSession = ctx.sessions.get(binding.sessionId);
|
|
435
484
|
if (sourceSession === void 0) throw new Error(`voice-assistant: source session "${binding.sessionId}" is not live`);
|
|
436
|
-
const defaults = ctx.agentDefaultModel.currentSelection();
|
|
437
|
-
const header = foldRequestHeader(sourceSession.events);
|
|
438
485
|
const presetId = resolveSessionPreset(sourceSession);
|
|
439
486
|
const presets = ctx.get("agentPresets");
|
|
440
487
|
let disposeVoiceMessage;
|
|
@@ -445,12 +492,12 @@ function apply(ctx, config = {}) {
|
|
|
445
492
|
...presetId === void 0 ? {} : { agentPreset: presetId }
|
|
446
493
|
},
|
|
447
494
|
agentOptions: {
|
|
448
|
-
provider:
|
|
449
|
-
model:
|
|
495
|
+
provider: selection.provider,
|
|
496
|
+
model: selection.model
|
|
450
497
|
},
|
|
451
498
|
setup: (agentCtx) => {
|
|
452
499
|
presets?.composeFrom(agentCtx, sourceAgent.ctx);
|
|
453
|
-
disposeVoiceMessage = installVoiceMessageTool(agentCtx, (input) => sendVoiceMessage(binding, input));
|
|
500
|
+
disposeVoiceMessage = installVoiceMessageTool(agentCtx, (input) => sendVoiceMessage(binding, input), () => binding.active?.id);
|
|
454
501
|
}
|
|
455
502
|
});
|
|
456
503
|
handles.set(taskSessionId, handle);
|
|
@@ -471,25 +518,24 @@ function apply(ctx, config = {}) {
|
|
|
471
518
|
disposeVoiceMessage
|
|
472
519
|
};
|
|
473
520
|
};
|
|
474
|
-
const resumeTaskAgent = async (binding, taskSessionId) => {
|
|
521
|
+
const resumeTaskAgent = async (binding, taskSessionId, selection) => {
|
|
475
522
|
const live = ctx.agents.get(taskSessionId);
|
|
476
523
|
if (live !== void 0) return {
|
|
477
524
|
agent: live,
|
|
478
|
-
disposeVoiceMessage: installVoiceMessageTool(live.ctx, (input) => sendVoiceMessage(binding, input))
|
|
525
|
+
disposeVoiceMessage: installVoiceMessageTool(live.ctx, (input) => sendVoiceMessage(binding, input), () => binding.active?.id)
|
|
479
526
|
};
|
|
480
527
|
const sourceAgent = await ensureAgent(binding);
|
|
481
|
-
const defaults = ctx.agentDefaultModel.currentSelection();
|
|
482
528
|
const presets = ctx.get("agentPresets");
|
|
483
529
|
let disposeVoiceMessage;
|
|
484
530
|
const handle = await ctx.agents.resume({
|
|
485
531
|
resumeSessionId: taskSessionId,
|
|
486
532
|
agentOptions: {
|
|
487
|
-
provider:
|
|
488
|
-
model:
|
|
533
|
+
provider: selection.provider,
|
|
534
|
+
model: selection.model
|
|
489
535
|
},
|
|
490
536
|
setup: (agentCtx) => {
|
|
491
537
|
presets?.composeFrom(agentCtx, sourceAgent.ctx);
|
|
492
|
-
disposeVoiceMessage = installVoiceMessageTool(agentCtx, (input) => sendVoiceMessage(binding, input));
|
|
538
|
+
disposeVoiceMessage = installVoiceMessageTool(agentCtx, (input) => sendVoiceMessage(binding, input), () => binding.active?.id);
|
|
493
539
|
}
|
|
494
540
|
});
|
|
495
541
|
handles.set(taskSessionId, handle);
|
|
@@ -504,7 +550,40 @@ function apply(ctx, config = {}) {
|
|
|
504
550
|
};
|
|
505
551
|
};
|
|
506
552
|
const ensureContinuousTaskAgent = async (binding) => {
|
|
507
|
-
|
|
553
|
+
const { selection, sourceHeaderSeq } = taskModelSelection(binding);
|
|
554
|
+
const current = binding.continuousTaskAgent;
|
|
555
|
+
if (current !== void 0 && sameModelSelection(current.selection, selection)) {
|
|
556
|
+
if (current.sourceHeaderSeq === sourceHeaderSeq) return current;
|
|
557
|
+
const refreshed = {
|
|
558
|
+
...current,
|
|
559
|
+
sourceHeaderSeq
|
|
560
|
+
};
|
|
561
|
+
binding.continuousTaskAgent = refreshed;
|
|
562
|
+
return refreshed;
|
|
563
|
+
}
|
|
564
|
+
if (current !== void 0) {
|
|
565
|
+
binding.continuousTaskAgent = void 0;
|
|
566
|
+
taskBindings.delete(current.taskSessionId);
|
|
567
|
+
try {
|
|
568
|
+
await current.agent.whenIdle();
|
|
569
|
+
} catch (error) {
|
|
570
|
+
ctx.logger.warn(error instanceof Error ? error : new Error(String(error)));
|
|
571
|
+
}
|
|
572
|
+
try {
|
|
573
|
+
current.disposeVoiceMessage();
|
|
574
|
+
} catch (error) {
|
|
575
|
+
ctx.logger.warn(error instanceof Error ? error : new Error(String(error)));
|
|
576
|
+
}
|
|
577
|
+
const handle = handles.get(current.taskSessionId);
|
|
578
|
+
if (handle !== void 0) {
|
|
579
|
+
handles.delete(current.taskSessionId);
|
|
580
|
+
try {
|
|
581
|
+
await handle.dispose();
|
|
582
|
+
} catch (error) {
|
|
583
|
+
ctx.logger.warn(error instanceof Error ? error : new Error(String(error)));
|
|
584
|
+
}
|
|
585
|
+
}
|
|
586
|
+
}
|
|
508
587
|
const sourceSession = requireSourceSession(binding);
|
|
509
588
|
const previousState = sourceSession.events.findLast((event) => event.type === "voice/agent-binding-state");
|
|
510
589
|
const previousBound = sourceSession.events.findLast((event) => event.type === "voice/task-session-bound");
|
|
@@ -513,7 +592,9 @@ function apply(ctx, config = {}) {
|
|
|
513
592
|
if (previousTaskSessionId !== void 0) try {
|
|
514
593
|
const resource = {
|
|
515
594
|
taskSessionId: previousTaskSessionId,
|
|
516
|
-
|
|
595
|
+
selection,
|
|
596
|
+
sourceHeaderSeq,
|
|
597
|
+
...await resumeTaskAgent(binding, previousTaskSessionId, selection)
|
|
517
598
|
};
|
|
518
599
|
binding.continuousTaskAgent = resource;
|
|
519
600
|
taskBindings.set(resource.taskSessionId, binding);
|
|
@@ -524,7 +605,9 @@ function apply(ctx, config = {}) {
|
|
|
524
605
|
const taskSessionId = SessionId(`session-${randomUUID()}`);
|
|
525
606
|
const resource = {
|
|
526
607
|
taskSessionId,
|
|
527
|
-
|
|
608
|
+
selection,
|
|
609
|
+
sourceHeaderSeq,
|
|
610
|
+
...await createTaskAgent(binding, taskSessionId, selection)
|
|
528
611
|
};
|
|
529
612
|
binding.continuousTaskAgent = resource;
|
|
530
613
|
taskBindings.set(taskSessionId, binding);
|
|
@@ -641,7 +724,7 @@ function apply(ctx, config = {}) {
|
|
|
641
724
|
}, false);
|
|
642
725
|
agent.steer(message);
|
|
643
726
|
};
|
|
644
|
-
const onTaskCommand = async (binding, voiceSessionId, call) => {
|
|
727
|
+
const onTaskCommand = async (binding, voiceSessionId, call, taskIdOverride) => {
|
|
645
728
|
const complete = (result) => {
|
|
646
729
|
ctx.voice.completeTaskCommand(voiceSessionId, call.id, result);
|
|
647
730
|
};
|
|
@@ -670,16 +753,21 @@ function apply(ctx, config = {}) {
|
|
|
670
753
|
route = await routeFrontendInput(ctx, requireSourceSession(binding).events, call.command.input);
|
|
671
754
|
} catch (error) {
|
|
672
755
|
ctx.logger.warn(error instanceof Error ? error : new Error(String(error)));
|
|
673
|
-
route = {
|
|
756
|
+
route = {
|
|
757
|
+
action: "delegate",
|
|
758
|
+
acknowledgement: DEFAULT_DELEGATION_ACKNOWLEDGEMENT
|
|
759
|
+
};
|
|
674
760
|
}
|
|
675
761
|
if (route.action === "delegate") {
|
|
762
|
+
const taskId = VoiceTaskId(randomUUID());
|
|
763
|
+
speakFragment(binding, taskId, route.acknowledgement);
|
|
676
764
|
await onTaskCommand(binding, voiceSessionId, {
|
|
677
765
|
id: call.id,
|
|
678
766
|
command: {
|
|
679
767
|
type: "realtime_delegation",
|
|
680
768
|
input: call.command.input
|
|
681
769
|
}
|
|
682
|
-
});
|
|
770
|
+
}, taskId);
|
|
683
771
|
return;
|
|
684
772
|
}
|
|
685
773
|
complete({ kind: "handled" });
|
|
@@ -696,16 +784,17 @@ function apply(ctx, config = {}) {
|
|
|
696
784
|
return;
|
|
697
785
|
}
|
|
698
786
|
cancelRewrite(binding);
|
|
699
|
-
const taskId = VoiceTaskId(randomUUID());
|
|
787
|
+
const taskId = taskIdOverride ?? VoiceTaskId(randomUUID());
|
|
700
788
|
const continuous = (config.taskSessionPolicy ?? "isolated") === "continuous";
|
|
701
789
|
let created;
|
|
702
790
|
try {
|
|
703
791
|
if (continuous) created = await ensureContinuousTaskAgent(binding);
|
|
704
792
|
else {
|
|
705
793
|
const taskSessionId = SessionId(`session-${randomUUID()}`);
|
|
794
|
+
const { selection } = taskModelSelection(binding);
|
|
706
795
|
created = {
|
|
707
796
|
taskSessionId,
|
|
708
|
-
...await createTaskAgent(binding, taskSessionId)
|
|
797
|
+
...await createTaskAgent(binding, taskSessionId, selection)
|
|
709
798
|
};
|
|
710
799
|
}
|
|
711
800
|
} catch (error) {
|
|
@@ -715,10 +804,21 @@ function apply(ctx, config = {}) {
|
|
|
715
804
|
const message = createUserMessage({
|
|
716
805
|
content: [{
|
|
717
806
|
type: "text",
|
|
718
|
-
text:
|
|
807
|
+
text: call.command.input
|
|
719
808
|
}],
|
|
720
809
|
source: { kind: "user" }
|
|
721
810
|
});
|
|
811
|
+
const transcriptContext = call.command.transcriptDelta?.trim();
|
|
812
|
+
const contextMessage = transcriptContext === void 0 || transcriptContext === "" ? void 0 : createUserMessage({
|
|
813
|
+
content: [{
|
|
814
|
+
type: "text",
|
|
815
|
+
text: `语音转写补充上下文:${transcriptContext}`
|
|
816
|
+
}],
|
|
817
|
+
source: {
|
|
818
|
+
kind: "plugin",
|
|
819
|
+
plugin: "voice-assistant"
|
|
820
|
+
}
|
|
821
|
+
});
|
|
722
822
|
const task = {
|
|
723
823
|
id: taskId,
|
|
724
824
|
interactionMode: "frontend-agent",
|
|
@@ -738,8 +838,12 @@ function apply(ctx, config = {}) {
|
|
|
738
838
|
taskSessionId: created.taskSessionId,
|
|
739
839
|
input: call.command.input
|
|
740
840
|
});
|
|
841
|
+
if (contextMessage !== void 0) created.agent.inject(contextMessage);
|
|
741
842
|
created.agent.followup(message);
|
|
742
843
|
} catch (error) {
|
|
844
|
+
if (contextMessage !== void 0) try {
|
|
845
|
+
created.agent.inbox.remove(contextMessage.id);
|
|
846
|
+
} catch {}
|
|
743
847
|
if (!continuous) taskBindings.delete(created.taskSessionId);
|
|
744
848
|
binding.active = void 0;
|
|
745
849
|
try {
|
|
@@ -780,7 +884,7 @@ function apply(ctx, config = {}) {
|
|
|
780
884
|
const message = createUserMessage({
|
|
781
885
|
content: [{
|
|
782
886
|
type: "text",
|
|
783
|
-
text:
|
|
887
|
+
text: call.command.message
|
|
784
888
|
}],
|
|
785
889
|
source: {
|
|
786
890
|
kind: "plugin",
|
|
@@ -953,6 +1057,8 @@ function apply(ctx, config = {}) {
|
|
|
953
1057
|
if (session.interactionMode === "frontend-agent") enqueue(binding, () => binding.voiceSessionId === session.id ? onTaskCommand(binding, session.id, event.call) : void 0);
|
|
954
1058
|
return;
|
|
955
1059
|
case "output_audio.started":
|
|
1060
|
+
debugVoiceLatency("audio-started");
|
|
1061
|
+
return;
|
|
956
1062
|
case "output_audio.delta":
|
|
957
1063
|
case "output_audio.done":
|
|
958
1064
|
case "task.observation":
|
|
@@ -1064,6 +1170,19 @@ function speechFragments(text, flush) {
|
|
|
1064
1170
|
rest
|
|
1065
1171
|
};
|
|
1066
1172
|
}
|
|
1173
|
+
const DEFAULT_DELEGATION_ACKNOWLEDGEMENT = "好的,我先查看一下。";
|
|
1174
|
+
const MAX_DELEGATION_ACKNOWLEDGEMENT_LENGTH = 40;
|
|
1175
|
+
const VOICE_LATENCY_DEBUG_PREFIX = "[DEBUG-VOICE-LATENCY]";
|
|
1176
|
+
function debugVoiceLatency(...values) {
|
|
1177
|
+
if (globalThis.process?.env?.DSH_VOICE_LATENCY_DEBUG !== "1") return;
|
|
1178
|
+
console.info(VOICE_LATENCY_DEBUG_PREFIX, (/* @__PURE__ */ new Date()).toISOString(), ...values);
|
|
1179
|
+
}
|
|
1180
|
+
function delegationAcknowledgement(value) {
|
|
1181
|
+
if (typeof value !== "string") return DEFAULT_DELEGATION_ACKNOWLEDGEMENT;
|
|
1182
|
+
const normalized = value.replace(/\s+/gu, " ").trim();
|
|
1183
|
+
if (normalized === "" || normalized.length > MAX_DELEGATION_ACKNOWLEDGEMENT_LENGTH) return DEFAULT_DELEGATION_ACKNOWLEDGEMENT;
|
|
1184
|
+
return normalized;
|
|
1185
|
+
}
|
|
1067
1186
|
async function routeFrontendInput(ctx, events, input) {
|
|
1068
1187
|
let llm;
|
|
1069
1188
|
try {
|
|
@@ -1071,7 +1190,10 @@ async function routeFrontendInput(ctx, events, input) {
|
|
|
1071
1190
|
} catch {
|
|
1072
1191
|
llm = void 0;
|
|
1073
1192
|
}
|
|
1074
|
-
if (llm === void 0) return {
|
|
1193
|
+
if (llm === void 0) return {
|
|
1194
|
+
action: "delegate",
|
|
1195
|
+
acknowledgement: DEFAULT_DELEGATION_ACKNOWLEDGEMENT
|
|
1196
|
+
};
|
|
1075
1197
|
const selection = ctx.agentDefaultModel.currentSelection();
|
|
1076
1198
|
const recentConversation = events.flatMap((event) => {
|
|
1077
1199
|
if (event.type !== "voice/utterance-end" || event.data.state !== "completed") return [];
|
|
@@ -1085,10 +1207,12 @@ async function routeFrontendInput(ctx, events, input) {
|
|
|
1085
1207
|
"判断下面这句话应该由语音助手直接回答,还是委派给后台编码 Agent。",
|
|
1086
1208
|
"普通寒暄、日常对话、无需读取本地项目或调用工具即可回答的问题,选择 chat,并直接给出自然简洁的中文回复。",
|
|
1087
1209
|
"只有需要查看或修改工作区文件、运行命令、测试、安装依赖或执行其他工具操作时,才选择 delegate。",
|
|
1210
|
+
"选择 delegate 时,同时给出一句简短自然的 acknowledgement,表示接下来要做什么。不能声称任务已经完成、已经找到结果,也不要提后台 Agent、工具或路由。",
|
|
1211
|
+
"acknowledgement 只能有一句,最多 40 个字符,例如“好的,我先检查一下相关代码。”。",
|
|
1088
1212
|
"只输出一行 JSON,不要 Markdown。格式只能是:",
|
|
1089
1213
|
"{\"action\":\"chat\",\"reply\":\"...\"}",
|
|
1090
1214
|
"或:",
|
|
1091
|
-
"{\"action\":\"delegate\"}",
|
|
1215
|
+
"{\"action\":\"delegate\",\"acknowledgement\":\"...\"}",
|
|
1092
1216
|
"",
|
|
1093
1217
|
"最近对话:",
|
|
1094
1218
|
recentConversation || "(无)",
|
|
@@ -1115,7 +1239,10 @@ async function routeFrontendInput(ctx, events, input) {
|
|
|
1115
1239
|
action: "chat",
|
|
1116
1240
|
reply: parsed.reply.trim()
|
|
1117
1241
|
};
|
|
1118
|
-
if (parsed.action === "delegate") return {
|
|
1242
|
+
if (parsed.action === "delegate") return {
|
|
1243
|
+
action: "delegate",
|
|
1244
|
+
acknowledgement: delegationAcknowledgement(parsed.acknowledgement)
|
|
1245
|
+
};
|
|
1119
1246
|
throw new Error("voice frontend router returned an invalid decision");
|
|
1120
1247
|
}
|
|
1121
1248
|
function fallbackSpeechText(text) {
|
|
@@ -1155,6 +1282,13 @@ function observeSessionEvent(binding, event, append, config, onDisposeError, rew
|
|
|
1155
1282
|
}
|
|
1156
1283
|
if (event.type !== "turn/end" || event.data.turn !== task.taskTurn) return false;
|
|
1157
1284
|
const status = terminalStatus(event.data.reason.kind);
|
|
1285
|
+
const failureReason = status === "failed" ? event.data.reason.kind === "error" ? event.data.reason.error.message : event.data.reason.kind : void 0;
|
|
1286
|
+
debugVoiceLatency("turn-end", {
|
|
1287
|
+
taskId: task.id,
|
|
1288
|
+
turn: task.taskTurn,
|
|
1289
|
+
status,
|
|
1290
|
+
...failureReason === void 0 ? {} : { failureReason }
|
|
1291
|
+
});
|
|
1158
1292
|
if (status === "completed" && task.waitingUser && task.completionDetail === void 0) {
|
|
1159
1293
|
delete task.taskTurn;
|
|
1160
1294
|
delete task.lastAssistantMessage;
|
|
@@ -1180,7 +1314,7 @@ function observeSessionEvent(binding, event, append, config, onDisposeError, rew
|
|
|
1180
1314
|
},
|
|
1181
1315
|
...message === void 0 ? {} : { voiceMessage: message },
|
|
1182
1316
|
...announcement === void 0 ? {} : { announcement },
|
|
1183
|
-
...
|
|
1317
|
+
...failureReason === void 0 ? {} : { reason: failureReason }
|
|
1184
1318
|
}, message !== void 0 || announcement !== void 0, true);
|
|
1185
1319
|
if (rewriteFinalResult) rewrite(task, resultText);
|
|
1186
1320
|
const disposeVoiceMessage = task.disposeVoiceMessage;
|
|
@@ -1216,26 +1350,6 @@ function taskRejection(binding, taskId) {
|
|
|
1216
1350
|
message: `task "${taskId}" is being cancelled`
|
|
1217
1351
|
};
|
|
1218
1352
|
}
|
|
1219
|
-
function renderRealtimeDelegation(taskId, input, transcriptDelta) {
|
|
1220
|
-
return [
|
|
1221
|
-
"<realtime_delegation>",
|
|
1222
|
-
` <delegation_id>${escapeXmlText(taskId)}</delegation_id>`,
|
|
1223
|
-
` <input>${escapeXmlText(input)}</input>`,
|
|
1224
|
-
...transcriptDelta === void 0 ? [] : [` <transcript_delta>${escapeXmlText(transcriptDelta)}</transcript_delta>`],
|
|
1225
|
-
"</realtime_delegation>"
|
|
1226
|
-
].join("\n");
|
|
1227
|
-
}
|
|
1228
|
-
function renderRealtimeDelegationUpdate(taskId, message) {
|
|
1229
|
-
return [
|
|
1230
|
-
"<realtime_delegation_update>",
|
|
1231
|
-
` <delegation_id>${escapeXmlText(taskId)}</delegation_id>`,
|
|
1232
|
-
` <message>${escapeXmlText(message)}</message>`,
|
|
1233
|
-
"</realtime_delegation_update>"
|
|
1234
|
-
].join("\n");
|
|
1235
|
-
}
|
|
1236
|
-
function escapeXmlText(value) {
|
|
1237
|
-
return value.replaceAll("&", "&").replaceAll("<", "<").replaceAll(">", ">");
|
|
1238
|
-
}
|
|
1239
1353
|
function terminalStatus(reason) {
|
|
1240
1354
|
if (reason === "completed") return "completed";
|
|
1241
1355
|
if (reason === "aborted") return "cancelled";
|
|
@@ -165,17 +165,17 @@ var LocalSession = class {
|
|
|
165
165
|
utteranceId,
|
|
166
166
|
text: event.text
|
|
167
167
|
});
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
168
|
+
const text = event.text.trim();
|
|
169
|
+
if (text === "") return;
|
|
170
|
+
if (this.interactionMode === "frontend-agent") this.emitTaskCommand(this.activeTaskId === void 0 ? {
|
|
171
|
+
type: "route_transcription",
|
|
172
|
+
input: text
|
|
173
|
+
} : {
|
|
174
|
+
type: "send_task_message",
|
|
175
|
+
taskId: this.activeTaskId,
|
|
176
|
+
message: text
|
|
177
|
+
});
|
|
178
|
+
this.interruptResponse();
|
|
179
179
|
return;
|
|
180
180
|
}
|
|
181
181
|
case "transcription.failed":
|
|
@@ -460,7 +460,6 @@ var NodeSpeechBackend = class {
|
|
|
460
460
|
const utterance = this.active;
|
|
461
461
|
if (utterance === void 0 || utterance.confirmed) return;
|
|
462
462
|
utterance.confirmed = true;
|
|
463
|
-
this.interrupt();
|
|
464
463
|
this.emit?.({
|
|
465
464
|
type: "transcription.started",
|
|
466
465
|
utteranceId: utterance.id
|
|
@@ -557,9 +556,26 @@ const Config = z.object({
|
|
|
557
556
|
maxUtteranceMs: z.natural().min(1e3).default(6e4)
|
|
558
557
|
});
|
|
559
558
|
const PACKAGE_ROOT = dirname(fileURLToPath(import.meta.url));
|
|
560
|
-
|
|
559
|
+
/**
|
|
560
|
+
* Load development configuration from either the source workspace or the
|
|
561
|
+
* bundled single-package layout. DSH-installed profiles still use the
|
|
562
|
+
* process environment, so no package-local `.env` is required in production.
|
|
563
|
+
*/
|
|
564
|
+
function loadProjectEnv() {
|
|
565
|
+
const candidates = [
|
|
566
|
+
resolve(process.cwd(), ".env"),
|
|
567
|
+
resolve(PACKAGE_ROOT, "../../../.env"),
|
|
568
|
+
resolve(PACKAGE_ROOT, "../../../../.env")
|
|
569
|
+
];
|
|
570
|
+
const loaded = /* @__PURE__ */ new Set();
|
|
571
|
+
for (const filename of candidates) {
|
|
572
|
+
if (loaded.has(filename) || !existsSync(filename)) continue;
|
|
573
|
+
loaded.add(filename);
|
|
574
|
+
loadEnvFile(filename);
|
|
575
|
+
}
|
|
576
|
+
}
|
|
561
577
|
function apply(ctx, config = {}) {
|
|
562
|
-
|
|
578
|
+
loadProjectEnv();
|
|
563
579
|
return ctx.voice.registerProvider({
|
|
564
580
|
id: "local",
|
|
565
581
|
available: () => true,
|
package/package.json
CHANGED
|
@@ -1,19 +1,32 @@
|
|
|
1
|
-
{
|
|
1
|
+
{
|
|
2
2
|
"name": "@flowingspring/dsh-voco",
|
|
3
|
-
"description": "
|
|
4
|
-
"version": "0.2.
|
|
5
|
-
"
|
|
6
|
-
"
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
"
|
|
10
|
-
"
|
|
11
|
-
"
|
|
3
|
+
"description": "Persistent voice conversations for DSH with cloud speech recognition, Edge TTS, and background Agent delegation",
|
|
4
|
+
"version": "0.2.2",
|
|
5
|
+
"keywords": [
|
|
6
|
+
"deepseek-harness",
|
|
7
|
+
"dsh",
|
|
8
|
+
"dsh-plugin",
|
|
9
|
+
"voice",
|
|
10
|
+
"speech-recognition",
|
|
11
|
+
"text-to-speech",
|
|
12
|
+
"agent"
|
|
13
|
+
],
|
|
14
|
+
"homepage": "https://github.com/lgquan/dsh-voco#readme",
|
|
15
|
+
"bugs": {
|
|
16
|
+
"url": "https://github.com/lgquan/dsh-voco/issues"
|
|
12
17
|
},
|
|
13
|
-
"
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
"
|
|
18
|
+
"publishConfig": {
|
|
19
|
+
"access": "public"
|
|
20
|
+
},
|
|
21
|
+
"repository": {
|
|
22
|
+
"type": "git",
|
|
23
|
+
"url": "git+https://github.com/lgquan/dsh-voco.git",
|
|
24
|
+
"directory": "packages/voice-app"
|
|
25
|
+
},
|
|
26
|
+
"type": "module",
|
|
27
|
+
"main": "lib/index.js",
|
|
28
|
+
"types": "lib/types/index.d.ts",
|
|
29
|
+
"exports": {
|
|
17
30
|
".": {
|
|
18
31
|
"types": "./lib/types/index.d.ts",
|
|
19
32
|
"default": "./lib/index.js"
|
|
@@ -32,17 +45,17 @@
|
|
|
32
45
|
"./voice-web": "./lib/plugins/voice-web.js",
|
|
33
46
|
"./cordis.patch.yml": "./cordis.patch.yml",
|
|
34
47
|
"./package.json": "./package.json"
|
|
35
|
-
},
|
|
36
|
-
"files": [
|
|
48
|
+
},
|
|
49
|
+
"files": [
|
|
37
50
|
"lib/index.js",
|
|
38
51
|
"lib/invariant.js",
|
|
39
52
|
"lib/client.js",
|
|
40
53
|
"lib/client.js.map",
|
|
41
54
|
"lib/plugins/*.js",
|
|
42
55
|
"cordis.patch.yml",
|
|
43
|
-
"lib/types/**/*.d.ts"
|
|
44
|
-
],
|
|
45
|
-
"license": "MIT",
|
|
56
|
+
"lib/types/**/*.d.ts"
|
|
57
|
+
],
|
|
58
|
+
"license": "MIT",
|
|
46
59
|
"dsh": {
|
|
47
60
|
"bundle": {
|
|
48
61
|
"patch": "./cordis.patch.yml"
|
|
@@ -105,15 +118,15 @@
|
|
|
105
118
|
"@deepseek-ai/dsh-system-prompt": "^0.1.1-rc.1",
|
|
106
119
|
"@deepseek-ai/dsh-tools": "^0.1.1-rc.1",
|
|
107
120
|
"@deepseek-ai/dsh-workspace": "^0.1.1-rc.1",
|
|
121
|
+
"@flowingspring/dsh-client-ui-voice": "workspace:*",
|
|
122
|
+
"@flowingspring/dsh-llm-tool-call-compat": "workspace:*",
|
|
123
|
+
"@flowingspring/dsh-voice": "workspace:*",
|
|
124
|
+
"@flowingspring/dsh-voice-assistant": "workspace:*",
|
|
125
|
+
"@flowingspring/dsh-voice-local": "workspace:*",
|
|
126
|
+
"@flowingspring/dsh-voice-web": "workspace:*",
|
|
108
127
|
"@types/react": "~18.3.1",
|
|
109
128
|
"@types/ws": "^8.18.1",
|
|
110
129
|
"react": "^18.2.0",
|
|
111
|
-
"react-dom": "^18.2.0"
|
|
112
|
-
"@flowingspring/dsh-client-ui-voice": "0.1.2",
|
|
113
|
-
"@flowingspring/dsh-voice-local": "0.1.0",
|
|
114
|
-
"@flowingspring/dsh-llm-tool-call-compat": "0.1.0",
|
|
115
|
-
"@flowingspring/dsh-voice-web": "0.1.2",
|
|
116
|
-
"@flowingspring/dsh-voice": "0.1.2",
|
|
117
|
-
"@flowingspring/dsh-voice-assistant": "0.1.2"
|
|
130
|
+
"react-dom": "^18.2.0"
|
|
118
131
|
}
|
|
119
|
-
}
|
|
132
|
+
}
|