claude-code-hub 1.26.0 → 1.27.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/features/voice/feature.json +42 -0
- package/lib/ext.js +50 -0
- package/lib/keymap.js +1 -0
- package/lib/voice/grammar.js +77 -0
- package/lib/voice/hex-stream.js +155 -0
- package/lib/voice/hex.js +53 -0
- package/lib/voice/index.js +111 -0
- package/lib/voice/title.js +77 -0
- package/package.json +6 -5
- package/packages/claude-hub-sdk/src/keys.js +1 -1
- package/public/app.js +519 -3
- package/public/index.html +89 -0
- package/server.js +67 -0
package/public/app.js
CHANGED
|
@@ -42,6 +42,13 @@ const MAX_EVENT_CHARS = 16 * 1024;
|
|
|
42
42
|
let defaultConfigDir = null;
|
|
43
43
|
// What Ctrl+Alt+W does: 'palette' opens the config-dir palette, 'swap' switches to the previous dir.
|
|
44
44
|
let configDirSwitch = 'palette';
|
|
45
|
+
// config.json voice.enabled; hub.voice has no default key without it.
|
|
46
|
+
let voiceOn = false;
|
|
47
|
+
// 'hex' or 'browser', which the hub picked for voice.provider 'auto'.
|
|
48
|
+
let voiceProvider = null;
|
|
49
|
+
// The grammar's commands, {action, label, param, say, help}, for the voice help panel.
|
|
50
|
+
let voiceCommands = [];
|
|
51
|
+
const voiceInBrowser = () => voiceProvider === 'browser';
|
|
45
52
|
// The app launcher's footer: the hub version from /api/config and its plugin from /api/apps/stats.
|
|
46
53
|
const hubInfo = { version: null, plugin: null };
|
|
47
54
|
// mode: 'project' (Ctrl+Alt+P), 'configDir' (Ctrl+Alt+W) or 'app' (Ctrl+Alt+A). One widget, three row sources.
|
|
@@ -280,6 +287,9 @@ async function init() {
|
|
|
280
287
|
defaultConfigDir = config.defaultConfigDir ?? null;
|
|
281
288
|
activeConfigDir = config.activeConfigDir ?? null;
|
|
282
289
|
configDirSwitch = config.configDirSwitch ?? 'palette';
|
|
290
|
+
voiceOn = !!config.voice;
|
|
291
|
+
voiceProvider = config.voice?.provider ?? null;
|
|
292
|
+
voiceCommands = config.voice?.commands ?? [];
|
|
283
293
|
applyTitle(config.activeConfigDir);
|
|
284
294
|
themeVars = config.themeVars ?? {};
|
|
285
295
|
hubThemes = config.themes ?? [];
|
|
@@ -291,6 +301,8 @@ async function init() {
|
|
|
291
301
|
switchTab(tabs.includes(saved) ? saved : (tabs[0] ?? Object.keys(apps)[0]));
|
|
292
302
|
listenMessages();
|
|
293
303
|
listenKeys();
|
|
304
|
+
if (voiceOn) listenVoiceRelease();
|
|
305
|
+
if (config.voice?.auto && voiceInBrowser()) noteBrowserVoice();
|
|
294
306
|
bindPalette();
|
|
295
307
|
bindSettings();
|
|
296
308
|
registerSW();
|
|
@@ -405,6 +417,505 @@ function showErrorToast(action, err) {
|
|
|
405
417
|
if (!unauthorized) errorToastTimer = setTimeout(hideErrorToast, ERROR_TOAST_MS);
|
|
406
418
|
}
|
|
407
419
|
|
|
420
|
+
// voice.provider 'auto' fell back to the browser, whose speech service is online: say so once.
|
|
421
|
+
const VOICE_NOTE_KEY = 'hub-voice-browser-noted';
|
|
422
|
+
|
|
423
|
+
function noteBrowserVoice() {
|
|
424
|
+
try {
|
|
425
|
+
if (localStorage.getItem(VOICE_NOTE_KEY)) return;
|
|
426
|
+
} catch {}
|
|
427
|
+
const toast = document.getElementById('voice-note');
|
|
428
|
+
document.getElementById('voice-note-ok').addEventListener('click', () => {
|
|
429
|
+
try {
|
|
430
|
+
localStorage.setItem(VOICE_NOTE_KEY, '1');
|
|
431
|
+
} catch {}
|
|
432
|
+
hideToast(toast);
|
|
433
|
+
});
|
|
434
|
+
showToast(toast);
|
|
435
|
+
}
|
|
436
|
+
|
|
437
|
+
// Hold to talk. Frames forward keydown only, so the press moves focus to the pill on the hub page:
|
|
438
|
+
// the release lands there, and focus goes back to the frame after it.
|
|
439
|
+
const voice = {
|
|
440
|
+
state: 'idle',
|
|
441
|
+
send: false,
|
|
442
|
+
app: null,
|
|
443
|
+
drafts: false,
|
|
444
|
+
frame: null,
|
|
445
|
+
started: null,
|
|
446
|
+
live: null,
|
|
447
|
+
cancelling: null,
|
|
448
|
+
dictating: null,
|
|
449
|
+
pillTimer: 0,
|
|
450
|
+
helpTimer: 0,
|
|
451
|
+
};
|
|
452
|
+
|
|
453
|
+
function voicePill(text, listening = false) {
|
|
454
|
+
const pill = document.getElementById('voice-pill');
|
|
455
|
+
clearTimeout(voice.pillTimer);
|
|
456
|
+
pill.textContent = text;
|
|
457
|
+
pill.classList.toggle('listening', listening);
|
|
458
|
+
pill.hidden = false;
|
|
459
|
+
return pill;
|
|
460
|
+
}
|
|
461
|
+
|
|
462
|
+
function hideVoicePill(afterMs = 0) {
|
|
463
|
+
clearTimeout(voice.pillTimer);
|
|
464
|
+
voice.pillTimer = setTimeout(() => {
|
|
465
|
+
document.getElementById('voice-pill').hidden = true;
|
|
466
|
+
}, afterMs);
|
|
467
|
+
}
|
|
468
|
+
|
|
469
|
+
const VOICE_HELP_LINGER_MS = 6000;
|
|
470
|
+
|
|
471
|
+
function helpRow(action, say, hint, what, kind = '') {
|
|
472
|
+
const row = el('div', `voice-help-row ${kind}`.trim());
|
|
473
|
+
row.dataset.action = action;
|
|
474
|
+
const line = el('div', 'say', say);
|
|
475
|
+
if (hint) line.append(' ', el('i', '', hint));
|
|
476
|
+
row.append(line, el('div', 'what', what));
|
|
477
|
+
return row;
|
|
478
|
+
}
|
|
479
|
+
|
|
480
|
+
// Voice runs only what the app on screen handles, so a hold never switches tabs.
|
|
481
|
+
const handledHere = (action) => actions[action]?.app === activeApp;
|
|
482
|
+
const voiceRowsHere = () => voiceCommands.filter((c) => handledHere(c.action));
|
|
483
|
+
const tabName = () => apps[activeApp]?.name ?? 'this tab';
|
|
484
|
+
|
|
485
|
+
// Open for the whole hold: what the user can say on this tab, with the row the words match lit.
|
|
486
|
+
function showVoiceHelp() {
|
|
487
|
+
const panel = document.getElementById('voice-help');
|
|
488
|
+
clearTimeout(voice.helpTimer);
|
|
489
|
+
const rows = voiceRowsHere().map((c) => helpRow(c.action, `Hub, ${c.say}`, `<${c.param}>`, c.help));
|
|
490
|
+
if (voice.drafts)
|
|
491
|
+
rows.push(helpRow('session.paste', 'anything else', '', 'Typed into the session as a draft', 'other'));
|
|
492
|
+
document.getElementById('voice-help-rows').replaceChildren(...rows);
|
|
493
|
+
document.getElementById('voice-help-title').textContent = `What you can say on ${tabName()}`;
|
|
494
|
+
document.getElementById('voice-help-tip').textContent = voice.drafts
|
|
495
|
+
? 'Enter during the hold sends · Backspace cancels'
|
|
496
|
+
: 'Backspace cancels';
|
|
497
|
+
panel.hidden = false;
|
|
498
|
+
}
|
|
499
|
+
|
|
500
|
+
function hideVoiceHelp(afterMs = 0) {
|
|
501
|
+
clearTimeout(voice.helpTimer);
|
|
502
|
+
voice.helpTimer = setTimeout(() => {
|
|
503
|
+
document.getElementById('voice-help').hidden = true;
|
|
504
|
+
}, afterMs);
|
|
505
|
+
}
|
|
506
|
+
|
|
507
|
+
function markVoiceHelp(action) {
|
|
508
|
+
for (const row of document.getElementById('voice-help-rows').children) {
|
|
509
|
+
row.classList.toggle('on', row.dataset.action === action);
|
|
510
|
+
}
|
|
511
|
+
}
|
|
512
|
+
|
|
513
|
+
// Live speech: the page records the mic and the hub relays it to the provider, which answers with
|
|
514
|
+
// the text so far. False after the hub answers 501 or the mic is refused: then the provider records.
|
|
515
|
+
let voiceStream = true;
|
|
516
|
+
const VOICE_RATE = 16000;
|
|
517
|
+
const VOICE_CHUNK = VOICE_RATE / 5;
|
|
518
|
+
const VOICE_TAP = `registerProcessor('voice-tap', class extends AudioWorkletProcessor {
|
|
519
|
+
process(inputs) { const ch = inputs[0] && inputs[0][0]; if (ch) this.port.postMessage(ch.slice(0)); return true; }
|
|
520
|
+
});`;
|
|
521
|
+
|
|
522
|
+
async function openMic(onSamples) {
|
|
523
|
+
const media = await navigator.mediaDevices.getUserMedia({
|
|
524
|
+
audio: { channelCount: 1, echoCancellation: true, noiseSuppression: true },
|
|
525
|
+
});
|
|
526
|
+
const ctx = new AudioContext({ sampleRate: VOICE_RATE });
|
|
527
|
+
try {
|
|
528
|
+
const url = URL.createObjectURL(new Blob([VOICE_TAP], { type: 'text/javascript' }));
|
|
529
|
+
await ctx.audioWorklet.addModule(url);
|
|
530
|
+
URL.revokeObjectURL(url);
|
|
531
|
+
const tap = new AudioWorkletNode(ctx, 'voice-tap');
|
|
532
|
+
tap.port.onmessage = (e) => onSamples(e.data);
|
|
533
|
+
ctx.createMediaStreamSource(media).connect(tap).connect(ctx.destination);
|
|
534
|
+
} catch (err) {
|
|
535
|
+
for (const t of media.getTracks()) t.stop();
|
|
536
|
+
ctx.close();
|
|
537
|
+
throw err;
|
|
538
|
+
}
|
|
539
|
+
return () => {
|
|
540
|
+
for (const t of media.getTracks()) t.stop();
|
|
541
|
+
ctx.close();
|
|
542
|
+
};
|
|
543
|
+
}
|
|
544
|
+
|
|
545
|
+
function startLive() {
|
|
546
|
+
const live = { buf: [], len: 0, queue: Promise.resolve(), close: null, opened: false, sent: '' };
|
|
547
|
+
live.ready = Promise.allSettled([
|
|
548
|
+
sendJson('POST', '/api/voice/stream/start'),
|
|
549
|
+
openMic((samples) => {
|
|
550
|
+
live.buf.push(samples);
|
|
551
|
+
live.len += samples.length;
|
|
552
|
+
if (live.close && live.len >= VOICE_CHUNK) sendAudio(live);
|
|
553
|
+
}),
|
|
554
|
+
]).then(([started, mic]) => {
|
|
555
|
+
if (started.status === 'fulfilled' && mic.status === 'fulfilled') {
|
|
556
|
+
live.close = mic.value;
|
|
557
|
+
return true;
|
|
558
|
+
}
|
|
559
|
+
if (mic.status === 'fulfilled') mic.value();
|
|
560
|
+
if (started.status === 'fulfilled') sendJson('POST', '/api/voice/stream/cancel').catch(() => {});
|
|
561
|
+
if (started.status === 'rejected' && started.reason.status !== 501) throw started.reason;
|
|
562
|
+
voiceStream = false;
|
|
563
|
+
return false;
|
|
564
|
+
});
|
|
565
|
+
live.finish = async () => {
|
|
566
|
+
live.close();
|
|
567
|
+
sendAudio(live);
|
|
568
|
+
await live.queue;
|
|
569
|
+
return sendJson('POST', heard('/api/voice/stream/finish'));
|
|
570
|
+
};
|
|
571
|
+
live.cancel = async () => {
|
|
572
|
+
live.close();
|
|
573
|
+
await live.queue;
|
|
574
|
+
await sendJson('POST', '/api/voice/stream/cancel');
|
|
575
|
+
};
|
|
576
|
+
return live;
|
|
577
|
+
}
|
|
578
|
+
|
|
579
|
+
const SpeechRec = window.SpeechRecognition || window.webkitSpeechRecognition;
|
|
580
|
+
|
|
581
|
+
// voice.provider 'browser': the browser's speech recognition hears the mic, and the hub gets only
|
|
582
|
+
// the text. One text post in flight at a time; a newer result waits for it and replaces older ones.
|
|
583
|
+
function startBrowserLive() {
|
|
584
|
+
if (!SpeechRec) throw new Error('This browser has no speech recognition. Use Chrome or Edge, or install Hex.');
|
|
585
|
+
const rec = new SpeechRec();
|
|
586
|
+
const live = { opened: false, sent: '', text: '', error: null, posting: null, ready: Promise.resolve(true) };
|
|
587
|
+
rec.lang = navigator.language;
|
|
588
|
+
rec.interimResults = true;
|
|
589
|
+
rec.continuous = true;
|
|
590
|
+
live.ended = new Promise((resolve) => (rec.onend = resolve));
|
|
591
|
+
rec.onerror = (e) => (live.error = e.error);
|
|
592
|
+
rec.onresult = (e) => {
|
|
593
|
+
live.text = Array.from(e.results, (r) => r[0].transcript).join('');
|
|
594
|
+
postText(live);
|
|
595
|
+
};
|
|
596
|
+
const settle = async () => {
|
|
597
|
+
while (live.posting) await live.posting;
|
|
598
|
+
};
|
|
599
|
+
live.finish = async () => {
|
|
600
|
+
rec.stop();
|
|
601
|
+
await live.ended;
|
|
602
|
+
await settle();
|
|
603
|
+
if (!live.text && ['not-allowed', 'service-not-allowed', 'network', 'audio-capture'].includes(live.error)) {
|
|
604
|
+
throw new Error(`Speech recognition: ${live.error}`);
|
|
605
|
+
}
|
|
606
|
+
return sendJson('POST', heard('/api/voice/text'), { text: live.text, final: true });
|
|
607
|
+
};
|
|
608
|
+
live.cancel = async () => {
|
|
609
|
+
rec.abort();
|
|
610
|
+
await settle();
|
|
611
|
+
};
|
|
612
|
+
rec.start();
|
|
613
|
+
return live;
|
|
614
|
+
}
|
|
615
|
+
|
|
616
|
+
function postText(live) {
|
|
617
|
+
if (live.posting) return;
|
|
618
|
+
const text = live.text;
|
|
619
|
+
live.posting = sendJson('POST', heard('/api/voice/text'), { text })
|
|
620
|
+
.then((r) => showLive(live, r))
|
|
621
|
+
.catch(() => {})
|
|
622
|
+
.finally(() => {
|
|
623
|
+
live.posting = null;
|
|
624
|
+
if (live.text !== text) postText(live);
|
|
625
|
+
});
|
|
626
|
+
}
|
|
627
|
+
|
|
628
|
+
// Posts run one at a time, so the provider gets the audio in order.
|
|
629
|
+
function sendAudio(live) {
|
|
630
|
+
if (!live.len) return;
|
|
631
|
+
const pcm = new Float32Array(live.len);
|
|
632
|
+
let at = 0;
|
|
633
|
+
for (const b of live.buf) {
|
|
634
|
+
pcm.set(b, at);
|
|
635
|
+
at += b.length;
|
|
636
|
+
}
|
|
637
|
+
live.buf = [];
|
|
638
|
+
live.len = 0;
|
|
639
|
+
live.queue = live.queue
|
|
640
|
+
.then(() =>
|
|
641
|
+
fetch(heard('/api/voice/stream/audio'), {
|
|
642
|
+
method: 'POST',
|
|
643
|
+
headers: { 'Content-Type': 'application/octet-stream' },
|
|
644
|
+
body: pcm.buffer,
|
|
645
|
+
}),
|
|
646
|
+
)
|
|
647
|
+
.then((res) => res.json())
|
|
648
|
+
.then((r) => r.ok && showLive(live, r))
|
|
649
|
+
.catch(() => {});
|
|
650
|
+
}
|
|
651
|
+
|
|
652
|
+
function invokeVoice(action, params) {
|
|
653
|
+
onInvoke({ id: 'voice', action, params }, (r) => {
|
|
654
|
+
if (!r.ok) showErrorToast('Voice', new Error(`${action} failed: ${r.reason}`));
|
|
655
|
+
});
|
|
656
|
+
}
|
|
657
|
+
|
|
658
|
+
const canLive = () => Object.hasOwn(actions, 'session.live');
|
|
659
|
+
// The server matches only the actions of the tab the hold started on.
|
|
660
|
+
const heard = (path) => `${path}?app=${encodeURIComponent(voice.app)}`;
|
|
661
|
+
const isDraft = (command) => command?.action === 'session.paste';
|
|
662
|
+
|
|
663
|
+
// Words typed into the terminal before a phrase was heard come back out of its input box.
|
|
664
|
+
function clearTyped(live) {
|
|
665
|
+
if (live.sent) invokeVoice('session.live', { text: '', final: 'true', cancel: 'true' });
|
|
666
|
+
live.sent = '';
|
|
667
|
+
}
|
|
668
|
+
|
|
669
|
+
// "new session" opens the dialog at once; the words after it, or any other words, show in Kanban as they come.
|
|
670
|
+
function openOnce(live, newSession) {
|
|
671
|
+
if (!newSession || live.opened) return;
|
|
672
|
+
live.opened = true;
|
|
673
|
+
clearTyped(live);
|
|
674
|
+
invokeVoice('session.new', {});
|
|
675
|
+
}
|
|
676
|
+
|
|
677
|
+
// A command row with `live` names a dialog its app fills as the user speaks: the dialog opens on the
|
|
678
|
+
// phrase and takes the command's params live, and the last call carries the final params.
|
|
679
|
+
const dialogFor = (action) => {
|
|
680
|
+
const dialog = voiceCommands.find((c) => c.action === action)?.live?.action;
|
|
681
|
+
return dialog && Object.hasOwn(actions, dialog) ? dialog : null;
|
|
682
|
+
};
|
|
683
|
+
|
|
684
|
+
function fillDialog(live, dialog, params = {}) {
|
|
685
|
+
const sent = JSON.stringify(params);
|
|
686
|
+
if (live.dialogSent === sent) return;
|
|
687
|
+
live.dialog = dialog;
|
|
688
|
+
live.dialogSent = sent;
|
|
689
|
+
invokeVoice(dialog, params);
|
|
690
|
+
}
|
|
691
|
+
|
|
692
|
+
function showLive(live, { text, newSession, command, action, params }) {
|
|
693
|
+
if (!canLive() || voice.live !== live || voice.state !== 'listening') return;
|
|
694
|
+
openOnce(live, newSession);
|
|
695
|
+
live.helpOnly = command === 'Hub' && !text;
|
|
696
|
+
const marked = newSession ? 'session.new' : command ? action : text && 'session.paste';
|
|
697
|
+
if (marked !== live.marked) markVoiceHelp(marked);
|
|
698
|
+
live.marked = marked;
|
|
699
|
+
if (command) {
|
|
700
|
+
clearTyped(live);
|
|
701
|
+
const dialog = dialogFor(action);
|
|
702
|
+
if (dialog) fillDialog(live, dialog, params);
|
|
703
|
+
const pill = text && !dialog ? `${command}: ${text}` : command;
|
|
704
|
+
if (pill !== live.pill) voicePill(pill, true);
|
|
705
|
+
live.pill = pill;
|
|
706
|
+
return;
|
|
707
|
+
}
|
|
708
|
+
if (!live.opened && !voice.drafts) return;
|
|
709
|
+
if (text === live.sent) return;
|
|
710
|
+
live.sent = text;
|
|
711
|
+
invokeVoice('session.live', { text });
|
|
712
|
+
}
|
|
713
|
+
|
|
714
|
+
async function finishLive(live) {
|
|
715
|
+
// The final params can wait on a model (a task title), so the dialog shows it is busy.
|
|
716
|
+
if (live.dialog) invokeVoice(live.dialog, { pending: 'true' });
|
|
717
|
+
const r = await live.finish();
|
|
718
|
+
if (!canLive()) return runVoiceCommand(r);
|
|
719
|
+
if (live.dialog) return finishDialog(live, r);
|
|
720
|
+
openOnce(live, r.newSession);
|
|
721
|
+
if (!live.opened && !isDraft(r.command)) {
|
|
722
|
+
clearTyped(live);
|
|
723
|
+
return runVoiceCommand(r);
|
|
724
|
+
}
|
|
725
|
+
hideVoiceHelp();
|
|
726
|
+
invokeVoice('session.live', { text: r.text, final: 'true', submit: String(voice.send && !live.opened) });
|
|
727
|
+
if (live.opened) flashPill('New session');
|
|
728
|
+
else hideVoicePill();
|
|
729
|
+
}
|
|
730
|
+
|
|
731
|
+
// The dialog stays open with the final params for the user to review; Enter during the hold saves it.
|
|
732
|
+
function finishDialog(live, r) {
|
|
733
|
+
const c = r.command;
|
|
734
|
+
if (c?.kind === 'invoke' && dialogFor(c.action) === live.dialog) {
|
|
735
|
+
hideVoiceHelp();
|
|
736
|
+
invokeVoice(live.dialog, { ...c.params, final: 'true', submit: String(voice.send) });
|
|
737
|
+
return flashPill(c.label);
|
|
738
|
+
}
|
|
739
|
+
invokeVoice(live.dialog, { final: 'true', cancel: 'true' });
|
|
740
|
+
return runVoiceCommand(r);
|
|
741
|
+
}
|
|
742
|
+
|
|
743
|
+
function flashPill(text) {
|
|
744
|
+
voicePill(text);
|
|
745
|
+
hideVoicePill(1500);
|
|
746
|
+
}
|
|
747
|
+
|
|
748
|
+
function voiceDown() {
|
|
749
|
+
if (voice.state !== 'idle') return;
|
|
750
|
+
voice.drafts = handledHere('session.paste');
|
|
751
|
+
if (!voice.drafts && !voiceRowsHere().length) return flashPill(`No voice commands on ${tabName()}`);
|
|
752
|
+
voice.state = 'listening';
|
|
753
|
+
voice.send = false;
|
|
754
|
+
voice.app = activeApp;
|
|
755
|
+
showVoiceHelp();
|
|
756
|
+
const active = document.activeElement;
|
|
757
|
+
voice.frame = active?.tagName === 'IFRAME' ? active : null;
|
|
758
|
+
voicePill('Listening', true).focus();
|
|
759
|
+
if (voiceInBrowser()) {
|
|
760
|
+
voice.started = null;
|
|
761
|
+
try {
|
|
762
|
+
voice.live = startBrowserLive();
|
|
763
|
+
} catch (err) {
|
|
764
|
+
voice.started = err;
|
|
765
|
+
}
|
|
766
|
+
return;
|
|
767
|
+
}
|
|
768
|
+
voice.live = voiceStream ? startLive() : null;
|
|
769
|
+
const cli = () => {
|
|
770
|
+
voice.started = sendJson('POST', '/api/voice/start').catch((err) => err);
|
|
771
|
+
};
|
|
772
|
+
if (voice.live)
|
|
773
|
+
voice.live.ready.then(
|
|
774
|
+
(ok) => ok || cli(),
|
|
775
|
+
() => {},
|
|
776
|
+
);
|
|
777
|
+
else cli();
|
|
778
|
+
}
|
|
779
|
+
|
|
780
|
+
async function voiceUp() {
|
|
781
|
+
if (voice.state !== 'listening') return;
|
|
782
|
+
voice.state = 'working';
|
|
783
|
+
voice.frame?.focus();
|
|
784
|
+
voice.frame = null;
|
|
785
|
+
voicePill('…');
|
|
786
|
+
const live = voice.live;
|
|
787
|
+
try {
|
|
788
|
+
if (live && (await live.ready)) return await finishLive(live);
|
|
789
|
+
const started = await voice.started;
|
|
790
|
+
if (started instanceof Error) throw started;
|
|
791
|
+
runVoiceCommand(await sendJson('POST', heard('/api/voice/stop')));
|
|
792
|
+
} catch (err) {
|
|
793
|
+
hideVoicePill();
|
|
794
|
+
hideVoiceHelp();
|
|
795
|
+
showErrorToast('Voice', err);
|
|
796
|
+
// Ends Kanban's live text, keeping the words, so the next hold does not erase from a stale start.
|
|
797
|
+
if (live?.sent && canLive()) invokeVoice('session.live', { text: live.sent, final: 'true', submit: 'false' });
|
|
798
|
+
if (live?.dialog) invokeVoice(live.dialog, { final: 'true' });
|
|
799
|
+
} finally {
|
|
800
|
+
voice.live = null;
|
|
801
|
+
voice.state = 'idle';
|
|
802
|
+
}
|
|
803
|
+
}
|
|
804
|
+
|
|
805
|
+
function runVoiceCommand({ text, command = {} }) {
|
|
806
|
+
hideVoiceHelp();
|
|
807
|
+
if (command.kind === 'none') return flashPill('No speech');
|
|
808
|
+
if (command.kind === 'invoke') {
|
|
809
|
+
const draft = isDraft(command);
|
|
810
|
+
invokeVoice(command.action, draft ? { ...command.params, submit: String(voice.send) } : command.params);
|
|
811
|
+
// A paste shows in the session, or in Kanban's copy toast, so the pill does not repeat the text.
|
|
812
|
+
return command.label ? flashPill(command.label) : hideVoicePill();
|
|
813
|
+
}
|
|
814
|
+
hideVoicePill();
|
|
815
|
+
// "Hub" alone asks what to say: the panel stays a while in place of the toast.
|
|
816
|
+
if (voice.live?.helpOnly) return hideVoiceHelp(VOICE_HELP_LINGER_MS);
|
|
817
|
+
showErrorToast("Didn't get that", new Error(`“${text}”`));
|
|
818
|
+
}
|
|
819
|
+
|
|
820
|
+
// `voice.dictate`, the one action the hub handles itself: an app's mic gets the provider's text with no command.
|
|
821
|
+
const DICTATE = 'voice.dictate';
|
|
822
|
+
// With the browser provider an app's mic runs its own browser speech, so the hub does not offer it.
|
|
823
|
+
const hubDictates = () => voiceOn && !voiceInBrowser();
|
|
824
|
+
|
|
825
|
+
async function dictate(op, reply) {
|
|
826
|
+
const busy = op === 'start' ? voice.state !== 'idle' : voice.state !== 'dictating';
|
|
827
|
+
if (busy || !['start', 'stop', 'cancel'].includes(op)) return reply({ ok: false, reason: 'busy' });
|
|
828
|
+
try {
|
|
829
|
+
if (op === 'start') {
|
|
830
|
+
voice.state = 'dictating';
|
|
831
|
+
voicePill('Dictating', true);
|
|
832
|
+
voice.dictating = sendJson('POST', '/api/voice/start');
|
|
833
|
+
await voice.dictating;
|
|
834
|
+
return reply({ ok: true });
|
|
835
|
+
}
|
|
836
|
+
voice.state = 'idle';
|
|
837
|
+
hideVoicePill();
|
|
838
|
+
await voice.dictating?.catch(() => {});
|
|
839
|
+
const r = await sendJson('POST', `/api/voice/${op === 'stop' ? 'dictate' : 'cancel'}`);
|
|
840
|
+
reply({ ok: true, text: r.text ?? '' });
|
|
841
|
+
} catch (err) {
|
|
842
|
+
voice.state = 'idle';
|
|
843
|
+
hideVoicePill();
|
|
844
|
+
reply({ ok: false, reason: err.message });
|
|
845
|
+
}
|
|
846
|
+
}
|
|
847
|
+
|
|
848
|
+
// Backspace mid-hold drops the recording and what it typed. The state stays 'cancelled' until the
|
|
849
|
+
// release, so the held key's auto-repeat does not start a new hold.
|
|
850
|
+
async function voiceCancel() {
|
|
851
|
+
voice.state = 'cancelled';
|
|
852
|
+
const live = voice.live;
|
|
853
|
+
voice.live = null;
|
|
854
|
+
flashPill('Cancelled');
|
|
855
|
+
hideVoiceHelp();
|
|
856
|
+
try {
|
|
857
|
+
if (live && (await live.ready)) {
|
|
858
|
+
await live.cancel();
|
|
859
|
+
if (canLive() && (live.sent || live.opened)) {
|
|
860
|
+
invokeVoice('session.live', { text: '', final: 'true', cancel: live.opened ? 'close' : 'true' });
|
|
861
|
+
}
|
|
862
|
+
if (live.dialog) invokeVoice(live.dialog, { final: 'true', cancel: 'true' });
|
|
863
|
+
} else {
|
|
864
|
+
await voice.started;
|
|
865
|
+
await sendJson('POST', '/api/voice/cancel');
|
|
866
|
+
}
|
|
867
|
+
} catch {}
|
|
868
|
+
}
|
|
869
|
+
|
|
870
|
+
// The hold ends when any key of the hub.voice combo comes up, the user's combo included.
|
|
871
|
+
function releasesVoice(e) {
|
|
872
|
+
const combo = Object.values(bindings()).find((b) => b.id === 'hub.voice')?.combo ?? 'ctrl+shift+Space';
|
|
873
|
+
const parts = combo.split('+');
|
|
874
|
+
const mod = { Control: 'ctrl', Alt: 'alt', Shift: 'shift', Meta: 'meta' }[e.key];
|
|
875
|
+
return mod ? parts.includes(mod) : comboOf({ key: e.key, code: e.code }) === parts.at(-1);
|
|
876
|
+
}
|
|
877
|
+
|
|
878
|
+
function listenVoiceRelease() {
|
|
879
|
+
document.addEventListener('keydown', (e) => {
|
|
880
|
+
if (voice.state !== 'listening') return;
|
|
881
|
+
if (e.key === 'Enter') {
|
|
882
|
+
e.preventDefault();
|
|
883
|
+
voice.send = true;
|
|
884
|
+
voicePill('Listening · sends on release', true);
|
|
885
|
+
return;
|
|
886
|
+
}
|
|
887
|
+
if (e.key !== 'Backspace') return;
|
|
888
|
+
e.preventDefault();
|
|
889
|
+
voice.cancelling = voiceCancel();
|
|
890
|
+
});
|
|
891
|
+
document.addEventListener('keyup', (e) => {
|
|
892
|
+
if (!releasesVoice(e)) return;
|
|
893
|
+
if (voice.state === 'cancelled') endCancelledHold();
|
|
894
|
+
else voiceUp();
|
|
895
|
+
});
|
|
896
|
+
window.addEventListener('blur', () => {
|
|
897
|
+
// A frame that takes focus mid-hold (Kanban's New session dialog) would get the release.
|
|
898
|
+
const held = voice.state === 'listening' || voice.state === 'cancelled';
|
|
899
|
+
if (held && document.activeElement?.tagName === 'IFRAME') {
|
|
900
|
+
setTimeout(() => document.getElementById('voice-pill').focus());
|
|
901
|
+
return;
|
|
902
|
+
}
|
|
903
|
+
if (voice.state === 'cancelled') return endCancelledHold();
|
|
904
|
+
voiceUp();
|
|
905
|
+
});
|
|
906
|
+
}
|
|
907
|
+
|
|
908
|
+
// A new hold waits for the cancel to finish, or its late stream/cancel would end the new stream.
|
|
909
|
+
// Idle before the frame takes focus, so the blur handler does not treat the hold as still held.
|
|
910
|
+
async function endCancelledHold() {
|
|
911
|
+
const frame = voice.frame;
|
|
912
|
+
voice.frame = null;
|
|
913
|
+
await voice.cancelling;
|
|
914
|
+
if (voice.state !== 'cancelled') return;
|
|
915
|
+
voice.state = 'idle';
|
|
916
|
+
frame?.focus();
|
|
917
|
+
}
|
|
918
|
+
|
|
408
919
|
function bindErrorToast() {
|
|
409
920
|
document.getElementById('error-close').addEventListener('click', hideErrorToast);
|
|
410
921
|
document.getElementById('error-reload').addEventListener('click', () => location.reload());
|
|
@@ -558,7 +1069,7 @@ function welcomeFor(data, appId) {
|
|
|
558
1069
|
forward: Object.keys(bindings()),
|
|
559
1070
|
keys: actionKeys(),
|
|
560
1071
|
themes: hubThemes,
|
|
561
|
-
actions: Object.keys(actions),
|
|
1072
|
+
actions: [...Object.keys(actions), ...(hubDictates() ? [DICTATE] : [])],
|
|
562
1073
|
origins: Object.keys(apps).map(originOf),
|
|
563
1074
|
settings: apps[appId]?.settings ?? {},
|
|
564
1075
|
};
|
|
@@ -621,6 +1132,7 @@ function fillUrl(template, params) {
|
|
|
621
1132
|
function onInvoke({ id, action, params = {}, open }, send) {
|
|
622
1133
|
if (typeof id !== 'string') return;
|
|
623
1134
|
const reply = (result) => send({ type: 'hub:result', id, ...result });
|
|
1135
|
+
if (action === DICTATE && hubDictates()) return dictate(params.op, reply);
|
|
624
1136
|
const spec = typeof action === 'string' && Object.hasOwn(actions, action) ? actions[action] : null;
|
|
625
1137
|
if (!spec) return reply({ ok: false, reason: 'unhandled' });
|
|
626
1138
|
if (badParams(spec.params, params)) return reply({ ok: false, reason: 'bad-params' });
|
|
@@ -628,7 +1140,10 @@ function onInvoke({ id, action, params = {}, open }, send) {
|
|
|
628
1140
|
const url = template && fillUrl(template, params);
|
|
629
1141
|
const paneSrc = open === 'pane' && url && srcOnOrigin(handler, apps[handler].url + url);
|
|
630
1142
|
if (paneSrc) return reply({ ok: true, handledBy: handler, url: paneSrc });
|
|
631
|
-
switchTab
|
|
1143
|
+
// Not again for the app on screen: switchTab focuses the frame and tells the app it is active,
|
|
1144
|
+
// which repaints Kanban's terminal, and voice sends several actions a second.
|
|
1145
|
+
if (handler !== activeApp) switchTab(handler);
|
|
1146
|
+
else if (id !== 'voice') iframes[handler]?.focus();
|
|
632
1147
|
if (mode === 'message' && liveApps.has(handler)) {
|
|
633
1148
|
postTo(handler, { type: 'hub:action', id, action, params });
|
|
634
1149
|
} else if (url && !guardedApps.has(handler)) {
|
|
@@ -768,6 +1283,7 @@ function hubActions() {
|
|
|
768
1283
|
run: (i) => switchTab(tabIds()[i]),
|
|
769
1284
|
count: 9,
|
|
770
1285
|
},
|
|
1286
|
+
{ id: 'hub.voice', combo: 'ctrl+shift+Space', off: !voiceOn, run: () => voiceDown() },
|
|
771
1287
|
];
|
|
772
1288
|
}
|
|
773
1289
|
|
|
@@ -781,7 +1297,7 @@ function bindings() {
|
|
|
781
1297
|
const own = (b) => Object.hasOwn(userKeys, b.id);
|
|
782
1298
|
const expand = (action) => {
|
|
783
1299
|
const combo = comboFor(action);
|
|
784
|
-
if (!combo) return [];
|
|
1300
|
+
if (!combo || action.off) return [];
|
|
785
1301
|
if (!action.count) return [[combo, { ...action, combo }]];
|
|
786
1302
|
return Array.from({ length: Math.min(action.count, tabs) }, (_, i) => {
|
|
787
1303
|
const c = combo.replace('{n}', i + 1);
|
package/public/index.html
CHANGED
|
@@ -524,6 +524,78 @@
|
|
|
524
524
|
}
|
|
525
525
|
|
|
526
526
|
.update-toast[hidden] { display: none; }
|
|
527
|
+
.voice-pill {
|
|
528
|
+
position: fixed;
|
|
529
|
+
left: 50%;
|
|
530
|
+
bottom: 16px;
|
|
531
|
+
z-index: 30;
|
|
532
|
+
transform: translateX(-50%);
|
|
533
|
+
padding: 6px 14px;
|
|
534
|
+
background: var(--surface);
|
|
535
|
+
border: 1px solid var(--border);
|
|
536
|
+
border-radius: 999px;
|
|
537
|
+
box-shadow: 0 6px 20px var(--shadow);
|
|
538
|
+
color: var(--text);
|
|
539
|
+
font: 13px ui-monospace, SFMono-Regular, Menlo, monospace;
|
|
540
|
+
outline: none;
|
|
541
|
+
}
|
|
542
|
+
.voice-pill[hidden] { display: none; }
|
|
543
|
+
.voice-pill.listening::before { content: '● '; color: #e5484d; }
|
|
544
|
+
.voice-help {
|
|
545
|
+
position: fixed;
|
|
546
|
+
right: 16px;
|
|
547
|
+
bottom: 16px;
|
|
548
|
+
z-index: 30;
|
|
549
|
+
width: min(360px, calc(100vw - 32px));
|
|
550
|
+
overflow: hidden;
|
|
551
|
+
background: var(--surface);
|
|
552
|
+
border: 1px solid var(--border);
|
|
553
|
+
border-radius: 12px;
|
|
554
|
+
box-shadow: 0 12px 36px var(--shadow);
|
|
555
|
+
color: var(--text);
|
|
556
|
+
font: 13px/1.45 ui-monospace, SFMono-Regular, Menlo, monospace;
|
|
557
|
+
animation: voice-help-in 0.16s ease-out;
|
|
558
|
+
}
|
|
559
|
+
@keyframes voice-help-in { from { opacity: 0; transform: translateX(12px); } }
|
|
560
|
+
.voice-help[hidden] { display: none; }
|
|
561
|
+
.voice-help-title {
|
|
562
|
+
display: flex;
|
|
563
|
+
align-items: center;
|
|
564
|
+
gap: 8px;
|
|
565
|
+
padding: 10px 14px;
|
|
566
|
+
border-bottom: 1px solid var(--border);
|
|
567
|
+
font-weight: 600;
|
|
568
|
+
}
|
|
569
|
+
.voice-help-title::before {
|
|
570
|
+
content: '';
|
|
571
|
+
width: 8px;
|
|
572
|
+
height: 8px;
|
|
573
|
+
border-radius: 50%;
|
|
574
|
+
background: #e5484d;
|
|
575
|
+
animation: voice-help-dot 1.4s ease-in-out infinite;
|
|
576
|
+
}
|
|
577
|
+
@keyframes voice-help-dot { 50% { opacity: 0.35; } }
|
|
578
|
+
.voice-help-rows { display: grid; padding: 4px 0; }
|
|
579
|
+
.voice-help-row {
|
|
580
|
+
display: grid;
|
|
581
|
+
padding: 8px 14px;
|
|
582
|
+
transition: background-color 0.15s, box-shadow 0.15s;
|
|
583
|
+
}
|
|
584
|
+
.voice-help-row + .voice-help-row { border-top: 1px dashed var(--border); }
|
|
585
|
+
.voice-help-row.on {
|
|
586
|
+
background: color-mix(in srgb, var(--accent) 12%, transparent);
|
|
587
|
+
box-shadow: inset 3px 0 0 var(--accent);
|
|
588
|
+
}
|
|
589
|
+
.voice-help-row > *, .voice-help-tip { white-space: nowrap; overflow: hidden; text-overflow: ellipsis; }
|
|
590
|
+
.voice-help-row .say { color: var(--accent); }
|
|
591
|
+
.voice-help-row.other .say { color: var(--text); }
|
|
592
|
+
.voice-help-row .say i { color: var(--text-dim); font-style: normal; }
|
|
593
|
+
.voice-help-row .what { color: var(--text-dim); font-size: 12px; }
|
|
594
|
+
.voice-help-tip { padding: 8px 14px; background: var(--bg); border-top: 1px solid var(--border); color: var(--text-dim); font-size: 12px; }
|
|
595
|
+
@media (prefers-reduced-motion: reduce) {
|
|
596
|
+
.voice-help, .voice-help-title::before { animation: none; }
|
|
597
|
+
.voice-help-row { transition: none; }
|
|
598
|
+
}
|
|
527
599
|
.update-toast.leaving { opacity: 0; transform: translateY(8px); }
|
|
528
600
|
.update-toast-body { flex: 1; min-width: 0; }
|
|
529
601
|
.update-toast-title { font-weight: 600; font-size: 15px; }
|
|
@@ -649,6 +721,23 @@
|
|
|
649
721
|
<button class="update-toast-close" id="error-close" type="button" aria-label="Close">×</button>
|
|
650
722
|
</div>
|
|
651
723
|
|
|
724
|
+
<div class="update-toast" id="voice-note" role="status" hidden>
|
|
725
|
+
<div class="update-toast-body">
|
|
726
|
+
<div class="update-toast-title">Voice uses the browser's speech recognition</div>
|
|
727
|
+
<div class="update-toast-meta">Hex is not installed, so the browser turns your speech into text. It sends the audio to an online service while you hold the keys. Install <a href="https://anoma.ly/hex" target="_blank" rel="noopener">Hex</a> to keep it on your machine.</div>
|
|
728
|
+
<div class="update-toast-actions">
|
|
729
|
+
<button class="update-toast-skip" id="voice-note-ok" type="button">Got it</button>
|
|
730
|
+
</div>
|
|
731
|
+
</div>
|
|
732
|
+
</div>
|
|
733
|
+
|
|
734
|
+
<div class="voice-pill" id="voice-pill" tabindex="-1" role="status" hidden></div>
|
|
735
|
+
<div class="voice-help" id="voice-help" role="status" hidden>
|
|
736
|
+
<div class="voice-help-title" id="voice-help-title">What you can say</div>
|
|
737
|
+
<div class="voice-help-rows" id="voice-help-rows"></div>
|
|
738
|
+
<div class="voice-help-tip" id="voice-help-tip"></div>
|
|
739
|
+
</div>
|
|
740
|
+
|
|
652
741
|
<script src="/sdk/keys.js"></script>
|
|
653
742
|
<script src="/app.js"></script>
|
|
654
743
|
</body>
|