opencharm 0.2.0 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/main.mjs CHANGED
@@ -1,5 +1,6 @@
1
1
  #!/usr/bin/env node
2
- import { chmodSync, existsSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, realpathSync, renameSync, rmSync, statSync, unlinkSync, writeFileSync } from "node:fs";
2
+ import { createRequire } from "node:module";
3
+ import { chmodSync, createWriteStream, existsSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, realpathSync, renameSync, rmSync, statSync, unlinkSync, writeFileSync } from "node:fs";
3
4
  import { homedir, tmpdir } from "node:os";
4
5
  import { dirname, extname, isAbsolute, join, normalize, resolve, sep } from "node:path";
5
6
  import { z } from "zod";
@@ -8,8 +9,9 @@ import { createInterface } from "node:readline";
8
9
  import { execFile, execFileSync, spawn, spawnSync } from "node:child_process";
9
10
  import { createHash, randomBytes, randomInt, randomUUID, scrypt, timingSafeEqual } from "node:crypto";
10
11
  import { createServer as createServer$1 } from "node:http";
11
- import { WebSocketServer } from "ws";
12
+ import WebSocket, { WebSocketServer } from "ws";
12
13
  import { Readable, Writable } from "node:stream";
14
+ import { pipeline } from "node:stream/promises";
13
15
  import * as acp from "@agentclientprotocol/sdk";
14
16
  import OpusScript from "opusscript";
15
17
  import { fileURLToPath } from "node:url";
@@ -710,7 +712,7 @@ const secret = {
710
712
  apiKey: z.string().min(1).optional(),
711
713
  apiKeyEnv: z.string().min(1).optional()
712
714
  };
713
- const voiceSchema = z.discriminatedUnion("provider", [
715
+ const legacyVoiceSchema = z.discriminatedUnion("provider", [
714
716
  z.object({
715
717
  provider: z.literal("fake"),
716
718
  transcript: z.string().optional()
@@ -729,6 +731,52 @@ const voiceSchema = z.discriminatedUnion("provider", [
729
731
  voice: z.string().optional()
730
732
  }).strict()
731
733
  ]);
734
+ const languageCode = z.string().regex(/^[a-z]{2}$/);
735
+ const listenSchema = z.discriminatedUnion("provider", [
736
+ z.object({ provider: z.literal("local") }).strict(),
737
+ z.object({
738
+ provider: z.literal("whisper"),
739
+ model: z.string().optional(),
740
+ command: z.string().optional()
741
+ }).strict(),
742
+ z.object({
743
+ provider: z.literal("openai"),
744
+ ...secret,
745
+ baseUrl: z.url().optional(),
746
+ model: z.string().optional()
747
+ }).strict(),
748
+ z.object({
749
+ provider: z.literal("fake"),
750
+ transcript: z.string().optional()
751
+ }).strict()
752
+ ]);
753
+ const speakSchema = z.discriminatedUnion("provider", [
754
+ z.object({
755
+ provider: z.literal("microsoft"),
756
+ voices: z.record(languageCode, z.string().regex(/^[A-Za-z]{2,3}-[A-Za-z]{2,4}-\w+$/)).optional()
757
+ }).strict(),
758
+ z.object({
759
+ provider: z.literal("local"),
760
+ speaker: z.number().int().min(0).max(9).optional()
761
+ }).strict(),
762
+ z.object({
763
+ provider: z.literal("system"),
764
+ voice: z.string().optional()
765
+ }).strict(),
766
+ z.object({
767
+ provider: z.literal("openai"),
768
+ ...secret,
769
+ baseUrl: z.url().optional(),
770
+ model: z.string().optional(),
771
+ voice: z.string().optional()
772
+ }).strict(),
773
+ z.object({ provider: z.literal("fake") }).strict()
774
+ ]);
775
+ const voiceSchema = z.union([legacyVoiceSchema, z.object({
776
+ listen: listenSchema,
777
+ speak: speakSchema,
778
+ language: languageCode.default("en")
779
+ }).strict()]);
732
780
  const agentSchema = z.discriminatedUnion("adapter", [
733
781
  z.object({ adapter: z.literal("fake") }).strict(),
734
782
  z.object({
@@ -809,6 +857,52 @@ function resolveAgent(agent, base) {
809
857
  ...agent.projects ? { projects: agent.projects.map((p) => expandPath(p, base)) } : {}
810
858
  };
811
859
  }
860
+ function normalizeVoice(voice) {
861
+ if ("listen" in voice) return voice;
862
+ switch (voice.provider) {
863
+ case "fake": return {
864
+ listen: {
865
+ provider: "fake",
866
+ ...voice.transcript ? { transcript: voice.transcript } : {}
867
+ },
868
+ speak: { provider: "fake" },
869
+ language: "en"
870
+ };
871
+ case "local": return {
872
+ listen: voice.whisperModel ? {
873
+ provider: "whisper",
874
+ model: voice.whisperModel
875
+ } : { provider: "local" },
876
+ speak: voice.sayVoice ? {
877
+ provider: "system",
878
+ voice: voice.sayVoice
879
+ } : { provider: "local" },
880
+ language: "en"
881
+ };
882
+ case "openai": {
883
+ const { sttModel, ttsModel, voice: name } = voice;
884
+ const rest = {
885
+ ...voice.apiKey ? { apiKey: voice.apiKey } : {},
886
+ ...voice.apiKeyEnv ? { apiKeyEnv: voice.apiKeyEnv } : {},
887
+ ...voice.baseUrl ? { baseUrl: voice.baseUrl } : {}
888
+ };
889
+ return {
890
+ listen: {
891
+ provider: "openai",
892
+ ...rest,
893
+ ...sttModel ? { model: sttModel } : {}
894
+ },
895
+ speak: {
896
+ provider: "openai",
897
+ ...rest,
898
+ ...ttsModel ? { model: ttsModel } : {},
899
+ ...name ? { voice: name } : {}
900
+ },
901
+ language: "en"
902
+ };
903
+ }
904
+ }
905
+ }
812
906
  function parseConfig(input, baseDir) {
813
907
  const result = schema.safeParse(input);
814
908
  if (!result.success) {
@@ -824,7 +918,8 @@ function parseConfig(input, baseDir) {
824
918
  ...config,
825
919
  statePath,
826
920
  adminSocket: config.adminSocket ?? defaultAdminSocket(statePath),
827
- agent
921
+ agent,
922
+ voice: normalizeVoice(config.voice)
828
923
  };
829
924
  }
830
925
  function loadConfig(path, options = {}) {
@@ -866,7 +961,7 @@ function sendAdmin(socketPath, request) {
866
961
  }
867
962
  //#endregion
868
963
  //#region package.json
869
- var version = "0.2.0";
964
+ var version = "0.4.0";
870
965
  function channelToCube(value) {
871
966
  if (value < 48) return 0;
872
967
  if (value < 115) return 1;
@@ -1010,6 +1105,7 @@ function promptHidden(question) {
1010
1105
  const statusSchema = z.object({
1011
1106
  charmd: z.string().regex(/^[\w.@+() -]{1,96}$/).optional().catch(void 0),
1012
1107
  starter: z.string().regex(/^[0-9a-f]{4,40}$/).optional().catch(void 0),
1108
+ voice: z.string().regex(/^[\w.+()%· -]{1,160}$/).optional().catch(void 0),
1013
1109
  charms: z.array(z.object({
1014
1110
  name: z.string(),
1015
1111
  state: z.string(),
@@ -1102,8 +1198,9 @@ async function look(ctx, args, deps) {
1102
1198
  if (Object.keys(update).length === 0) return;
1103
1199
  ctx.out.write(`${result.connected === 0 ? "No charm is unlocked right now: it gets the look when it unlocks." : `Sent to ${result.connected} charm${result.connected === 1 ? "" : "s"}.`} It lasts until charmd restarts; to keep it, set "charm" in opencharm.json.\n`);
1104
1200
  }
1105
- function printStatus(ctx, charms, charmd, starter) {
1201
+ function printStatus(ctx, charms, charmd, starter, voice) {
1106
1202
  if (charmd) ctx.out.write(`charmd ${charmd}\n`);
1203
+ if (voice) ctx.out.write(`voice ${voice}\n`);
1107
1204
  if (starter) ctx.out.write(`workspace from starter ${starter}\n`);
1108
1205
  if (charms.length === 0) {
1109
1206
  ctx.out.write("No charms paired yet. Start one and run: opencharm pair <code>\n");
@@ -1134,8 +1231,8 @@ async function runAdminCommand(ctx, command, args, deps) {
1134
1231
  fail$2(ctx, "Unexpected answer from charmd: is it the same version as this CLI?");
1135
1232
  return;
1136
1233
  }
1137
- const { charms, charmd, starter } = answer.data;
1138
- printStatus(ctx, charms, charmd, starter);
1234
+ const { charms, charmd, starter, voice } = answer.data;
1235
+ printStatus(ctx, charms, charmd, starter, voice);
1139
1236
  return;
1140
1237
  }
1141
1238
  const charm = firstPositional(args);
@@ -1349,6 +1446,7 @@ function runHelp(ctx) {
1349
1446
  opencharm setup --domain <d> install charmd as a service (Linux, sudo) [--dry-run]
1350
1447
  opencharm serve start charmd [--config <file>]
1351
1448
  opencharm sim the charm on screen, in your browser [--url <ws>] [--port <n>]
1449
+ opencharm voice the local voice's models [install: download them now]
1352
1450
  opencharm pair <code> pair the charm showing <code> [--name <name>]
1353
1451
  opencharm status paired charms and their state
1354
1452
  opencharm lock <charm> lock a charm now
@@ -1373,6 +1471,10 @@ function runHelp(ctx) {
1373
1471
  //#endregion
1374
1472
  //#region src/commands/init.ts
1375
1473
  const STARTER = "https://github.com/opencharm-labs/opencharm-starter.git";
1474
+ const DEFAULT_VOICE = {
1475
+ listen: { provider: "local" },
1476
+ speak: { provider: "microsoft" }
1477
+ };
1376
1478
  function checkLook(options) {
1377
1479
  const result = z.object({
1378
1480
  name: lookFields.name,
@@ -1409,13 +1511,9 @@ function initWorkspace(target, options) {
1409
1511
  const path = join(target, "opencharm.json");
1410
1512
  if (!existsSync(path)) throw new Error(`${from} has no opencharm.json: is it an OpenCharm starter? (${target} was cloned; delete it)`);
1411
1513
  const config = JSON.parse(readFileSync(path, "utf8"));
1412
- const voice = options.platform === "darwin" ? "local" : "fake";
1413
1514
  const hasLook = Object.keys(look).length > 0;
1414
- if (config.voice?.provider === voice && config.agent?.agent === agent && !hasLook) return;
1415
- config.voice = {
1416
- ...config.voice,
1417
- provider: voice
1418
- };
1515
+ if (JSON.stringify(config.voice) === JSON.stringify(DEFAULT_VOICE) && config.agent?.agent === agent && !hasLook) return;
1516
+ config.voice = DEFAULT_VOICE;
1419
1517
  config.agent = {
1420
1518
  ...config.agent,
1421
1519
  agent
@@ -1434,8 +1532,7 @@ function runInit(ctx, args) {
1434
1532
  from: flagValue(args, "--from"),
1435
1533
  agent: flagValue(args, "--agent"),
1436
1534
  name,
1437
- colour: flagValue(args, "--colour") ?? flagValue(args, "--color"),
1438
- platform: process.platform
1535
+ colour: flagValue(args, "--colour") ?? flagValue(args, "--color")
1439
1536
  });
1440
1537
  } catch (error) {
1441
1538
  ctx.err.write(`${error instanceof Error ? error.message : String(error)}\n`);
@@ -1451,7 +1548,8 @@ function runInit(ctx, args) {
1451
1548
  opencharm sim no charm yet? it appears in your browser (another terminal)
1452
1549
  opencharm pair <code> the code on the charm (or the emulator); you choose its PIN
1453
1550
 
1454
- The charm is called ${charm}: its name, colour and greeting are in opencharm.json ("charm"). README.md has the voice setup.
1551
+ The charm is called ${charm}: its name, colour and greeting are in opencharm.json ("charm").
1552
+ ${ctx.style.dim("Voice: it listens on this computer; it speaks with Microsoft's free voices, so the text of each spoken reply goes to Microsoft. To keep everything here, set \"speak\": { \"provider\": \"local\" } in opencharm.json. The first start downloads the voice models (about 620 MB): opencharm voice.")}
1455
1553
  `);
1456
1554
  }
1457
1555
  //#endregion
@@ -1731,7 +1829,7 @@ function parseInput(schema, input) {
1731
1829
  if (!result.success) throw new Error(result.error.issues.map((i) => i.message).join("; "));
1732
1830
  return result.data;
1733
1831
  }
1734
- function createAdminActions({ store, pairing, sessions, look, agentCanChangeLook, identity, starter }) {
1832
+ function createAdminActions({ store, pairing, sessions, look, agentCanChangeLook, identity, starter, voice }) {
1735
1833
  function sessionsOf(charmId) {
1736
1834
  return [...sessions()].filter((s) => s.charmId === charmId && s.state !== "closed");
1737
1835
  }
@@ -1840,6 +1938,7 @@ function createAdminActions({ store, pairing, sessions, look, agentCanChangeLook
1840
1938
  return {
1841
1939
  charmd: identity ?? "unknown",
1842
1940
  ...starter ? { starter } : {},
1941
+ ...voice ? { voice: voice() } : {},
1843
1942
  charms: store.list().map((charm) => {
1844
1943
  const live = sessionsOf(charm.id).at(-1);
1845
1944
  return {
@@ -2185,6 +2284,7 @@ var Session = class {
2185
2284
  type: "charm",
2186
2285
  op: "unlocked"
2187
2286
  });
2287
+ this.#activeTurn()?.warm();
2188
2288
  }
2189
2289
  #issuePairCode() {
2190
2290
  try {
@@ -2657,6 +2757,449 @@ function readOggOpus(data) {
2657
2757
  };
2658
2758
  }
2659
2759
  //#endregion
2760
+ //#region ../charmd/src/voice/language.ts
2761
+ const COMMON_WORDS = {
2762
+ en: [
2763
+ "the",
2764
+ "and",
2765
+ "is",
2766
+ "are",
2767
+ "you",
2768
+ "to",
2769
+ "of",
2770
+ "it",
2771
+ "that",
2772
+ "what",
2773
+ "this",
2774
+ "for",
2775
+ "with",
2776
+ "have",
2777
+ "i",
2778
+ "my",
2779
+ "your",
2780
+ "do",
2781
+ "can",
2782
+ "on",
2783
+ "in",
2784
+ "be",
2785
+ "was",
2786
+ "will",
2787
+ "not",
2788
+ "me",
2789
+ "please",
2790
+ "tomorrow",
2791
+ "today"
2792
+ ],
2793
+ it: [
2794
+ "il",
2795
+ "lo",
2796
+ "la",
2797
+ "gli",
2798
+ "le",
2799
+ "di",
2800
+ "che",
2801
+ "è",
2802
+ "e",
2803
+ "un",
2804
+ "una",
2805
+ "per",
2806
+ "con",
2807
+ "non",
2808
+ "sono",
2809
+ "mi",
2810
+ "ti",
2811
+ "ho",
2812
+ "hai",
2813
+ "cosa",
2814
+ "come",
2815
+ "del",
2816
+ "della",
2817
+ "dei",
2818
+ "delle",
2819
+ "nel",
2820
+ "nella",
2821
+ "fai",
2822
+ "oggi",
2823
+ "domani",
2824
+ "anche",
2825
+ "questo",
2826
+ "quello",
2827
+ "sì",
2828
+ "grazie"
2829
+ ],
2830
+ es: [
2831
+ "el",
2832
+ "los",
2833
+ "las",
2834
+ "de",
2835
+ "que",
2836
+ "es",
2837
+ "y",
2838
+ "un",
2839
+ "una",
2840
+ "por",
2841
+ "con",
2842
+ "no",
2843
+ "para",
2844
+ "lo",
2845
+ "mi",
2846
+ "tu",
2847
+ "qué",
2848
+ "cómo",
2849
+ "está",
2850
+ "son",
2851
+ "hoy",
2852
+ "mañana",
2853
+ "gracias",
2854
+ "pero",
2855
+ "muy",
2856
+ "también"
2857
+ ],
2858
+ fr: [
2859
+ "le",
2860
+ "les",
2861
+ "des",
2862
+ "de",
2863
+ "et",
2864
+ "est",
2865
+ "un",
2866
+ "une",
2867
+ "pour",
2868
+ "avec",
2869
+ "pas",
2870
+ "je",
2871
+ "tu",
2872
+ "vous",
2873
+ "nous",
2874
+ "que",
2875
+ "qui",
2876
+ "ce",
2877
+ "cette",
2878
+ "dans",
2879
+ "sur",
2880
+ "aujourd'hui",
2881
+ "demain",
2882
+ "merci",
2883
+ "oui",
2884
+ "mais",
2885
+ "très"
2886
+ ],
2887
+ de: [
2888
+ "der",
2889
+ "die",
2890
+ "das",
2891
+ "und",
2892
+ "ist",
2893
+ "ein",
2894
+ "eine",
2895
+ "nicht",
2896
+ "ich",
2897
+ "du",
2898
+ "sie",
2899
+ "wir",
2900
+ "mit",
2901
+ "für",
2902
+ "auf",
2903
+ "zu",
2904
+ "den",
2905
+ "dem",
2906
+ "was",
2907
+ "wie",
2908
+ "heute",
2909
+ "morgen",
2910
+ "danke",
2911
+ "ja",
2912
+ "auch",
2913
+ "bitte"
2914
+ ],
2915
+ pt: [
2916
+ "o",
2917
+ "os",
2918
+ "as",
2919
+ "de",
2920
+ "que",
2921
+ "é",
2922
+ "e",
2923
+ "um",
2924
+ "uma",
2925
+ "para",
2926
+ "com",
2927
+ "não",
2928
+ "eu",
2929
+ "você",
2930
+ "do",
2931
+ "da",
2932
+ "dos",
2933
+ "das",
2934
+ "no",
2935
+ "na",
2936
+ "hoje",
2937
+ "amanhã",
2938
+ "obrigado",
2939
+ "obrigada",
2940
+ "sim",
2941
+ "também",
2942
+ "muito"
2943
+ ]
2944
+ };
2945
+ const LANGUAGES = Object.keys(COMMON_WORDS);
2946
+ const WORDS = new Map(LANGUAGES.map((language) => [language, new Set(COMMON_WORDS[language])]));
2947
+ function guessLanguage(text, among = LANGUAGES) {
2948
+ const words = text.toLowerCase().match(/[\p{L}]+(?:'(?:hui))?/gu) ?? [];
2949
+ const [best, second] = among.map((language) => ({
2950
+ language,
2951
+ score: words.filter((word) => WORDS.get(language)?.has(word)).length
2952
+ })).sort((a, b) => b.score - a.score);
2953
+ if (!best || best.score < 2) return void 0;
2954
+ return best.score > (second?.score ?? 0) ? best.language : void 0;
2955
+ }
2956
+ //#endregion
2957
+ //#region ../charmd/src/voice/models.ts
2958
+ const RELEASES = "https://github.com/k2-fsa/sherpa-onnx/releases/download";
2959
+ const MODELS = {
2960
+ parakeet: {
2961
+ label: "Parakeet (listening)",
2962
+ url: `${RELEASES}/asr-models/sherpa-onnx-nemo-parakeet-tdt-0.6b-v3-int8.tar.bz2`,
2963
+ sha256: "5793d0fd397c5778d2cf2126994d58e9d56b1be7c04d13c7a15bb1b4eafb16bf",
2964
+ bytes: 487170055,
2965
+ folder: "sherpa-onnx-nemo-parakeet-tdt-0.6b-v3-int8",
2966
+ files: [
2967
+ "encoder.int8.onnx",
2968
+ "decoder.int8.onnx",
2969
+ "joiner.int8.onnx",
2970
+ "tokens.txt"
2971
+ ]
2972
+ },
2973
+ supertonic: {
2974
+ label: "Supertonic (speaking)",
2975
+ url: `${RELEASES}/tts-models/sherpa-onnx-supertonic-3-tts-int8-2026-05-11.tar.bz2`,
2976
+ sha256: "82fa96f91c4ef8abaae3a14a3f4153facf88bed821d1f7331cec2700f432c427",
2977
+ bytes: 128774318,
2978
+ folder: "sherpa-onnx-supertonic-3-tts-int8-2026-05-11",
2979
+ files: [
2980
+ "duration_predictor.int8.onnx",
2981
+ "text_encoder.int8.onnx",
2982
+ "vector_estimator.int8.onnx",
2983
+ "vocoder.int8.onnx",
2984
+ "tts.json",
2985
+ "unicode_indexer.bin",
2986
+ "voice.bin"
2987
+ ]
2988
+ }
2989
+ };
2990
+ var VoiceNotReady = class extends Error {};
2991
+ const ABANDONED_MS = 36e5;
2992
+ const states = /* @__PURE__ */ new Map();
2993
+ const installing = /* @__PURE__ */ new Map();
2994
+ function defaultModelsDir() {
2995
+ return join(homedir(), ".opencharm", "models");
2996
+ }
2997
+ function isInstalled(spec, dir) {
2998
+ return spec.files.every((file) => existsSync(join(dir, spec.folder, file)));
2999
+ }
3000
+ function untar(archive, into) {
3001
+ return new Promise((resolve, reject) => {
3002
+ execFile("tar", [
3003
+ "-xjf",
3004
+ archive,
3005
+ "-C",
3006
+ into
3007
+ ], (error, _out, stderr) => {
3008
+ if (!error) resolve();
3009
+ else reject(/* @__PURE__ */ new Error(`Couldn't unpack the model: ${stderr.trim().slice(0, 200) || error.message}`));
3010
+ });
3011
+ });
3012
+ }
3013
+ async function download(spec, archive, options) {
3014
+ const response = await fetch(spec.url, { signal: options.signal ?? null });
3015
+ if (!response.ok || !response.body) throw new Error(`Couldn't download ${spec.label}: ${response.status} ${response.statusText}`);
3016
+ const total = Number(response.headers.get("content-length")) || spec.bytes;
3017
+ const hash = createHash("sha256");
3018
+ let received = 0;
3019
+ const body = Readable.fromWeb(response.body);
3020
+ body.on("data", (chunk) => {
3021
+ hash.update(chunk);
3022
+ received += chunk.length;
3023
+ if (spec.bytes && received > spec.bytes) body.destroy(/* @__PURE__ */ new Error(`${spec.label} was bigger than expected, so it was discarded`));
3024
+ else options.onProgress?.(received, total);
3025
+ });
3026
+ await pipeline(body, createWriteStream(archive, { mode: 384 }), { ...options.signal ? { signal: options.signal } : {} });
3027
+ if (spec.bytes && received !== spec.bytes || hash.digest("hex") !== spec.sha256) throw new Error(`${spec.label} didn't match its pinned checksum, so it was discarded`);
3028
+ }
3029
+ function removeAbandoned(dir, folder) {
3030
+ for (const name of readdirSync(dir)) {
3031
+ if (!name.startsWith(`.download-${folder}-`)) continue;
3032
+ const path = join(dir, name);
3033
+ try {
3034
+ if (Date.now() - statSync(path).mtimeMs > ABANDONED_MS) rmSync(path, {
3035
+ recursive: true,
3036
+ force: true
3037
+ });
3038
+ } catch {}
3039
+ }
3040
+ }
3041
+ async function install(spec, dir, options) {
3042
+ mkdirSync(dir, { recursive: true });
3043
+ removeAbandoned(dir, spec.folder);
3044
+ const temp = join(dir, `.download-${spec.folder}-${randomBytes(6).toString("hex")}`);
3045
+ mkdirSync(temp);
3046
+ try {
3047
+ const archive = join(temp, "model.tar.bz2");
3048
+ await download(spec, archive, options);
3049
+ const unpacked = join(temp, "unpacked");
3050
+ mkdirSync(unpacked);
3051
+ await untar(archive, unpacked);
3052
+ if (!isInstalled(spec, unpacked)) throw new Error(`${spec.label} is missing files`);
3053
+ rmSync(join(dir, spec.folder), {
3054
+ recursive: true,
3055
+ force: true
3056
+ });
3057
+ renameSync(join(unpacked, spec.folder), join(dir, spec.folder));
3058
+ return join(dir, spec.folder);
3059
+ } finally {
3060
+ rmSync(temp, {
3061
+ recursive: true,
3062
+ force: true
3063
+ });
3064
+ }
3065
+ }
3066
+ function modelState(id, dir = defaultModelsDir()) {
3067
+ if (isInstalled(MODELS[id], dir)) return { state: "ready" };
3068
+ return states.get(join(dir, id)) ?? { state: "missing" };
3069
+ }
3070
+ function installModel(id, options = {}) {
3071
+ const spec = options.spec ?? MODELS[id];
3072
+ const dir = options.dir ?? defaultModelsDir();
3073
+ if (isInstalled(spec, dir)) return Promise.resolve(join(dir, spec.folder));
3074
+ const key = join(dir, id);
3075
+ const running = installing.get(key);
3076
+ if (running) return running;
3077
+ states.set(key, {
3078
+ state: "downloading",
3079
+ received: 0,
3080
+ total: spec.bytes
3081
+ });
3082
+ const work = install(spec, dir, {
3083
+ ...options,
3084
+ onProgress: (received, total) => {
3085
+ states.set(key, {
3086
+ state: "downloading",
3087
+ received,
3088
+ total
3089
+ });
3090
+ options.onProgress?.(received, total);
3091
+ }
3092
+ }).then((path) => {
3093
+ states.set(key, { state: "ready" });
3094
+ return path;
3095
+ }).catch((error) => {
3096
+ states.set(key, {
3097
+ state: "failed",
3098
+ error: error instanceof Error ? error.message : String(error),
3099
+ at: Date.now()
3100
+ });
3101
+ throw error;
3102
+ }).finally(() => installing.delete(key));
3103
+ installing.set(key, work);
3104
+ return work;
3105
+ }
3106
+ //#endregion
3107
+ //#region ../charmd/src/voice/packets.ts
3108
+ function asError(problem) {
3109
+ if (problem instanceof Error) return problem;
3110
+ return new Error(typeof problem === "string" ? problem : "The voice failed");
3111
+ }
3112
+ var PacketStream = class {
3113
+ #packets = [];
3114
+ #done = false;
3115
+ #error;
3116
+ #waiters = [];
3117
+ constructor(source) {
3118
+ this.#pump(source);
3119
+ }
3120
+ async #pump(source) {
3121
+ try {
3122
+ for await (const packet of source) {
3123
+ this.#packets.push(packet);
3124
+ this.#wake();
3125
+ }
3126
+ } catch (error) {
3127
+ this.#error = asError(error);
3128
+ } finally {
3129
+ this.#done = true;
3130
+ this.#wake();
3131
+ }
3132
+ }
3133
+ #wake() {
3134
+ for (const waiter of this.#waiters.splice(0)) waiter();
3135
+ }
3136
+ #next() {
3137
+ return new Promise((resolve) => this.#waiters.push(resolve));
3138
+ }
3139
+ async first() {
3140
+ while (this.#packets.length === 0) {
3141
+ if (this.#error !== void 0) throw this.#error;
3142
+ if (this.#done) throw new Error("The voice produced no audio");
3143
+ await this.#next();
3144
+ }
3145
+ }
3146
+ async *[Symbol.asyncIterator]() {
3147
+ let index = 0;
3148
+ for (;;) {
3149
+ if (index < this.#packets.length) {
3150
+ yield this.#packets[index++];
3151
+ continue;
3152
+ }
3153
+ if (this.#error !== void 0) throw this.#error;
3154
+ if (this.#done) return;
3155
+ await this.#next();
3156
+ }
3157
+ }
3158
+ };
3159
+ function packetsOf(speaker, text, signal, language) {
3160
+ if (speaker.stream) return speaker.stream(text, signal, language);
3161
+ return (async function* () {
3162
+ yield* readOggOpus(await speaker.synthesize(text, signal, language)).packets;
3163
+ })();
3164
+ }
3165
+ function createChannel() {
3166
+ const items = [];
3167
+ let done = false;
3168
+ let error;
3169
+ let waiter;
3170
+ const wake = () => {
3171
+ waiter?.();
3172
+ waiter = void 0;
3173
+ };
3174
+ return {
3175
+ push(item) {
3176
+ items.push(item);
3177
+ wake();
3178
+ },
3179
+ end() {
3180
+ done = true;
3181
+ wake();
3182
+ },
3183
+ fail(problem) {
3184
+ error = asError(problem);
3185
+ done = true;
3186
+ wake();
3187
+ },
3188
+ async *[Symbol.asyncIterator]() {
3189
+ for (;;) {
3190
+ const item = items.shift();
3191
+ if (item !== void 0) {
3192
+ yield item;
3193
+ continue;
3194
+ }
3195
+ if (error !== void 0) throw error;
3196
+ if (done) return;
3197
+ await new Promise((resolve) => waiter = resolve);
3198
+ }
3199
+ }
3200
+ };
3201
+ }
3202
+ //#endregion
2660
3203
  //#region ../charmd/src/turn/sentences.ts
2661
3204
  const MIN_CHARS = 10;
2662
3205
  const ABBREVIATIONS = /* @__PURE__ */ new Set([
@@ -2671,9 +3214,12 @@ const ABBREVIATIONS = /* @__PURE__ */ new Set([
2671
3214
  "no."
2672
3215
  ]);
2673
3216
  const BOUNDARY = /[.!?…]+(?=\s)|\n+/g;
3217
+ const FIRST_BOUNDARY = /[.!?…]+(?=\s)|[,;:](?=\s)|\n+/g;
3218
+ const FIRST_CLAUSE_CHARS = 20;
2674
3219
  var SentenceSplitter = class {
2675
3220
  #buffer = "";
2676
3221
  #pending = "";
3222
+ #started = false;
2677
3223
  #take(piece) {
2678
3224
  const text = piece.replace(/\s+/g, " ").trim();
2679
3225
  if (!text) return [];
@@ -2681,14 +3227,16 @@ var SentenceSplitter = class {
2681
3227
  if (this.#pending.length < MIN_CHARS) return [];
2682
3228
  const out = this.#pending;
2683
3229
  this.#pending = "";
3230
+ this.#started = true;
2684
3231
  return [out];
2685
3232
  }
2686
3233
  push(chunk) {
2687
3234
  this.#buffer += chunk;
2688
3235
  const out = [];
2689
3236
  let start = 0;
2690
- for (const match of this.#buffer.matchAll(BOUNDARY)) {
3237
+ for (const match of this.#buffer.matchAll(this.#started ? BOUNDARY : FIRST_BOUNDARY)) {
2691
3238
  const end = match.index + match[0].length;
3239
+ if (/^[,;:]/.test(match[0]) && (this.#started || this.#pending.length + end - start < FIRST_CLAUSE_CHARS)) continue;
2692
3240
  const lastWord = this.#buffer.slice(start, end).trim().split(/\s+/).at(-1)?.toLowerCase() ?? "";
2693
3241
  if (match[0].startsWith(".") && ABBREVIATIONS.has(lastWord)) continue;
2694
3242
  out.push(...this.#take(this.#buffer.slice(start, end)));
@@ -2711,12 +3259,13 @@ function cleanForSpeech(text) {
2711
3259
  //#endregion
2712
3260
  //#region ../charmd/src/turn/turn.ts
2713
3261
  const MAX_FRAMES = 1e3;
3262
+ const LOOKAHEAD = 2;
2714
3263
  const AGENT_LABELS = { ...Object.fromEntries(Object.entries(ACP_AGENTS).map(([name, agent]) => [name, agent.label])) };
2715
3264
  var TurnTimeout = class extends Error {};
2716
3265
  var StageError = class extends Error {
2717
3266
  stage;
2718
3267
  constructor(stage, cause) {
2719
- super(cause instanceof Error ? cause.message : String(cause));
3268
+ super(cause instanceof Error ? cause.message : String(cause), { cause });
2720
3269
  this.stage = stage;
2721
3270
  }
2722
3271
  };
@@ -2778,6 +3327,10 @@ var TurnController = class {
2778
3327
  });
2779
3328
  this.#speaking = false;
2780
3329
  }
3330
+ warm() {
3331
+ this.#deps.agent.warm?.(this.#deps.sessionKey);
3332
+ this.#deps.voice.warm?.().catch(() => void 0);
3333
+ }
2781
3334
  listenStart() {
2782
3335
  if (this.#state === "thinking" || this.#state === "speaking") this.#cancel();
2783
3336
  this.#state = "listening";
@@ -2819,6 +3372,21 @@ var TurnController = class {
2819
3372
  let outcome = "done";
2820
3373
  const startClock = () => setTimeout(() => controller.abort(new TurnTimeout()), this.#deps.timeoutMs);
2821
3374
  let timer = startClock();
3375
+ let speakingStarted = false;
3376
+ let asking = false;
3377
+ const restartClock = () => {
3378
+ clearTimeout(timer);
3379
+ timer = startClock();
3380
+ };
3381
+ const clock = {
3382
+ started: () => {
3383
+ speakingStarted = true;
3384
+ restartClock();
3385
+ },
3386
+ touch: () => {
3387
+ if (speakingStarted && !asking) restartClock();
3388
+ }
3389
+ };
2822
3390
  const timeoutGuard = new Promise((_, reject) => signal.addEventListener("abort", () => {
2823
3391
  if (signal.reason instanceof TurnTimeout) reject(signal.reason);
2824
3392
  }));
@@ -2843,6 +3411,7 @@ var TurnController = class {
2843
3411
  const ask = this.#deps.ask;
2844
3412
  const label = AGENT_LABELS[this.#deps.agent.name] ?? "Your agent";
2845
3413
  if (ask) this.#asking = async (text, options) => {
3414
+ asking = true;
2846
3415
  clearTimeout(timer);
2847
3416
  try {
2848
3417
  return await ask(text, {
@@ -2850,6 +3419,7 @@ var TurnController = class {
2850
3419
  signal
2851
3420
  });
2852
3421
  } finally {
3422
+ asking = false;
2853
3423
  if (!signal.aborted) timer = startClock();
2854
3424
  }
2855
3425
  };
@@ -2859,7 +3429,7 @@ var TurnController = class {
2859
3429
  signal,
2860
3430
  ...ask ? { ask: (action) => this.ask(`${label} wants to ${action}.`) } : {}
2861
3431
  });
2862
- said = await this.#speak(reply, signal, live, timings, started, timeoutGuard);
3432
+ said = await this.#speak(reply, signal, live, timings, started, timeoutGuard, guessLanguage(heard) ?? this.#deps.voice.language ?? "en", clock);
2863
3433
  if (!live()) return;
2864
3434
  if (this.#speaking) this.#deps.send({
2865
3435
  type: "tts",
@@ -2871,6 +3441,7 @@ var TurnController = class {
2871
3441
  this.#endFace = void 0;
2872
3442
  this.#face(end.state, end.text);
2873
3443
  } catch (error) {
3444
+ if (!signal.aborted) controller.abort(error);
2874
3445
  outcome = error instanceof TurnTimeout ? "timeout" : error instanceof StageError ? `${error.stage}-error` : "error";
2875
3446
  if (!live()) return;
2876
3447
  if (this.#speaking) this.#deps.send({
@@ -2900,9 +3471,18 @@ var TurnController = class {
2900
3471
  });
2901
3472
  }
2902
3473
  }
2903
- async #speak(source, signal, live, timings, started, timeoutGuard) {
3474
+ async #speak(source, signal, live, timings, started, timeoutGuard, language, clock) {
2904
3475
  const now = this.#deps.now ?? Date.now;
3476
+ const going = () => live() && !signal.aborted;
3477
+ let speaking = language;
2905
3478
  const queue = [];
3479
+ let playing = 0;
3480
+ const startUpTo = (end) => {
3481
+ for (let i = playing; i < Math.min(end, queue.length); i++) {
3482
+ const item = queue[i];
3483
+ item.packets ??= new PacketStream(packetsOf(this.#deps.voice, item.text, signal, item.language));
3484
+ }
3485
+ };
2906
3486
  let wake = () => void 0;
2907
3487
  let sourceDone = false;
2908
3488
  let said = "";
@@ -2910,19 +3490,23 @@ var TurnController = class {
2910
3490
  const text = cleanForSpeech(sentence);
2911
3491
  if (!text) return;
2912
3492
  said += `${text} `;
2913
- const pending = inStage("tts", () => this.#deps.voice.synthesize(text, signal)).then((audio) => ({
3493
+ speaking = guessLanguage(text) ?? speaking;
3494
+ const sentenceLanguage = speaking;
3495
+ timings.firstSentenceMs ??= now() - started;
3496
+ queue.push({
2914
3497
  text,
2915
- ogg: audio
2916
- }));
2917
- pending.catch(() => void 0);
2918
- queue.push(pending);
3498
+ language: sentenceLanguage
3499
+ });
3500
+ startUpTo(playing + LOOKAHEAD);
2919
3501
  wake();
2920
3502
  };
2921
3503
  const produce = (async () => {
2922
3504
  const splitter = new SentenceSplitter();
2923
3505
  await inStage("agent", async () => {
2924
3506
  for await (const chunk of source) {
2925
- if (!live()) return;
3507
+ if (!going()) return;
3508
+ timings.agentFirstMs ??= now() - started;
3509
+ clock.touch();
2926
3510
  for (const sentence of splitter.push(chunk)) enqueue(sentence);
2927
3511
  }
2928
3512
  });
@@ -2933,19 +3517,23 @@ var TurnController = class {
2933
3517
  });
2934
3518
  produce.catch(() => void 0);
2935
3519
  const play = async () => {
2936
- let index = 0;
2937
3520
  for (;;) {
2938
- if (index >= queue.length) {
3521
+ if (!going()) return;
3522
+ if (playing >= queue.length) {
2939
3523
  if (sourceDone) return;
2940
3524
  await new Promise((resolve) => wake = resolve);
2941
3525
  continue;
2942
3526
  }
2943
- const { text, ogg } = await Promise.race([queue[index++], timeoutGuard]);
2944
- if (!live()) return;
3527
+ startUpTo(playing + LOOKAHEAD);
3528
+ const { text, packets } = queue[playing];
3529
+ if (!packets) throw new Error("A sentence wasn't started");
3530
+ await Promise.race([inStage("tts", () => packets.first()), timeoutGuard]);
3531
+ if (!going()) return;
2945
3532
  if (!this.#speaking) {
2946
3533
  this.#speaking = true;
2947
3534
  this.#state = "speaking";
2948
3535
  timings.firstAudioMs = now() - started;
3536
+ clock.started();
2949
3537
  this.#deps.send({
2950
3538
  type: "tts",
2951
3539
  state: "start"
@@ -2956,7 +3544,8 @@ var TurnController = class {
2956
3544
  state: "sentence_start",
2957
3545
  text
2958
3546
  });
2959
- await this.#sendPaced(readOggOpus(ogg).packets, live);
3547
+ await Promise.race([inStage("tts", () => this.#sendPaced(packets, going, clock.touch)), timeoutGuard]);
3548
+ playing += 1;
2960
3549
  }
2961
3550
  };
2962
3551
  await Promise.all([Promise.race([produce, timeoutGuard]), play()]);
@@ -2968,7 +3557,16 @@ var TurnController = class {
2968
3557
  const live = () => this.#generation === generation;
2969
3558
  const controller = new AbortController();
2970
3559
  this.#controller = controller;
2971
- const timer = setTimeout(() => controller.abort(new TurnTimeout()), this.#deps.timeoutMs);
3560
+ const startClock = () => setTimeout(() => controller.abort(new TurnTimeout()), this.#deps.timeoutMs);
3561
+ let timer = startClock();
3562
+ const restartClock = () => {
3563
+ clearTimeout(timer);
3564
+ timer = startClock();
3565
+ };
3566
+ const clock = {
3567
+ started: restartClock,
3568
+ touch: restartClock
3569
+ };
2972
3570
  const timeoutGuard = new Promise((_, reject) => controller.signal.addEventListener("abort", () => {
2973
3571
  if (controller.signal.reason instanceof TurnTimeout) reject(controller.signal.reason);
2974
3572
  }));
@@ -2977,7 +3575,7 @@ var TurnController = class {
2977
3575
  yield await Promise.resolve(text);
2978
3576
  }
2979
3577
  try {
2980
- await this.#speak(once(), controller.signal, live, {}, Date.now(), timeoutGuard);
3578
+ await this.#speak(once(), controller.signal, live, {}, Date.now(), timeoutGuard, guessLanguage(text) ?? this.#deps.voice.language ?? "en", clock);
2981
3579
  if (!live()) return;
2982
3580
  if (this.#speaking) this.#deps.send({
2983
3581
  type: "tts",
@@ -2987,6 +3585,7 @@ var TurnController = class {
2987
3585
  this.#state = "idle";
2988
3586
  this.#face("idle");
2989
3587
  } catch (error) {
3588
+ if (!controller.signal.aborted) controller.abort(error);
2990
3589
  if (!live()) return;
2991
3590
  if (this.#speaking) this.#deps.send({
2992
3591
  type: "tts",
@@ -3001,16 +3600,19 @@ var TurnController = class {
3001
3600
  }
3002
3601
  #failureLine(error) {
3003
3602
  if (error instanceof TurnTimeout) return "That took too long.";
3603
+ if (error instanceof StageError && error.cause instanceof VoiceNotReady) return error.cause.message;
3004
3604
  if (error instanceof StageError && error.stage === "stt") return "I couldn't hear that.";
3005
3605
  if (error instanceof StageError && error.stage === "tts") return "I couldn't speak that.";
3006
3606
  return `Can't reach ${AGENT_LABELS[this.#deps.agent.name] ?? "your agent"}`;
3007
3607
  }
3008
- async #sendPaced(packets, live) {
3608
+ async #sendPaced(packets, live, onPacket) {
3009
3609
  const sleep = this.#deps.sleep ?? sleepMs;
3010
- for (const [index, packet] of packets.entries()) {
3610
+ let index = 0;
3611
+ for await (const packet of packets) {
3011
3612
  if (!live()) return;
3012
3613
  this.#deps.sendAudio(packet);
3013
- if (index >= 2) await sleep(opusPacketSamples48k(packet) / 48);
3614
+ onPacket();
3615
+ if (index++ >= 2) await sleep(opusPacketSamples48k(packet) / 48);
3014
3616
  }
3015
3617
  }
3016
3618
  dispose() {
@@ -3520,6 +4122,205 @@ function createLocalVoice(options) {
3520
4122
  };
3521
4123
  }
3522
4124
  //#endregion
4125
+ //#region ../charmd/src/audio/webm-opus.ts
4126
+ const SEGMENT = 408125543;
4127
+ const CLUSTER = 524531317;
4128
+ const BLOCK_GROUP = 160;
4129
+ const BLOCK = 161;
4130
+ const SIMPLE_BLOCK = 163;
4131
+ const CONTAINERS = /* @__PURE__ */ new Set([
4132
+ SEGMENT,
4133
+ CLUSTER,
4134
+ BLOCK_GROUP
4135
+ ]);
4136
+ var CutShort = class extends Error {};
4137
+ function readVint(data, offset, keepMarker) {
4138
+ const first = data[offset];
4139
+ if (first === void 0) throw new CutShort();
4140
+ if (first === 0) throw new Error("Not a WebM file: bad element header");
4141
+ const length = Math.clz32(first) - 23;
4142
+ if (offset + length > data.length) throw new CutShort();
4143
+ let value = keepMarker ? first : first & 255 >> length;
4144
+ let allOnes = value === 255 >> length;
4145
+ for (let i = 1; i < length; i++) {
4146
+ const byte = data[offset + i];
4147
+ value = value * 256 + byte;
4148
+ allOnes &&= byte === 255;
4149
+ }
4150
+ return {
4151
+ value,
4152
+ length,
4153
+ unknown: !keepMarker && allOnes
4154
+ };
4155
+ }
4156
+ function frameOf(block) {
4157
+ const track = readVint(block, 0, false);
4158
+ const flags = block[track.length + 2];
4159
+ if (flags === void 0) throw new Error("Not a WebM file: empty block");
4160
+ if (flags & 6) throw new Error("WebM lacing isn't supported");
4161
+ return block.subarray(track.length + 3);
4162
+ }
4163
+ function parse(data) {
4164
+ const packets = [];
4165
+ let offset = 0;
4166
+ while (offset < data.length) {
4167
+ let id;
4168
+ let size;
4169
+ try {
4170
+ id = readVint(data, offset, true);
4171
+ size = readVint(data, offset + id.length, false);
4172
+ } catch (error) {
4173
+ if (error instanceof CutShort) break;
4174
+ throw error;
4175
+ }
4176
+ const body = offset + id.length + size.length;
4177
+ if (CONTAINERS.has(id.value)) {
4178
+ offset = body;
4179
+ continue;
4180
+ }
4181
+ if (size.unknown) throw new Error("Not a WebM file: an element of unknown size");
4182
+ const end = body + size.value;
4183
+ if (end > data.length) break;
4184
+ if (id.value === SIMPLE_BLOCK || id.value === BLOCK) {
4185
+ const frame = frameOf(data.subarray(body, end));
4186
+ if (frame.length) packets.push(frame);
4187
+ }
4188
+ offset = end;
4189
+ }
4190
+ return {
4191
+ packets,
4192
+ consumed: offset
4193
+ };
4194
+ }
4195
+ var WebmOpusReader = class {
4196
+ #pending = Buffer.alloc(0);
4197
+ push(chunk) {
4198
+ this.#pending = Buffer.concat([this.#pending, chunk]);
4199
+ const { packets, consumed } = parse(this.#pending);
4200
+ this.#pending = this.#pending.subarray(consumed);
4201
+ return packets;
4202
+ }
4203
+ get complete() {
4204
+ return this.#pending.length === 0;
4205
+ }
4206
+ };
4207
+ //#endregion
4208
+ //#region ../charmd/src/voice/microsoft.ts
4209
+ const TRUSTED_CLIENT_TOKEN = "6A5AA1D4EAFF4E9FB37E23D68491D6F4";
4210
+ const SERVICE_URL = "wss://speech.platform.bing.com/consumer/speech/synthesize/readaloud/edge/v1";
4211
+ const GEC_VERSION = "1-143.0.3650.96";
4212
+ const USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/143.0.0.0 Safari/537.36 Edg/143.0.0.0";
4213
+ const ORIGIN = "chrome-extension://jdiccldimpdaibmpdkjnbmckianbfold";
4214
+ const OUTPUT_FORMAT = "webm-24khz-16bit-mono-opus";
4215
+ const DEFAULT_VOICES = {
4216
+ en: "en-US-AvaMultilingualNeural",
4217
+ it: "it-IT-IsabellaNeural",
4218
+ es: "es-ES-ElviraNeural",
4219
+ fr: "fr-FR-DeniseNeural",
4220
+ de: "de-DE-KatjaNeural",
4221
+ pt: "pt-BR-FranciscaNeural"
4222
+ };
4223
+ const WINDOWS_EPOCH_SECONDS = 11644473600;
4224
+ function secMsGec(nowMs = Date.now()) {
4225
+ const seconds = Math.floor(nowMs / 1e3) + WINDOWS_EPOCH_SECONDS;
4226
+ const ticks = BigInt(seconds - seconds % 300) * 10000000n;
4227
+ return createHash("sha256").update(`${ticks}${TRUSTED_CLIENT_TOKEN}`).digest("hex").toUpperCase();
4228
+ }
4229
+ function escapeXml(text) {
4230
+ return text.replace(/&/g, "&amp;").replace(/</g, "&lt;").replace(/>/g, "&gt;").replace(/"/g, "&quot;").replace(/'/g, "&apos;");
4231
+ }
4232
+ function ssml(text, voice) {
4233
+ return `<speak version='1.0' xmlns='http://www.w3.org/2001/10/synthesis' xml:lang='${voice.split("-").slice(0, 2).join("-")}'><voice name='${voice}'><prosody pitch='+0Hz' rate='+0%' volume='+0%'>${escapeXml(text)}</prosody></voice></speak>`;
4234
+ }
4235
+ function message(path, contentType, body, requestId) {
4236
+ return `X-Timestamp:${(/* @__PURE__ */ new Date()).toISOString()}\r\n` + (requestId ? `X-RequestId:${requestId}\r\n` : "") + `Content-Type:${contentType}\r\nPath:${path}\r\n\r\n${body}`;
4237
+ }
4238
+ function audioOf(frame) {
4239
+ if (frame.length < 2) return void 0;
4240
+ const headerLength = frame.readUInt16BE(0);
4241
+ return frame.subarray(2, 2 + headerLength).toString().includes("Path:audio") ? frame.subarray(2 + headerLength) : void 0;
4242
+ }
4243
+ function voiceFor(language, voices) {
4244
+ return voices[language] ?? voices.en ?? DEFAULT_VOICES.en;
4245
+ }
4246
+ function connect(text, language, voices, signal, options) {
4247
+ const timeoutMs = options.timeoutMs ?? 15e3;
4248
+ const connection = randomUUID().replace(/-/g, "");
4249
+ const url = `${options.url ?? SERVICE_URL}?TrustedClientToken=${TRUSTED_CLIENT_TOKEN}&Sec-MS-GEC=${secMsGec()}&Sec-MS-GEC-Version=${GEC_VERSION}&ConnectionId=${connection}`;
4250
+ const packets = createChannel();
4251
+ const reader = new WebmOpusReader();
4252
+ let sent = 0;
4253
+ let settled = false;
4254
+ const socket = new WebSocket(url, {
4255
+ headers: {
4256
+ "User-Agent": USER_AGENT,
4257
+ Origin: ORIGIN
4258
+ },
4259
+ handshakeTimeout: timeoutMs
4260
+ });
4261
+ const finish = (error) => {
4262
+ if (settled) return;
4263
+ settled = true;
4264
+ clearTimeout(timer);
4265
+ signal.removeEventListener("abort", onAbort);
4266
+ socket.terminate();
4267
+ if (error) packets.fail(error);
4268
+ else if (sent === 0) packets.fail(/* @__PURE__ */ new Error("Microsoft's voice sent no audio"));
4269
+ else packets.end();
4270
+ };
4271
+ const onAbort = () => finish(signal.reason instanceof Error ? signal.reason : /* @__PURE__ */ new Error("Aborted"));
4272
+ const timer = setTimeout(() => finish(/* @__PURE__ */ new Error("Microsoft's voice took too long")), timeoutMs);
4273
+ if (signal.aborted) onAbort();
4274
+ else signal.addEventListener("abort", onAbort, { once: true });
4275
+ socket.on("open", () => {
4276
+ socket.send(message("speech.config", "application/json; charset=utf-8", JSON.stringify({ context: { synthesis: { audio: {
4277
+ metadataoptions: {
4278
+ sentenceBoundaryEnabled: "false",
4279
+ wordBoundaryEnabled: "false"
4280
+ },
4281
+ outputFormat: OUTPUT_FORMAT
4282
+ } } } })));
4283
+ socket.send(message("ssml", "application/ssml+xml", ssml(text, voiceFor(language, voices)), connection));
4284
+ });
4285
+ socket.on("message", (data, isBinary) => {
4286
+ const frame = Buffer.isBuffer(data) ? data : Buffer.concat(data);
4287
+ if (!isBinary) {
4288
+ if (frame.toString().includes("Path:turn.end")) finish();
4289
+ return;
4290
+ }
4291
+ const audio = audioOf(frame);
4292
+ if (!audio?.length) return;
4293
+ try {
4294
+ for (const packet of reader.push(audio)) {
4295
+ sent += 1;
4296
+ packets.push(packet);
4297
+ }
4298
+ } catch (error) {
4299
+ finish(error instanceof Error ? error : new Error(String(error)));
4300
+ }
4301
+ });
4302
+ socket.on("unexpected-response", (_request, response) => finish(/* @__PURE__ */ new Error(`Microsoft's voice refused the request: ${response.statusCode}`)));
4303
+ socket.on("error", (error) => finish(/* @__PURE__ */ new Error(`Microsoft's voice failed: ${error.message}`)));
4304
+ socket.on("close", () => finish(/* @__PURE__ */ new Error("Microsoft's voice closed before it finished")));
4305
+ return packets;
4306
+ }
4307
+ function createMicrosoftSpeaker(options = {}) {
4308
+ const voices = {
4309
+ ...DEFAULT_VOICES,
4310
+ ...options.voices
4311
+ };
4312
+ const stream = (text, signal, language = "en") => connect(text, language, voices, signal, options);
4313
+ return {
4314
+ name: "microsoft",
4315
+ stream,
4316
+ async synthesize(text, signal, language = "en") {
4317
+ const packets = [];
4318
+ for await (const packet of stream(text, signal, language)) packets.push(packet);
4319
+ return writeOggOpus(packets, { inputSampleRate: 24e3 });
4320
+ }
4321
+ };
4322
+ }
4323
+ //#endregion
3523
4324
  //#region ../charmd/src/voice/openai.ts
3524
4325
  function createOpenAiVoice(options) {
3525
4326
  const apiKey = options.apiKey ?? process.env.OPENAI_API_KEY;
@@ -3563,22 +4364,260 @@ function createOpenAiVoice(options) {
3563
4364
  };
3564
4365
  }
3565
4366
  //#endregion
4367
+ //#region ../charmd/src/voice/sherpa.ts
4368
+ const SUPERTONIC_LANGUAGES = /* @__PURE__ */ new Set([
4369
+ "en",
4370
+ "it",
4371
+ "es",
4372
+ "fr",
4373
+ "de",
4374
+ "pt"
4375
+ ]);
4376
+ const DEFAULT_SPEAKER = 5;
4377
+ const OUTPUT_RATE = 24e3;
4378
+ let sherpa;
4379
+ function loadSherpa() {
4380
+ if (sherpa) return sherpa;
4381
+ try {
4382
+ sherpa = createRequire(import.meta.url)("sherpa-onnx-node");
4383
+ return sherpa;
4384
+ } catch (error) {
4385
+ throw new Error(`The local voice can't run on this computer: sherpa-onnx didn't load (${error instanceof Error ? error.message.split("\n")[0] : String(error)})`, { cause: error });
4386
+ }
4387
+ }
4388
+ function ready(id, dir) {
4389
+ const state = modelState(id, dir);
4390
+ if (state.state === "ready") return join(dir, MODELS[id].folder);
4391
+ if (state.state === "failed" && Date.now() - state.at < 3e5) throw new VoiceNotReady(`Couldn't get ${MODELS[id].label}: ${state.error}`.slice(0, 160));
4392
+ if (state.state !== "downloading") installModel(id, { dir }).catch(() => void 0);
4393
+ const now = modelState(id, dir);
4394
+ const percent = now.state === "downloading" && now.total ? ` (${Math.floor(now.received * 100 / now.total)}%)` : "";
4395
+ throw new VoiceNotReady(`${MODELS[id].label} is still downloading${percent}`);
4396
+ }
4397
+ function toFloat(pcm) {
4398
+ const out = new Float32Array(pcm.length / 2);
4399
+ for (let i = 0; i < out.length; i++) out[i] = pcm.readInt16LE(i * 2) / 32768;
4400
+ return out;
4401
+ }
4402
+ function toPcm(samples) {
4403
+ const out = Buffer.alloc(samples.length * 2);
4404
+ for (let i = 0; i < samples.length; i++) out.writeInt16LE(Math.round(Math.max(-1, Math.min(1, samples[i] ?? 0)) * 32767), i * 2);
4405
+ return out;
4406
+ }
4407
+ function createParakeetListener(options = {}) {
4408
+ const dir = options.modelsDir ?? defaultModelsDir();
4409
+ let recognizer;
4410
+ const load = () => {
4411
+ const folder = ready("parakeet", dir);
4412
+ recognizer ??= loadSherpa().OfflineRecognizer.createAsync({
4413
+ featConfig: {
4414
+ sampleRate: 16e3,
4415
+ featureDim: 80
4416
+ },
4417
+ modelConfig: {
4418
+ transducer: {
4419
+ encoder: join(folder, "encoder.int8.onnx"),
4420
+ decoder: join(folder, "decoder.int8.onnx"),
4421
+ joiner: join(folder, "joiner.int8.onnx")
4422
+ },
4423
+ tokens: join(folder, "tokens.txt"),
4424
+ modelType: "nemo_transducer",
4425
+ numThreads: options.threads ?? 4,
4426
+ provider: "cpu"
4427
+ }
4428
+ }).catch((error) => {
4429
+ recognizer = void 0;
4430
+ throw error;
4431
+ });
4432
+ return recognizer;
4433
+ };
4434
+ return {
4435
+ name: "local",
4436
+ warm: async () => {
4437
+ await load();
4438
+ },
4439
+ async transcribe(ogg) {
4440
+ const engine = await load();
4441
+ const stream = engine.createStream();
4442
+ stream.acceptWaveform({
4443
+ samples: toFloat(oggOpusToPcm(ogg, 16e3)),
4444
+ sampleRate: 16e3
4445
+ });
4446
+ return ((await engine.decodeAsync(stream)).text ?? "").replace(/\s+/g, " ").trim();
4447
+ }
4448
+ };
4449
+ }
4450
+ function createSupertonicSpeaker(options = {}) {
4451
+ const dir = options.modelsDir ?? defaultModelsDir();
4452
+ let tts;
4453
+ const load = () => {
4454
+ const folder = ready("supertonic", dir);
4455
+ const file = (name) => join(folder, name);
4456
+ tts ??= loadSherpa().OfflineTts.createAsync({
4457
+ model: {
4458
+ supertonic: {
4459
+ durationPredictor: file("duration_predictor.int8.onnx"),
4460
+ textEncoder: file("text_encoder.int8.onnx"),
4461
+ vectorEstimator: file("vector_estimator.int8.onnx"),
4462
+ vocoder: file("vocoder.int8.onnx"),
4463
+ ttsJson: file("tts.json"),
4464
+ unicodeIndexer: file("unicode_indexer.bin"),
4465
+ voiceStyle: file("voice.bin")
4466
+ },
4467
+ numThreads: options.threads ?? 4,
4468
+ provider: "cpu"
4469
+ },
4470
+ maxNumSentences: 1
4471
+ }).catch((error) => {
4472
+ tts = void 0;
4473
+ throw error;
4474
+ });
4475
+ return tts;
4476
+ };
4477
+ return {
4478
+ name: "local",
4479
+ onDevice: true,
4480
+ warm: async () => {
4481
+ await load();
4482
+ },
4483
+ async synthesize(text, signal, language = "en") {
4484
+ const engine = await load();
4485
+ const sherpaModule = loadSherpa();
4486
+ const audio = await engine.generateAsync({
4487
+ text,
4488
+ generationConfig: new sherpaModule.GenerationConfig({
4489
+ sid: options.speaker ?? DEFAULT_SPEAKER,
4490
+ numSteps: 8,
4491
+ extra: { lang: SUPERTONIC_LANGUAGES.has(language) ? language : "en" }
4492
+ }),
4493
+ onProgress: () => signal.aborted ? 0 : 1
4494
+ });
4495
+ signal.throwIfAborted();
4496
+ return pcmToOggOpus(toPcm(audio.sampleRate === OUTPUT_RATE ? audio.samples : new sherpaModule.LinearResampler(audio.sampleRate, OUTPUT_RATE).flush(audio.samples)), OUTPUT_RATE);
4497
+ }
4498
+ };
4499
+ }
4500
+ //#endregion
4501
+ //#region ../charmd/src/voice/speak.ts
4502
+ const FIRST_AUDIO_MS = 3e3;
4503
+ const REST_MS = 6e4;
4504
+ async function firstWithin(packets, ms, name) {
4505
+ let timer;
4506
+ try {
4507
+ await Promise.race([packets.first(), new Promise((_, reject) => {
4508
+ timer = setTimeout(() => reject(/* @__PURE__ */ new Error(`${name} had no audio within ${ms} ms`)), ms);
4509
+ })]);
4510
+ } finally {
4511
+ clearTimeout(timer);
4512
+ }
4513
+ }
4514
+ function speakWithFallback(chain, options = {}) {
4515
+ const [first] = chain;
4516
+ if (!first) throw new Error("No voice to speak with");
4517
+ if (chain.length === 1) return first;
4518
+ const now = options.now ?? Date.now;
4519
+ const resting = /* @__PURE__ */ new Map();
4520
+ async function* stream(text, signal, language) {
4521
+ let lastError;
4522
+ for (const [index, speaker] of chain.entries()) {
4523
+ const last = index === chain.length - 1;
4524
+ if (!last && (resting.get(speaker) ?? 0) > now()) continue;
4525
+ const controller = new AbortController();
4526
+ const abort = () => controller.abort(signal.reason);
4527
+ signal.addEventListener("abort", abort, { once: true });
4528
+ const packets = new PacketStream(packetsOf(speaker, text, controller.signal, language));
4529
+ try {
4530
+ if (last || speaker.onDevice) await packets.first();
4531
+ else await firstWithin(packets, options.timeoutMs ?? FIRST_AUDIO_MS, speaker.name);
4532
+ } catch (error) {
4533
+ signal.removeEventListener("abort", abort);
4534
+ controller.abort(error);
4535
+ signal.throwIfAborted();
4536
+ lastError = error;
4537
+ resting.set(speaker, now() + (options.restMs ?? REST_MS));
4538
+ continue;
4539
+ }
4540
+ try {
4541
+ yield* packets;
4542
+ } finally {
4543
+ signal.removeEventListener("abort", abort);
4544
+ }
4545
+ return;
4546
+ }
4547
+ throw lastError instanceof Error ? lastError : new Error(String(lastError));
4548
+ }
4549
+ return {
4550
+ name: first.name,
4551
+ stream,
4552
+ async synthesize(text, signal, language) {
4553
+ const packets = [];
4554
+ for await (const packet of stream(text, signal, language)) packets.push(packet);
4555
+ return writeOggOpus(packets, { inputSampleRate: 24e3 });
4556
+ },
4557
+ warm: async () => {
4558
+ await Promise.allSettled(chain.map((speaker) => speaker.warm?.() ?? Promise.resolve()));
4559
+ }
4560
+ };
4561
+ }
4562
+ //#endregion
3566
4563
  //#region ../charmd/src/wiring.ts
3567
4564
  function key(config, fallbackEnv) {
3568
4565
  const env = config.apiKeyEnv ?? fallbackEnv;
3569
4566
  return config.apiKey ?? (env ? process.env[env] : void 0);
3570
4567
  }
3571
- function createVoiceFromConfig(config) {
3572
- const voice = config.voice;
3573
- switch (voice.provider) {
3574
- case "fake": return createFakeVoice({ transcript: voice.transcript });
3575
- case "local": return createLocalVoice(voice);
4568
+ function createListener(config) {
4569
+ switch (config.provider) {
4570
+ case "fake": return createFakeVoice(config);
4571
+ case "local": return createParakeetListener();
4572
+ case "whisper": return createLocalVoice({
4573
+ ...config.model ? { whisperModel: config.model } : {},
4574
+ ...config.command ? { whisperCommand: config.command } : {}
4575
+ });
4576
+ case "openai": return createOpenAiVoice({
4577
+ ...config,
4578
+ apiKey: key(config, "OPENAI_API_KEY"),
4579
+ ...config.model ? { sttModel: config.model } : {}
4580
+ });
4581
+ }
4582
+ }
4583
+ function createSpeaker(config) {
4584
+ switch (config.provider) {
4585
+ case "fake": return createFakeVoice();
4586
+ case "microsoft": return createMicrosoftSpeaker(config.voices ? { voices: config.voices } : {});
4587
+ case "local": return createSupertonicSpeaker(config.speaker === void 0 ? {} : { speaker: config.speaker });
4588
+ case "system": return {
4589
+ ...createLocalVoice(config.voice ? { sayVoice: config.voice } : {}),
4590
+ name: "system",
4591
+ onDevice: true
4592
+ };
3576
4593
  case "openai": return createOpenAiVoice({
3577
- ...voice,
3578
- apiKey: key(voice, "OPENAI_API_KEY")
4594
+ ...config,
4595
+ apiKey: key(config, "OPENAI_API_KEY"),
4596
+ ...config.model ? { ttsModel: config.model } : {}
3579
4597
  });
3580
4598
  }
3581
4599
  }
4600
+ function speakerChain(config) {
4601
+ if (config.provider === "fake" || config.provider === "system") return createSpeaker(config);
4602
+ const chain = [createSpeaker(config)];
4603
+ if (config.provider === "microsoft") chain.push(createSupertonicSpeaker());
4604
+ if (process.platform === "darwin") chain.push(createSpeaker({ provider: "system" }));
4605
+ return speakWithFallback(chain);
4606
+ }
4607
+ function createVoiceFromConfig(config) {
4608
+ const listener = createListener(config.voice.listen);
4609
+ const speaker = speakerChain(config.voice.speak);
4610
+ return {
4611
+ name: listener.name === speaker.name ? listener.name : `${listener.name}+${speaker.name}`,
4612
+ transcribe: listener.transcribe,
4613
+ synthesize: speaker.synthesize,
4614
+ ...speaker.stream ? { stream: speaker.stream } : {},
4615
+ language: config.voice.language,
4616
+ warm: async () => {
4617
+ await Promise.allSettled([listener.warm?.(), speaker.warm?.()]);
4618
+ }
4619
+ };
4620
+ }
3582
4621
  function charmToolsServers(config, tools) {
3583
4622
  const agent = config.agent;
3584
4623
  if (agent.adapter !== "acp" || !agent.charmTools || !tools) return [];
@@ -3631,6 +4670,7 @@ async function startDaemon(config, options = {}) {
3631
4670
  });
3632
4671
  const sessions = /* @__PURE__ */ new Map();
3633
4672
  const voice = createVoiceFromConfig(config);
4673
+ voice.warm?.().catch(() => void 0);
3634
4674
  const log = options.log ?? ((entry) => {
3635
4675
  if (!options.quiet) console.log(JSON.stringify({
3636
4676
  time: (/* @__PURE__ */ new Date()).toISOString(),
@@ -3650,7 +4690,15 @@ async function startDaemon(config, options = {}) {
3650
4690
  look,
3651
4691
  agentCanChangeLook,
3652
4692
  identity: options.identity,
3653
- starter: options.starter
4693
+ starter: options.starter,
4694
+ voice: () => {
4695
+ const downloading = ["parakeet", "supertonic"].flatMap((id) => {
4696
+ const state = modelState(id);
4697
+ if (state.state === "failed") return [`${MODELS[id].label} failed`];
4698
+ return state.state === "downloading" && state.total ? [`${MODELS[id].label} downloading ${Math.floor(state.received * 100 / state.total)}%`] : [];
4699
+ });
4700
+ return [voice.name, ...downloading].join(" · ");
4701
+ }
3654
4702
  });
3655
4703
  const adminServer = await startAdminServer(config.adminSocket, admin);
3656
4704
  let websocketUrl = config.publicUrl ?? "";
@@ -3783,7 +4831,7 @@ function formatIdentity(unit, version, commit) {
3783
4831
  let cached;
3784
4832
  function identity() {
3785
4833
  if (cached) return cached;
3786
- const commit = "f4edfa0";
4834
+ const commit = "42da0c7";
3787
4835
  cached = {
3788
4836
  version: `cli@${version}`,
3789
4837
  commit,
@@ -3832,6 +4880,45 @@ async function runServe(ctx, args) {
3832
4880
  process.once("SIGTERM", stop);
3833
4881
  }
3834
4882
  //#endregion
4883
+ //#region src/commands/voice.ts
4884
+ const IDS = Object.keys(MODELS);
4885
+ function megabytes(bytes) {
4886
+ return `${Math.round(bytes / 1e6)} MB`;
4887
+ }
4888
+ async function runVoice(ctx, args, deps = {}) {
4889
+ const install = deps.install ?? installModel;
4890
+ const state = deps.state ?? modelState;
4891
+ const action = firstPositional(args) ?? "";
4892
+ if (action !== "" && action !== "install") {
4893
+ ctx.err.write(`Unknown voice command "${action}". Try: opencharm voice install\n`);
4894
+ process.exitCode = 1;
4895
+ return;
4896
+ }
4897
+ for (const id of IDS) {
4898
+ const model = MODELS[id];
4899
+ if (action === "") {
4900
+ const ready = state(id).state === "ready";
4901
+ ctx.out.write(`${model.label}: ${ready ? "installed" : `not downloaded yet (${megabytes(model.bytes)})`}\n`);
4902
+ continue;
4903
+ }
4904
+ let shown = -1;
4905
+ try {
4906
+ await install(id, { onProgress: (received, total) => {
4907
+ const percent = Math.floor(received * 100 / (total || model.bytes));
4908
+ if (percent >= shown + 10) {
4909
+ shown = percent - percent % 10;
4910
+ ctx.out.write(`${model.label}: ${shown}%\n`);
4911
+ }
4912
+ } });
4913
+ ctx.out.write(`${model.label}: installed\n`);
4914
+ } catch (error) {
4915
+ ctx.err.write(`${error instanceof Error ? error.message : String(error)}\n`);
4916
+ process.exitCode = 1;
4917
+ return;
4918
+ }
4919
+ }
4920
+ }
4921
+ //#endregion
3835
4922
  //#region src/commands/setup.ts
3836
4923
  const USER = "charmd";
3837
4924
  const STATE_DIR = "/var/lib/charmd";
@@ -4206,6 +5293,9 @@ async function main(argv) {
4206
5293
  case "serve":
4207
5294
  await runServe(ctx, rest);
4208
5295
  return;
5296
+ case "voice":
5297
+ await runVoice(ctx, rest);
5298
+ return;
4209
5299
  case "mcp": {
4210
5300
  const socket = flagValue(rest, "--socket") ?? loadConfig(flagValue(rest, "--config")).adminSocket;
4211
5301
  await serveMcp(process.stdin, process.stdout, {