opencharm 0.2.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -0
- package/dist/main.mjs +1137 -47
- package/package.json +2 -1
- package/sim/charm_sim.wasm +0 -0
package/dist/main.mjs
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
|
-
import {
|
|
2
|
+
import { createRequire } from "node:module";
|
|
3
|
+
import { chmodSync, createWriteStream, existsSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, realpathSync, renameSync, rmSync, statSync, unlinkSync, writeFileSync } from "node:fs";
|
|
3
4
|
import { homedir, tmpdir } from "node:os";
|
|
4
5
|
import { dirname, extname, isAbsolute, join, normalize, resolve, sep } from "node:path";
|
|
5
6
|
import { z } from "zod";
|
|
@@ -8,8 +9,9 @@ import { createInterface } from "node:readline";
|
|
|
8
9
|
import { execFile, execFileSync, spawn, spawnSync } from "node:child_process";
|
|
9
10
|
import { createHash, randomBytes, randomInt, randomUUID, scrypt, timingSafeEqual } from "node:crypto";
|
|
10
11
|
import { createServer as createServer$1 } from "node:http";
|
|
11
|
-
import { WebSocketServer } from "ws";
|
|
12
|
+
import WebSocket, { WebSocketServer } from "ws";
|
|
12
13
|
import { Readable, Writable } from "node:stream";
|
|
14
|
+
import { pipeline } from "node:stream/promises";
|
|
13
15
|
import * as acp from "@agentclientprotocol/sdk";
|
|
14
16
|
import OpusScript from "opusscript";
|
|
15
17
|
import { fileURLToPath } from "node:url";
|
|
@@ -710,7 +712,7 @@ const secret = {
|
|
|
710
712
|
apiKey: z.string().min(1).optional(),
|
|
711
713
|
apiKeyEnv: z.string().min(1).optional()
|
|
712
714
|
};
|
|
713
|
-
const
|
|
715
|
+
const legacyVoiceSchema = z.discriminatedUnion("provider", [
|
|
714
716
|
z.object({
|
|
715
717
|
provider: z.literal("fake"),
|
|
716
718
|
transcript: z.string().optional()
|
|
@@ -729,6 +731,52 @@ const voiceSchema = z.discriminatedUnion("provider", [
|
|
|
729
731
|
voice: z.string().optional()
|
|
730
732
|
}).strict()
|
|
731
733
|
]);
|
|
734
|
+
const languageCode = z.string().regex(/^[a-z]{2}$/);
|
|
735
|
+
const listenSchema = z.discriminatedUnion("provider", [
|
|
736
|
+
z.object({ provider: z.literal("local") }).strict(),
|
|
737
|
+
z.object({
|
|
738
|
+
provider: z.literal("whisper"),
|
|
739
|
+
model: z.string().optional(),
|
|
740
|
+
command: z.string().optional()
|
|
741
|
+
}).strict(),
|
|
742
|
+
z.object({
|
|
743
|
+
provider: z.literal("openai"),
|
|
744
|
+
...secret,
|
|
745
|
+
baseUrl: z.url().optional(),
|
|
746
|
+
model: z.string().optional()
|
|
747
|
+
}).strict(),
|
|
748
|
+
z.object({
|
|
749
|
+
provider: z.literal("fake"),
|
|
750
|
+
transcript: z.string().optional()
|
|
751
|
+
}).strict()
|
|
752
|
+
]);
|
|
753
|
+
const speakSchema = z.discriminatedUnion("provider", [
|
|
754
|
+
z.object({
|
|
755
|
+
provider: z.literal("microsoft"),
|
|
756
|
+
voices: z.record(languageCode, z.string().regex(/^[A-Za-z]{2,3}-[A-Za-z]{2,4}-\w+$/)).optional()
|
|
757
|
+
}).strict(),
|
|
758
|
+
z.object({
|
|
759
|
+
provider: z.literal("local"),
|
|
760
|
+
speaker: z.number().int().min(0).max(9).optional()
|
|
761
|
+
}).strict(),
|
|
762
|
+
z.object({
|
|
763
|
+
provider: z.literal("system"),
|
|
764
|
+
voice: z.string().optional()
|
|
765
|
+
}).strict(),
|
|
766
|
+
z.object({
|
|
767
|
+
provider: z.literal("openai"),
|
|
768
|
+
...secret,
|
|
769
|
+
baseUrl: z.url().optional(),
|
|
770
|
+
model: z.string().optional(),
|
|
771
|
+
voice: z.string().optional()
|
|
772
|
+
}).strict(),
|
|
773
|
+
z.object({ provider: z.literal("fake") }).strict()
|
|
774
|
+
]);
|
|
775
|
+
const voiceSchema = z.union([legacyVoiceSchema, z.object({
|
|
776
|
+
listen: listenSchema,
|
|
777
|
+
speak: speakSchema,
|
|
778
|
+
language: languageCode.default("en")
|
|
779
|
+
}).strict()]);
|
|
732
780
|
const agentSchema = z.discriminatedUnion("adapter", [
|
|
733
781
|
z.object({ adapter: z.literal("fake") }).strict(),
|
|
734
782
|
z.object({
|
|
@@ -809,6 +857,52 @@ function resolveAgent(agent, base) {
|
|
|
809
857
|
...agent.projects ? { projects: agent.projects.map((p) => expandPath(p, base)) } : {}
|
|
810
858
|
};
|
|
811
859
|
}
|
|
860
|
+
function normalizeVoice(voice) {
|
|
861
|
+
if ("listen" in voice) return voice;
|
|
862
|
+
switch (voice.provider) {
|
|
863
|
+
case "fake": return {
|
|
864
|
+
listen: {
|
|
865
|
+
provider: "fake",
|
|
866
|
+
...voice.transcript ? { transcript: voice.transcript } : {}
|
|
867
|
+
},
|
|
868
|
+
speak: { provider: "fake" },
|
|
869
|
+
language: "en"
|
|
870
|
+
};
|
|
871
|
+
case "local": return {
|
|
872
|
+
listen: voice.whisperModel ? {
|
|
873
|
+
provider: "whisper",
|
|
874
|
+
model: voice.whisperModel
|
|
875
|
+
} : { provider: "local" },
|
|
876
|
+
speak: voice.sayVoice ? {
|
|
877
|
+
provider: "system",
|
|
878
|
+
voice: voice.sayVoice
|
|
879
|
+
} : { provider: "local" },
|
|
880
|
+
language: "en"
|
|
881
|
+
};
|
|
882
|
+
case "openai": {
|
|
883
|
+
const { sttModel, ttsModel, voice: name } = voice;
|
|
884
|
+
const rest = {
|
|
885
|
+
...voice.apiKey ? { apiKey: voice.apiKey } : {},
|
|
886
|
+
...voice.apiKeyEnv ? { apiKeyEnv: voice.apiKeyEnv } : {},
|
|
887
|
+
...voice.baseUrl ? { baseUrl: voice.baseUrl } : {}
|
|
888
|
+
};
|
|
889
|
+
return {
|
|
890
|
+
listen: {
|
|
891
|
+
provider: "openai",
|
|
892
|
+
...rest,
|
|
893
|
+
...sttModel ? { model: sttModel } : {}
|
|
894
|
+
},
|
|
895
|
+
speak: {
|
|
896
|
+
provider: "openai",
|
|
897
|
+
...rest,
|
|
898
|
+
...ttsModel ? { model: ttsModel } : {},
|
|
899
|
+
...name ? { voice: name } : {}
|
|
900
|
+
},
|
|
901
|
+
language: "en"
|
|
902
|
+
};
|
|
903
|
+
}
|
|
904
|
+
}
|
|
905
|
+
}
|
|
812
906
|
function parseConfig(input, baseDir) {
|
|
813
907
|
const result = schema.safeParse(input);
|
|
814
908
|
if (!result.success) {
|
|
@@ -824,7 +918,8 @@ function parseConfig(input, baseDir) {
|
|
|
824
918
|
...config,
|
|
825
919
|
statePath,
|
|
826
920
|
adminSocket: config.adminSocket ?? defaultAdminSocket(statePath),
|
|
827
|
-
agent
|
|
921
|
+
agent,
|
|
922
|
+
voice: normalizeVoice(config.voice)
|
|
828
923
|
};
|
|
829
924
|
}
|
|
830
925
|
function loadConfig(path, options = {}) {
|
|
@@ -866,7 +961,7 @@ function sendAdmin(socketPath, request) {
|
|
|
866
961
|
}
|
|
867
962
|
//#endregion
|
|
868
963
|
//#region package.json
|
|
869
|
-
var version = "0.
|
|
964
|
+
var version = "0.3.0";
|
|
870
965
|
function channelToCube(value) {
|
|
871
966
|
if (value < 48) return 0;
|
|
872
967
|
if (value < 115) return 1;
|
|
@@ -1010,6 +1105,7 @@ function promptHidden(question) {
|
|
|
1010
1105
|
const statusSchema = z.object({
|
|
1011
1106
|
charmd: z.string().regex(/^[\w.@+() -]{1,96}$/).optional().catch(void 0),
|
|
1012
1107
|
starter: z.string().regex(/^[0-9a-f]{4,40}$/).optional().catch(void 0),
|
|
1108
|
+
voice: z.string().regex(/^[\w.+()%· -]{1,160}$/).optional().catch(void 0),
|
|
1013
1109
|
charms: z.array(z.object({
|
|
1014
1110
|
name: z.string(),
|
|
1015
1111
|
state: z.string(),
|
|
@@ -1102,8 +1198,9 @@ async function look(ctx, args, deps) {
|
|
|
1102
1198
|
if (Object.keys(update).length === 0) return;
|
|
1103
1199
|
ctx.out.write(`${result.connected === 0 ? "No charm is unlocked right now: it gets the look when it unlocks." : `Sent to ${result.connected} charm${result.connected === 1 ? "" : "s"}.`} It lasts until charmd restarts; to keep it, set "charm" in opencharm.json.\n`);
|
|
1104
1200
|
}
|
|
1105
|
-
function printStatus(ctx, charms, charmd, starter) {
|
|
1201
|
+
function printStatus(ctx, charms, charmd, starter, voice) {
|
|
1106
1202
|
if (charmd) ctx.out.write(`charmd ${charmd}\n`);
|
|
1203
|
+
if (voice) ctx.out.write(`voice ${voice}\n`);
|
|
1107
1204
|
if (starter) ctx.out.write(`workspace from starter ${starter}\n`);
|
|
1108
1205
|
if (charms.length === 0) {
|
|
1109
1206
|
ctx.out.write("No charms paired yet. Start one and run: opencharm pair <code>\n");
|
|
@@ -1134,8 +1231,8 @@ async function runAdminCommand(ctx, command, args, deps) {
|
|
|
1134
1231
|
fail$2(ctx, "Unexpected answer from charmd: is it the same version as this CLI?");
|
|
1135
1232
|
return;
|
|
1136
1233
|
}
|
|
1137
|
-
const { charms, charmd, starter } = answer.data;
|
|
1138
|
-
printStatus(ctx, charms, charmd, starter);
|
|
1234
|
+
const { charms, charmd, starter, voice } = answer.data;
|
|
1235
|
+
printStatus(ctx, charms, charmd, starter, voice);
|
|
1139
1236
|
return;
|
|
1140
1237
|
}
|
|
1141
1238
|
const charm = firstPositional(args);
|
|
@@ -1349,6 +1446,7 @@ function runHelp(ctx) {
|
|
|
1349
1446
|
opencharm setup --domain <d> install charmd as a service (Linux, sudo) [--dry-run]
|
|
1350
1447
|
opencharm serve start charmd [--config <file>]
|
|
1351
1448
|
opencharm sim the charm on screen, in your browser [--url <ws>] [--port <n>]
|
|
1449
|
+
opencharm voice the local voice's models [install: download them now]
|
|
1352
1450
|
opencharm pair <code> pair the charm showing <code> [--name <name>]
|
|
1353
1451
|
opencharm status paired charms and their state
|
|
1354
1452
|
opencharm lock <charm> lock a charm now
|
|
@@ -1373,6 +1471,10 @@ function runHelp(ctx) {
|
|
|
1373
1471
|
//#endregion
|
|
1374
1472
|
//#region src/commands/init.ts
|
|
1375
1473
|
const STARTER = "https://github.com/opencharm-labs/opencharm-starter.git";
|
|
1474
|
+
const DEFAULT_VOICE = {
|
|
1475
|
+
listen: { provider: "local" },
|
|
1476
|
+
speak: { provider: "microsoft" }
|
|
1477
|
+
};
|
|
1376
1478
|
function checkLook(options) {
|
|
1377
1479
|
const result = z.object({
|
|
1378
1480
|
name: lookFields.name,
|
|
@@ -1409,13 +1511,9 @@ function initWorkspace(target, options) {
|
|
|
1409
1511
|
const path = join(target, "opencharm.json");
|
|
1410
1512
|
if (!existsSync(path)) throw new Error(`${from} has no opencharm.json: is it an OpenCharm starter? (${target} was cloned; delete it)`);
|
|
1411
1513
|
const config = JSON.parse(readFileSync(path, "utf8"));
|
|
1412
|
-
const voice = options.platform === "darwin" ? "local" : "fake";
|
|
1413
1514
|
const hasLook = Object.keys(look).length > 0;
|
|
1414
|
-
if (config.voice
|
|
1415
|
-
config.voice =
|
|
1416
|
-
...config.voice,
|
|
1417
|
-
provider: voice
|
|
1418
|
-
};
|
|
1515
|
+
if (JSON.stringify(config.voice) === JSON.stringify(DEFAULT_VOICE) && config.agent?.agent === agent && !hasLook) return;
|
|
1516
|
+
config.voice = DEFAULT_VOICE;
|
|
1419
1517
|
config.agent = {
|
|
1420
1518
|
...config.agent,
|
|
1421
1519
|
agent
|
|
@@ -1434,8 +1532,7 @@ function runInit(ctx, args) {
|
|
|
1434
1532
|
from: flagValue(args, "--from"),
|
|
1435
1533
|
agent: flagValue(args, "--agent"),
|
|
1436
1534
|
name,
|
|
1437
|
-
colour: flagValue(args, "--colour") ?? flagValue(args, "--color")
|
|
1438
|
-
platform: process.platform
|
|
1535
|
+
colour: flagValue(args, "--colour") ?? flagValue(args, "--color")
|
|
1439
1536
|
});
|
|
1440
1537
|
} catch (error) {
|
|
1441
1538
|
ctx.err.write(`${error instanceof Error ? error.message : String(error)}\n`);
|
|
@@ -1451,7 +1548,8 @@ function runInit(ctx, args) {
|
|
|
1451
1548
|
opencharm sim no charm yet? it appears in your browser (another terminal)
|
|
1452
1549
|
opencharm pair <code> the code on the charm (or the emulator); you choose its PIN
|
|
1453
1550
|
|
|
1454
|
-
The charm is called ${charm}: its name, colour and greeting are in opencharm.json ("charm").
|
|
1551
|
+
The charm is called ${charm}: its name, colour and greeting are in opencharm.json ("charm").
|
|
1552
|
+
${ctx.style.dim("Voice: it listens on this computer; it speaks with Microsoft's free voices, so the text of each spoken reply goes to Microsoft. To keep everything here, set \"speak\": { \"provider\": \"local\" } in opencharm.json. The first start downloads the voice models (about 620 MB): opencharm voice.")}
|
|
1455
1553
|
`);
|
|
1456
1554
|
}
|
|
1457
1555
|
//#endregion
|
|
@@ -1731,7 +1829,7 @@ function parseInput(schema, input) {
|
|
|
1731
1829
|
if (!result.success) throw new Error(result.error.issues.map((i) => i.message).join("; "));
|
|
1732
1830
|
return result.data;
|
|
1733
1831
|
}
|
|
1734
|
-
function createAdminActions({ store, pairing, sessions, look, agentCanChangeLook, identity, starter }) {
|
|
1832
|
+
function createAdminActions({ store, pairing, sessions, look, agentCanChangeLook, identity, starter, voice }) {
|
|
1735
1833
|
function sessionsOf(charmId) {
|
|
1736
1834
|
return [...sessions()].filter((s) => s.charmId === charmId && s.state !== "closed");
|
|
1737
1835
|
}
|
|
@@ -1840,6 +1938,7 @@ function createAdminActions({ store, pairing, sessions, look, agentCanChangeLook
|
|
|
1840
1938
|
return {
|
|
1841
1939
|
charmd: identity ?? "unknown",
|
|
1842
1940
|
...starter ? { starter } : {},
|
|
1941
|
+
...voice ? { voice: voice() } : {},
|
|
1843
1942
|
charms: store.list().map((charm) => {
|
|
1844
1943
|
const live = sessionsOf(charm.id).at(-1);
|
|
1845
1944
|
return {
|
|
@@ -2185,6 +2284,7 @@ var Session = class {
|
|
|
2185
2284
|
type: "charm",
|
|
2186
2285
|
op: "unlocked"
|
|
2187
2286
|
});
|
|
2287
|
+
this.#activeTurn()?.warm();
|
|
2188
2288
|
}
|
|
2189
2289
|
#issuePairCode() {
|
|
2190
2290
|
try {
|
|
@@ -2657,6 +2757,449 @@ function readOggOpus(data) {
|
|
|
2657
2757
|
};
|
|
2658
2758
|
}
|
|
2659
2759
|
//#endregion
|
|
2760
|
+
//#region ../charmd/src/voice/language.ts
|
|
2761
|
+
const COMMON_WORDS = {
|
|
2762
|
+
en: [
|
|
2763
|
+
"the",
|
|
2764
|
+
"and",
|
|
2765
|
+
"is",
|
|
2766
|
+
"are",
|
|
2767
|
+
"you",
|
|
2768
|
+
"to",
|
|
2769
|
+
"of",
|
|
2770
|
+
"it",
|
|
2771
|
+
"that",
|
|
2772
|
+
"what",
|
|
2773
|
+
"this",
|
|
2774
|
+
"for",
|
|
2775
|
+
"with",
|
|
2776
|
+
"have",
|
|
2777
|
+
"i",
|
|
2778
|
+
"my",
|
|
2779
|
+
"your",
|
|
2780
|
+
"do",
|
|
2781
|
+
"can",
|
|
2782
|
+
"on",
|
|
2783
|
+
"in",
|
|
2784
|
+
"be",
|
|
2785
|
+
"was",
|
|
2786
|
+
"will",
|
|
2787
|
+
"not",
|
|
2788
|
+
"me",
|
|
2789
|
+
"please",
|
|
2790
|
+
"tomorrow",
|
|
2791
|
+
"today"
|
|
2792
|
+
],
|
|
2793
|
+
it: [
|
|
2794
|
+
"il",
|
|
2795
|
+
"lo",
|
|
2796
|
+
"la",
|
|
2797
|
+
"gli",
|
|
2798
|
+
"le",
|
|
2799
|
+
"di",
|
|
2800
|
+
"che",
|
|
2801
|
+
"è",
|
|
2802
|
+
"e",
|
|
2803
|
+
"un",
|
|
2804
|
+
"una",
|
|
2805
|
+
"per",
|
|
2806
|
+
"con",
|
|
2807
|
+
"non",
|
|
2808
|
+
"sono",
|
|
2809
|
+
"mi",
|
|
2810
|
+
"ti",
|
|
2811
|
+
"ho",
|
|
2812
|
+
"hai",
|
|
2813
|
+
"cosa",
|
|
2814
|
+
"come",
|
|
2815
|
+
"del",
|
|
2816
|
+
"della",
|
|
2817
|
+
"dei",
|
|
2818
|
+
"delle",
|
|
2819
|
+
"nel",
|
|
2820
|
+
"nella",
|
|
2821
|
+
"fai",
|
|
2822
|
+
"oggi",
|
|
2823
|
+
"domani",
|
|
2824
|
+
"anche",
|
|
2825
|
+
"questo",
|
|
2826
|
+
"quello",
|
|
2827
|
+
"sì",
|
|
2828
|
+
"grazie"
|
|
2829
|
+
],
|
|
2830
|
+
es: [
|
|
2831
|
+
"el",
|
|
2832
|
+
"los",
|
|
2833
|
+
"las",
|
|
2834
|
+
"de",
|
|
2835
|
+
"que",
|
|
2836
|
+
"es",
|
|
2837
|
+
"y",
|
|
2838
|
+
"un",
|
|
2839
|
+
"una",
|
|
2840
|
+
"por",
|
|
2841
|
+
"con",
|
|
2842
|
+
"no",
|
|
2843
|
+
"para",
|
|
2844
|
+
"lo",
|
|
2845
|
+
"mi",
|
|
2846
|
+
"tu",
|
|
2847
|
+
"qué",
|
|
2848
|
+
"cómo",
|
|
2849
|
+
"está",
|
|
2850
|
+
"son",
|
|
2851
|
+
"hoy",
|
|
2852
|
+
"mañana",
|
|
2853
|
+
"gracias",
|
|
2854
|
+
"pero",
|
|
2855
|
+
"muy",
|
|
2856
|
+
"también"
|
|
2857
|
+
],
|
|
2858
|
+
fr: [
|
|
2859
|
+
"le",
|
|
2860
|
+
"les",
|
|
2861
|
+
"des",
|
|
2862
|
+
"de",
|
|
2863
|
+
"et",
|
|
2864
|
+
"est",
|
|
2865
|
+
"un",
|
|
2866
|
+
"une",
|
|
2867
|
+
"pour",
|
|
2868
|
+
"avec",
|
|
2869
|
+
"pas",
|
|
2870
|
+
"je",
|
|
2871
|
+
"tu",
|
|
2872
|
+
"vous",
|
|
2873
|
+
"nous",
|
|
2874
|
+
"que",
|
|
2875
|
+
"qui",
|
|
2876
|
+
"ce",
|
|
2877
|
+
"cette",
|
|
2878
|
+
"dans",
|
|
2879
|
+
"sur",
|
|
2880
|
+
"aujourd'hui",
|
|
2881
|
+
"demain",
|
|
2882
|
+
"merci",
|
|
2883
|
+
"oui",
|
|
2884
|
+
"mais",
|
|
2885
|
+
"très"
|
|
2886
|
+
],
|
|
2887
|
+
de: [
|
|
2888
|
+
"der",
|
|
2889
|
+
"die",
|
|
2890
|
+
"das",
|
|
2891
|
+
"und",
|
|
2892
|
+
"ist",
|
|
2893
|
+
"ein",
|
|
2894
|
+
"eine",
|
|
2895
|
+
"nicht",
|
|
2896
|
+
"ich",
|
|
2897
|
+
"du",
|
|
2898
|
+
"sie",
|
|
2899
|
+
"wir",
|
|
2900
|
+
"mit",
|
|
2901
|
+
"für",
|
|
2902
|
+
"auf",
|
|
2903
|
+
"zu",
|
|
2904
|
+
"den",
|
|
2905
|
+
"dem",
|
|
2906
|
+
"was",
|
|
2907
|
+
"wie",
|
|
2908
|
+
"heute",
|
|
2909
|
+
"morgen",
|
|
2910
|
+
"danke",
|
|
2911
|
+
"ja",
|
|
2912
|
+
"auch",
|
|
2913
|
+
"bitte"
|
|
2914
|
+
],
|
|
2915
|
+
pt: [
|
|
2916
|
+
"o",
|
|
2917
|
+
"os",
|
|
2918
|
+
"as",
|
|
2919
|
+
"de",
|
|
2920
|
+
"que",
|
|
2921
|
+
"é",
|
|
2922
|
+
"e",
|
|
2923
|
+
"um",
|
|
2924
|
+
"uma",
|
|
2925
|
+
"para",
|
|
2926
|
+
"com",
|
|
2927
|
+
"não",
|
|
2928
|
+
"eu",
|
|
2929
|
+
"você",
|
|
2930
|
+
"do",
|
|
2931
|
+
"da",
|
|
2932
|
+
"dos",
|
|
2933
|
+
"das",
|
|
2934
|
+
"no",
|
|
2935
|
+
"na",
|
|
2936
|
+
"hoje",
|
|
2937
|
+
"amanhã",
|
|
2938
|
+
"obrigado",
|
|
2939
|
+
"obrigada",
|
|
2940
|
+
"sim",
|
|
2941
|
+
"também",
|
|
2942
|
+
"muito"
|
|
2943
|
+
]
|
|
2944
|
+
};
|
|
2945
|
+
const LANGUAGES = Object.keys(COMMON_WORDS);
|
|
2946
|
+
const WORDS = new Map(LANGUAGES.map((language) => [language, new Set(COMMON_WORDS[language])]));
|
|
2947
|
+
function guessLanguage(text, among = LANGUAGES) {
|
|
2948
|
+
const words = text.toLowerCase().match(/[\p{L}]+(?:'(?:hui))?/gu) ?? [];
|
|
2949
|
+
const [best, second] = among.map((language) => ({
|
|
2950
|
+
language,
|
|
2951
|
+
score: words.filter((word) => WORDS.get(language)?.has(word)).length
|
|
2952
|
+
})).sort((a, b) => b.score - a.score);
|
|
2953
|
+
if (!best || best.score < 2) return void 0;
|
|
2954
|
+
return best.score > (second?.score ?? 0) ? best.language : void 0;
|
|
2955
|
+
}
|
|
2956
|
+
//#endregion
|
|
2957
|
+
//#region ../charmd/src/voice/models.ts
|
|
2958
|
+
const RELEASES = "https://github.com/k2-fsa/sherpa-onnx/releases/download";
|
|
2959
|
+
const MODELS = {
|
|
2960
|
+
parakeet: {
|
|
2961
|
+
label: "Parakeet (listening)",
|
|
2962
|
+
url: `${RELEASES}/asr-models/sherpa-onnx-nemo-parakeet-tdt-0.6b-v3-int8.tar.bz2`,
|
|
2963
|
+
sha256: "5793d0fd397c5778d2cf2126994d58e9d56b1be7c04d13c7a15bb1b4eafb16bf",
|
|
2964
|
+
bytes: 487170055,
|
|
2965
|
+
folder: "sherpa-onnx-nemo-parakeet-tdt-0.6b-v3-int8",
|
|
2966
|
+
files: [
|
|
2967
|
+
"encoder.int8.onnx",
|
|
2968
|
+
"decoder.int8.onnx",
|
|
2969
|
+
"joiner.int8.onnx",
|
|
2970
|
+
"tokens.txt"
|
|
2971
|
+
]
|
|
2972
|
+
},
|
|
2973
|
+
supertonic: {
|
|
2974
|
+
label: "Supertonic (speaking)",
|
|
2975
|
+
url: `${RELEASES}/tts-models/sherpa-onnx-supertonic-3-tts-int8-2026-05-11.tar.bz2`,
|
|
2976
|
+
sha256: "82fa96f91c4ef8abaae3a14a3f4153facf88bed821d1f7331cec2700f432c427",
|
|
2977
|
+
bytes: 128774318,
|
|
2978
|
+
folder: "sherpa-onnx-supertonic-3-tts-int8-2026-05-11",
|
|
2979
|
+
files: [
|
|
2980
|
+
"duration_predictor.int8.onnx",
|
|
2981
|
+
"text_encoder.int8.onnx",
|
|
2982
|
+
"vector_estimator.int8.onnx",
|
|
2983
|
+
"vocoder.int8.onnx",
|
|
2984
|
+
"tts.json",
|
|
2985
|
+
"unicode_indexer.bin",
|
|
2986
|
+
"voice.bin"
|
|
2987
|
+
]
|
|
2988
|
+
}
|
|
2989
|
+
};
|
|
2990
|
+
var VoiceNotReady = class extends Error {};
|
|
2991
|
+
const ABANDONED_MS = 36e5;
|
|
2992
|
+
const states = /* @__PURE__ */ new Map();
|
|
2993
|
+
const installing = /* @__PURE__ */ new Map();
|
|
2994
|
+
function defaultModelsDir() {
|
|
2995
|
+
return join(homedir(), ".opencharm", "models");
|
|
2996
|
+
}
|
|
2997
|
+
function isInstalled(spec, dir) {
|
|
2998
|
+
return spec.files.every((file) => existsSync(join(dir, spec.folder, file)));
|
|
2999
|
+
}
|
|
3000
|
+
function untar(archive, into) {
|
|
3001
|
+
return new Promise((resolve, reject) => {
|
|
3002
|
+
execFile("tar", [
|
|
3003
|
+
"-xjf",
|
|
3004
|
+
archive,
|
|
3005
|
+
"-C",
|
|
3006
|
+
into
|
|
3007
|
+
], (error, _out, stderr) => {
|
|
3008
|
+
if (!error) resolve();
|
|
3009
|
+
else reject(/* @__PURE__ */ new Error(`Couldn't unpack the model: ${stderr.trim().slice(0, 200) || error.message}`));
|
|
3010
|
+
});
|
|
3011
|
+
});
|
|
3012
|
+
}
|
|
3013
|
+
async function download(spec, archive, options) {
|
|
3014
|
+
const response = await fetch(spec.url, { signal: options.signal ?? null });
|
|
3015
|
+
if (!response.ok || !response.body) throw new Error(`Couldn't download ${spec.label}: ${response.status} ${response.statusText}`);
|
|
3016
|
+
const total = Number(response.headers.get("content-length")) || spec.bytes;
|
|
3017
|
+
const hash = createHash("sha256");
|
|
3018
|
+
let received = 0;
|
|
3019
|
+
const body = Readable.fromWeb(response.body);
|
|
3020
|
+
body.on("data", (chunk) => {
|
|
3021
|
+
hash.update(chunk);
|
|
3022
|
+
received += chunk.length;
|
|
3023
|
+
if (spec.bytes && received > spec.bytes) body.destroy(/* @__PURE__ */ new Error(`${spec.label} was bigger than expected, so it was discarded`));
|
|
3024
|
+
else options.onProgress?.(received, total);
|
|
3025
|
+
});
|
|
3026
|
+
await pipeline(body, createWriteStream(archive, { mode: 384 }), { ...options.signal ? { signal: options.signal } : {} });
|
|
3027
|
+
if (spec.bytes && received !== spec.bytes || hash.digest("hex") !== spec.sha256) throw new Error(`${spec.label} didn't match its pinned checksum, so it was discarded`);
|
|
3028
|
+
}
|
|
3029
|
+
function removeAbandoned(dir, folder) {
|
|
3030
|
+
for (const name of readdirSync(dir)) {
|
|
3031
|
+
if (!name.startsWith(`.download-${folder}-`)) continue;
|
|
3032
|
+
const path = join(dir, name);
|
|
3033
|
+
try {
|
|
3034
|
+
if (Date.now() - statSync(path).mtimeMs > ABANDONED_MS) rmSync(path, {
|
|
3035
|
+
recursive: true,
|
|
3036
|
+
force: true
|
|
3037
|
+
});
|
|
3038
|
+
} catch {}
|
|
3039
|
+
}
|
|
3040
|
+
}
|
|
3041
|
+
async function install(spec, dir, options) {
|
|
3042
|
+
mkdirSync(dir, { recursive: true });
|
|
3043
|
+
removeAbandoned(dir, spec.folder);
|
|
3044
|
+
const temp = join(dir, `.download-${spec.folder}-${randomBytes(6).toString("hex")}`);
|
|
3045
|
+
mkdirSync(temp);
|
|
3046
|
+
try {
|
|
3047
|
+
const archive = join(temp, "model.tar.bz2");
|
|
3048
|
+
await download(spec, archive, options);
|
|
3049
|
+
const unpacked = join(temp, "unpacked");
|
|
3050
|
+
mkdirSync(unpacked);
|
|
3051
|
+
await untar(archive, unpacked);
|
|
3052
|
+
if (!isInstalled(spec, unpacked)) throw new Error(`${spec.label} is missing files`);
|
|
3053
|
+
rmSync(join(dir, spec.folder), {
|
|
3054
|
+
recursive: true,
|
|
3055
|
+
force: true
|
|
3056
|
+
});
|
|
3057
|
+
renameSync(join(unpacked, spec.folder), join(dir, spec.folder));
|
|
3058
|
+
return join(dir, spec.folder);
|
|
3059
|
+
} finally {
|
|
3060
|
+
rmSync(temp, {
|
|
3061
|
+
recursive: true,
|
|
3062
|
+
force: true
|
|
3063
|
+
});
|
|
3064
|
+
}
|
|
3065
|
+
}
|
|
3066
|
+
function modelState(id, dir = defaultModelsDir()) {
|
|
3067
|
+
if (isInstalled(MODELS[id], dir)) return { state: "ready" };
|
|
3068
|
+
return states.get(join(dir, id)) ?? { state: "missing" };
|
|
3069
|
+
}
|
|
3070
|
+
function installModel(id, options = {}) {
|
|
3071
|
+
const spec = options.spec ?? MODELS[id];
|
|
3072
|
+
const dir = options.dir ?? defaultModelsDir();
|
|
3073
|
+
if (isInstalled(spec, dir)) return Promise.resolve(join(dir, spec.folder));
|
|
3074
|
+
const key = join(dir, id);
|
|
3075
|
+
const running = installing.get(key);
|
|
3076
|
+
if (running) return running;
|
|
3077
|
+
states.set(key, {
|
|
3078
|
+
state: "downloading",
|
|
3079
|
+
received: 0,
|
|
3080
|
+
total: spec.bytes
|
|
3081
|
+
});
|
|
3082
|
+
const work = install(spec, dir, {
|
|
3083
|
+
...options,
|
|
3084
|
+
onProgress: (received, total) => {
|
|
3085
|
+
states.set(key, {
|
|
3086
|
+
state: "downloading",
|
|
3087
|
+
received,
|
|
3088
|
+
total
|
|
3089
|
+
});
|
|
3090
|
+
options.onProgress?.(received, total);
|
|
3091
|
+
}
|
|
3092
|
+
}).then((path) => {
|
|
3093
|
+
states.set(key, { state: "ready" });
|
|
3094
|
+
return path;
|
|
3095
|
+
}).catch((error) => {
|
|
3096
|
+
states.set(key, {
|
|
3097
|
+
state: "failed",
|
|
3098
|
+
error: error instanceof Error ? error.message : String(error),
|
|
3099
|
+
at: Date.now()
|
|
3100
|
+
});
|
|
3101
|
+
throw error;
|
|
3102
|
+
}).finally(() => installing.delete(key));
|
|
3103
|
+
installing.set(key, work);
|
|
3104
|
+
return work;
|
|
3105
|
+
}
|
|
3106
|
+
//#endregion
|
|
3107
|
+
//#region ../charmd/src/voice/packets.ts
|
|
3108
|
+
function asError(problem) {
|
|
3109
|
+
if (problem instanceof Error) return problem;
|
|
3110
|
+
return new Error(typeof problem === "string" ? problem : "The voice failed");
|
|
3111
|
+
}
|
|
3112
|
+
var PacketStream = class {
|
|
3113
|
+
#packets = [];
|
|
3114
|
+
#done = false;
|
|
3115
|
+
#error;
|
|
3116
|
+
#waiters = [];
|
|
3117
|
+
constructor(source) {
|
|
3118
|
+
this.#pump(source);
|
|
3119
|
+
}
|
|
3120
|
+
async #pump(source) {
|
|
3121
|
+
try {
|
|
3122
|
+
for await (const packet of source) {
|
|
3123
|
+
this.#packets.push(packet);
|
|
3124
|
+
this.#wake();
|
|
3125
|
+
}
|
|
3126
|
+
} catch (error) {
|
|
3127
|
+
this.#error = asError(error);
|
|
3128
|
+
} finally {
|
|
3129
|
+
this.#done = true;
|
|
3130
|
+
this.#wake();
|
|
3131
|
+
}
|
|
3132
|
+
}
|
|
3133
|
+
#wake() {
|
|
3134
|
+
for (const waiter of this.#waiters.splice(0)) waiter();
|
|
3135
|
+
}
|
|
3136
|
+
#next() {
|
|
3137
|
+
return new Promise((resolve) => this.#waiters.push(resolve));
|
|
3138
|
+
}
|
|
3139
|
+
async first() {
|
|
3140
|
+
while (this.#packets.length === 0) {
|
|
3141
|
+
if (this.#error !== void 0) throw this.#error;
|
|
3142
|
+
if (this.#done) throw new Error("The voice produced no audio");
|
|
3143
|
+
await this.#next();
|
|
3144
|
+
}
|
|
3145
|
+
}
|
|
3146
|
+
async *[Symbol.asyncIterator]() {
|
|
3147
|
+
let index = 0;
|
|
3148
|
+
for (;;) {
|
|
3149
|
+
if (index < this.#packets.length) {
|
|
3150
|
+
yield this.#packets[index++];
|
|
3151
|
+
continue;
|
|
3152
|
+
}
|
|
3153
|
+
if (this.#error !== void 0) throw this.#error;
|
|
3154
|
+
if (this.#done) return;
|
|
3155
|
+
await this.#next();
|
|
3156
|
+
}
|
|
3157
|
+
}
|
|
3158
|
+
};
|
|
3159
|
+
function packetsOf(speaker, text, signal, language) {
|
|
3160
|
+
if (speaker.stream) return speaker.stream(text, signal, language);
|
|
3161
|
+
return (async function* () {
|
|
3162
|
+
yield* readOggOpus(await speaker.synthesize(text, signal, language)).packets;
|
|
3163
|
+
})();
|
|
3164
|
+
}
|
|
3165
|
+
function createChannel() {
|
|
3166
|
+
const items = [];
|
|
3167
|
+
let done = false;
|
|
3168
|
+
let error;
|
|
3169
|
+
let waiter;
|
|
3170
|
+
const wake = () => {
|
|
3171
|
+
waiter?.();
|
|
3172
|
+
waiter = void 0;
|
|
3173
|
+
};
|
|
3174
|
+
return {
|
|
3175
|
+
push(item) {
|
|
3176
|
+
items.push(item);
|
|
3177
|
+
wake();
|
|
3178
|
+
},
|
|
3179
|
+
end() {
|
|
3180
|
+
done = true;
|
|
3181
|
+
wake();
|
|
3182
|
+
},
|
|
3183
|
+
fail(problem) {
|
|
3184
|
+
error = asError(problem);
|
|
3185
|
+
done = true;
|
|
3186
|
+
wake();
|
|
3187
|
+
},
|
|
3188
|
+
async *[Symbol.asyncIterator]() {
|
|
3189
|
+
for (;;) {
|
|
3190
|
+
const item = items.shift();
|
|
3191
|
+
if (item !== void 0) {
|
|
3192
|
+
yield item;
|
|
3193
|
+
continue;
|
|
3194
|
+
}
|
|
3195
|
+
if (error !== void 0) throw error;
|
|
3196
|
+
if (done) return;
|
|
3197
|
+
await new Promise((resolve) => waiter = resolve);
|
|
3198
|
+
}
|
|
3199
|
+
}
|
|
3200
|
+
};
|
|
3201
|
+
}
|
|
3202
|
+
//#endregion
|
|
2660
3203
|
//#region ../charmd/src/turn/sentences.ts
|
|
2661
3204
|
const MIN_CHARS = 10;
|
|
2662
3205
|
const ABBREVIATIONS = /* @__PURE__ */ new Set([
|
|
@@ -2671,9 +3214,12 @@ const ABBREVIATIONS = /* @__PURE__ */ new Set([
|
|
|
2671
3214
|
"no."
|
|
2672
3215
|
]);
|
|
2673
3216
|
const BOUNDARY = /[.!?…]+(?=\s)|\n+/g;
|
|
3217
|
+
const FIRST_BOUNDARY = /[.!?…]+(?=\s)|[,;:](?=\s)|\n+/g;
|
|
3218
|
+
const FIRST_CLAUSE_CHARS = 20;
|
|
2674
3219
|
var SentenceSplitter = class {
|
|
2675
3220
|
#buffer = "";
|
|
2676
3221
|
#pending = "";
|
|
3222
|
+
#started = false;
|
|
2677
3223
|
#take(piece) {
|
|
2678
3224
|
const text = piece.replace(/\s+/g, " ").trim();
|
|
2679
3225
|
if (!text) return [];
|
|
@@ -2681,14 +3227,16 @@ var SentenceSplitter = class {
|
|
|
2681
3227
|
if (this.#pending.length < MIN_CHARS) return [];
|
|
2682
3228
|
const out = this.#pending;
|
|
2683
3229
|
this.#pending = "";
|
|
3230
|
+
this.#started = true;
|
|
2684
3231
|
return [out];
|
|
2685
3232
|
}
|
|
2686
3233
|
push(chunk) {
|
|
2687
3234
|
this.#buffer += chunk;
|
|
2688
3235
|
const out = [];
|
|
2689
3236
|
let start = 0;
|
|
2690
|
-
for (const match of this.#buffer.matchAll(BOUNDARY)) {
|
|
3237
|
+
for (const match of this.#buffer.matchAll(this.#started ? BOUNDARY : FIRST_BOUNDARY)) {
|
|
2691
3238
|
const end = match.index + match[0].length;
|
|
3239
|
+
if (/^[,;:]/.test(match[0]) && (this.#started || this.#pending.length + end - start < FIRST_CLAUSE_CHARS)) continue;
|
|
2692
3240
|
const lastWord = this.#buffer.slice(start, end).trim().split(/\s+/).at(-1)?.toLowerCase() ?? "";
|
|
2693
3241
|
if (match[0].startsWith(".") && ABBREVIATIONS.has(lastWord)) continue;
|
|
2694
3242
|
out.push(...this.#take(this.#buffer.slice(start, end)));
|
|
@@ -2711,12 +3259,13 @@ function cleanForSpeech(text) {
|
|
|
2711
3259
|
//#endregion
|
|
2712
3260
|
//#region ../charmd/src/turn/turn.ts
|
|
2713
3261
|
const MAX_FRAMES = 1e3;
|
|
3262
|
+
const LOOKAHEAD = 2;
|
|
2714
3263
|
const AGENT_LABELS = { ...Object.fromEntries(Object.entries(ACP_AGENTS).map(([name, agent]) => [name, agent.label])) };
|
|
2715
3264
|
var TurnTimeout = class extends Error {};
|
|
2716
3265
|
var StageError = class extends Error {
|
|
2717
3266
|
stage;
|
|
2718
3267
|
constructor(stage, cause) {
|
|
2719
|
-
super(cause instanceof Error ? cause.message : String(cause));
|
|
3268
|
+
super(cause instanceof Error ? cause.message : String(cause), { cause });
|
|
2720
3269
|
this.stage = stage;
|
|
2721
3270
|
}
|
|
2722
3271
|
};
|
|
@@ -2778,6 +3327,10 @@ var TurnController = class {
|
|
|
2778
3327
|
});
|
|
2779
3328
|
this.#speaking = false;
|
|
2780
3329
|
}
|
|
3330
|
+
warm() {
|
|
3331
|
+
this.#deps.agent.warm?.(this.#deps.sessionKey);
|
|
3332
|
+
this.#deps.voice.warm?.().catch(() => void 0);
|
|
3333
|
+
}
|
|
2781
3334
|
listenStart() {
|
|
2782
3335
|
if (this.#state === "thinking" || this.#state === "speaking") this.#cancel();
|
|
2783
3336
|
this.#state = "listening";
|
|
@@ -2819,6 +3372,21 @@ var TurnController = class {
|
|
|
2819
3372
|
let outcome = "done";
|
|
2820
3373
|
const startClock = () => setTimeout(() => controller.abort(new TurnTimeout()), this.#deps.timeoutMs);
|
|
2821
3374
|
let timer = startClock();
|
|
3375
|
+
let speakingStarted = false;
|
|
3376
|
+
let asking = false;
|
|
3377
|
+
const restartClock = () => {
|
|
3378
|
+
clearTimeout(timer);
|
|
3379
|
+
timer = startClock();
|
|
3380
|
+
};
|
|
3381
|
+
const clock = {
|
|
3382
|
+
started: () => {
|
|
3383
|
+
speakingStarted = true;
|
|
3384
|
+
restartClock();
|
|
3385
|
+
},
|
|
3386
|
+
touch: () => {
|
|
3387
|
+
if (speakingStarted && !asking) restartClock();
|
|
3388
|
+
}
|
|
3389
|
+
};
|
|
2822
3390
|
const timeoutGuard = new Promise((_, reject) => signal.addEventListener("abort", () => {
|
|
2823
3391
|
if (signal.reason instanceof TurnTimeout) reject(signal.reason);
|
|
2824
3392
|
}));
|
|
@@ -2843,6 +3411,7 @@ var TurnController = class {
|
|
|
2843
3411
|
const ask = this.#deps.ask;
|
|
2844
3412
|
const label = AGENT_LABELS[this.#deps.agent.name] ?? "Your agent";
|
|
2845
3413
|
if (ask) this.#asking = async (text, options) => {
|
|
3414
|
+
asking = true;
|
|
2846
3415
|
clearTimeout(timer);
|
|
2847
3416
|
try {
|
|
2848
3417
|
return await ask(text, {
|
|
@@ -2850,6 +3419,7 @@ var TurnController = class {
|
|
|
2850
3419
|
signal
|
|
2851
3420
|
});
|
|
2852
3421
|
} finally {
|
|
3422
|
+
asking = false;
|
|
2853
3423
|
if (!signal.aborted) timer = startClock();
|
|
2854
3424
|
}
|
|
2855
3425
|
};
|
|
@@ -2859,7 +3429,7 @@ var TurnController = class {
|
|
|
2859
3429
|
signal,
|
|
2860
3430
|
...ask ? { ask: (action) => this.ask(`${label} wants to ${action}.`) } : {}
|
|
2861
3431
|
});
|
|
2862
|
-
said = await this.#speak(reply, signal, live, timings, started, timeoutGuard);
|
|
3432
|
+
said = await this.#speak(reply, signal, live, timings, started, timeoutGuard, guessLanguage(heard) ?? this.#deps.voice.language ?? "en", clock);
|
|
2863
3433
|
if (!live()) return;
|
|
2864
3434
|
if (this.#speaking) this.#deps.send({
|
|
2865
3435
|
type: "tts",
|
|
@@ -2871,6 +3441,7 @@ var TurnController = class {
|
|
|
2871
3441
|
this.#endFace = void 0;
|
|
2872
3442
|
this.#face(end.state, end.text);
|
|
2873
3443
|
} catch (error) {
|
|
3444
|
+
if (!signal.aborted) controller.abort(error);
|
|
2874
3445
|
outcome = error instanceof TurnTimeout ? "timeout" : error instanceof StageError ? `${error.stage}-error` : "error";
|
|
2875
3446
|
if (!live()) return;
|
|
2876
3447
|
if (this.#speaking) this.#deps.send({
|
|
@@ -2900,9 +3471,18 @@ var TurnController = class {
|
|
|
2900
3471
|
});
|
|
2901
3472
|
}
|
|
2902
3473
|
}
|
|
2903
|
-
async #speak(source, signal, live, timings, started, timeoutGuard) {
|
|
3474
|
+
async #speak(source, signal, live, timings, started, timeoutGuard, language, clock) {
|
|
2904
3475
|
const now = this.#deps.now ?? Date.now;
|
|
3476
|
+
const going = () => live() && !signal.aborted;
|
|
3477
|
+
let speaking = language;
|
|
2905
3478
|
const queue = [];
|
|
3479
|
+
let playing = 0;
|
|
3480
|
+
const startUpTo = (end) => {
|
|
3481
|
+
for (let i = playing; i < Math.min(end, queue.length); i++) {
|
|
3482
|
+
const item = queue[i];
|
|
3483
|
+
item.packets ??= new PacketStream(packetsOf(this.#deps.voice, item.text, signal, item.language));
|
|
3484
|
+
}
|
|
3485
|
+
};
|
|
2906
3486
|
let wake = () => void 0;
|
|
2907
3487
|
let sourceDone = false;
|
|
2908
3488
|
let said = "";
|
|
@@ -2910,19 +3490,23 @@ var TurnController = class {
|
|
|
2910
3490
|
const text = cleanForSpeech(sentence);
|
|
2911
3491
|
if (!text) return;
|
|
2912
3492
|
said += `${text} `;
|
|
2913
|
-
|
|
3493
|
+
speaking = guessLanguage(text) ?? speaking;
|
|
3494
|
+
const sentenceLanguage = speaking;
|
|
3495
|
+
timings.firstSentenceMs ??= now() - started;
|
|
3496
|
+
queue.push({
|
|
2914
3497
|
text,
|
|
2915
|
-
|
|
2916
|
-
})
|
|
2917
|
-
|
|
2918
|
-
queue.push(pending);
|
|
3498
|
+
language: sentenceLanguage
|
|
3499
|
+
});
|
|
3500
|
+
startUpTo(playing + LOOKAHEAD);
|
|
2919
3501
|
wake();
|
|
2920
3502
|
};
|
|
2921
3503
|
const produce = (async () => {
|
|
2922
3504
|
const splitter = new SentenceSplitter();
|
|
2923
3505
|
await inStage("agent", async () => {
|
|
2924
3506
|
for await (const chunk of source) {
|
|
2925
|
-
if (!
|
|
3507
|
+
if (!going()) return;
|
|
3508
|
+
timings.agentFirstMs ??= now() - started;
|
|
3509
|
+
clock.touch();
|
|
2926
3510
|
for (const sentence of splitter.push(chunk)) enqueue(sentence);
|
|
2927
3511
|
}
|
|
2928
3512
|
});
|
|
@@ -2933,19 +3517,23 @@ var TurnController = class {
|
|
|
2933
3517
|
});
|
|
2934
3518
|
produce.catch(() => void 0);
|
|
2935
3519
|
const play = async () => {
|
|
2936
|
-
let index = 0;
|
|
2937
3520
|
for (;;) {
|
|
2938
|
-
if (
|
|
3521
|
+
if (!going()) return;
|
|
3522
|
+
if (playing >= queue.length) {
|
|
2939
3523
|
if (sourceDone) return;
|
|
2940
3524
|
await new Promise((resolve) => wake = resolve);
|
|
2941
3525
|
continue;
|
|
2942
3526
|
}
|
|
2943
|
-
|
|
2944
|
-
|
|
3527
|
+
startUpTo(playing + LOOKAHEAD);
|
|
3528
|
+
const { text, packets } = queue[playing];
|
|
3529
|
+
if (!packets) throw new Error("A sentence wasn't started");
|
|
3530
|
+
await Promise.race([inStage("tts", () => packets.first()), timeoutGuard]);
|
|
3531
|
+
if (!going()) return;
|
|
2945
3532
|
if (!this.#speaking) {
|
|
2946
3533
|
this.#speaking = true;
|
|
2947
3534
|
this.#state = "speaking";
|
|
2948
3535
|
timings.firstAudioMs = now() - started;
|
|
3536
|
+
clock.started();
|
|
2949
3537
|
this.#deps.send({
|
|
2950
3538
|
type: "tts",
|
|
2951
3539
|
state: "start"
|
|
@@ -2956,7 +3544,8 @@ var TurnController = class {
|
|
|
2956
3544
|
state: "sentence_start",
|
|
2957
3545
|
text
|
|
2958
3546
|
});
|
|
2959
|
-
await this.#sendPaced(
|
|
3547
|
+
await Promise.race([inStage("tts", () => this.#sendPaced(packets, going, clock.touch)), timeoutGuard]);
|
|
3548
|
+
playing += 1;
|
|
2960
3549
|
}
|
|
2961
3550
|
};
|
|
2962
3551
|
await Promise.all([Promise.race([produce, timeoutGuard]), play()]);
|
|
@@ -2968,7 +3557,16 @@ var TurnController = class {
|
|
|
2968
3557
|
const live = () => this.#generation === generation;
|
|
2969
3558
|
const controller = new AbortController();
|
|
2970
3559
|
this.#controller = controller;
|
|
2971
|
-
const
|
|
3560
|
+
const startClock = () => setTimeout(() => controller.abort(new TurnTimeout()), this.#deps.timeoutMs);
|
|
3561
|
+
let timer = startClock();
|
|
3562
|
+
const restartClock = () => {
|
|
3563
|
+
clearTimeout(timer);
|
|
3564
|
+
timer = startClock();
|
|
3565
|
+
};
|
|
3566
|
+
const clock = {
|
|
3567
|
+
started: restartClock,
|
|
3568
|
+
touch: restartClock
|
|
3569
|
+
};
|
|
2972
3570
|
const timeoutGuard = new Promise((_, reject) => controller.signal.addEventListener("abort", () => {
|
|
2973
3571
|
if (controller.signal.reason instanceof TurnTimeout) reject(controller.signal.reason);
|
|
2974
3572
|
}));
|
|
@@ -2977,7 +3575,7 @@ var TurnController = class {
|
|
|
2977
3575
|
yield await Promise.resolve(text);
|
|
2978
3576
|
}
|
|
2979
3577
|
try {
|
|
2980
|
-
await this.#speak(once(), controller.signal, live, {}, Date.now(), timeoutGuard);
|
|
3578
|
+
await this.#speak(once(), controller.signal, live, {}, Date.now(), timeoutGuard, guessLanguage(text) ?? this.#deps.voice.language ?? "en", clock);
|
|
2981
3579
|
if (!live()) return;
|
|
2982
3580
|
if (this.#speaking) this.#deps.send({
|
|
2983
3581
|
type: "tts",
|
|
@@ -2987,6 +3585,7 @@ var TurnController = class {
|
|
|
2987
3585
|
this.#state = "idle";
|
|
2988
3586
|
this.#face("idle");
|
|
2989
3587
|
} catch (error) {
|
|
3588
|
+
if (!controller.signal.aborted) controller.abort(error);
|
|
2990
3589
|
if (!live()) return;
|
|
2991
3590
|
if (this.#speaking) this.#deps.send({
|
|
2992
3591
|
type: "tts",
|
|
@@ -3001,16 +3600,19 @@ var TurnController = class {
|
|
|
3001
3600
|
}
|
|
3002
3601
|
#failureLine(error) {
|
|
3003
3602
|
if (error instanceof TurnTimeout) return "That took too long.";
|
|
3603
|
+
if (error instanceof StageError && error.cause instanceof VoiceNotReady) return error.cause.message;
|
|
3004
3604
|
if (error instanceof StageError && error.stage === "stt") return "I couldn't hear that.";
|
|
3005
3605
|
if (error instanceof StageError && error.stage === "tts") return "I couldn't speak that.";
|
|
3006
3606
|
return `Can't reach ${AGENT_LABELS[this.#deps.agent.name] ?? "your agent"}`;
|
|
3007
3607
|
}
|
|
3008
|
-
async #sendPaced(packets, live) {
|
|
3608
|
+
async #sendPaced(packets, live, onPacket) {
|
|
3009
3609
|
const sleep = this.#deps.sleep ?? sleepMs;
|
|
3010
|
-
|
|
3610
|
+
let index = 0;
|
|
3611
|
+
for await (const packet of packets) {
|
|
3011
3612
|
if (!live()) return;
|
|
3012
3613
|
this.#deps.sendAudio(packet);
|
|
3013
|
-
|
|
3614
|
+
onPacket();
|
|
3615
|
+
if (index++ >= 2) await sleep(opusPacketSamples48k(packet) / 48);
|
|
3014
3616
|
}
|
|
3015
3617
|
}
|
|
3016
3618
|
dispose() {
|
|
@@ -3520,6 +4122,205 @@ function createLocalVoice(options) {
|
|
|
3520
4122
|
};
|
|
3521
4123
|
}
|
|
3522
4124
|
//#endregion
|
|
4125
|
+
//#region ../charmd/src/audio/webm-opus.ts
|
|
4126
|
+
const SEGMENT = 408125543;
|
|
4127
|
+
const CLUSTER = 524531317;
|
|
4128
|
+
const BLOCK_GROUP = 160;
|
|
4129
|
+
const BLOCK = 161;
|
|
4130
|
+
const SIMPLE_BLOCK = 163;
|
|
4131
|
+
const CONTAINERS = /* @__PURE__ */ new Set([
|
|
4132
|
+
SEGMENT,
|
|
4133
|
+
CLUSTER,
|
|
4134
|
+
BLOCK_GROUP
|
|
4135
|
+
]);
|
|
4136
|
+
var CutShort = class extends Error {};
|
|
4137
|
+
function readVint(data, offset, keepMarker) {
|
|
4138
|
+
const first = data[offset];
|
|
4139
|
+
if (first === void 0) throw new CutShort();
|
|
4140
|
+
if (first === 0) throw new Error("Not a WebM file: bad element header");
|
|
4141
|
+
const length = Math.clz32(first) - 23;
|
|
4142
|
+
if (offset + length > data.length) throw new CutShort();
|
|
4143
|
+
let value = keepMarker ? first : first & 255 >> length;
|
|
4144
|
+
let allOnes = value === 255 >> length;
|
|
4145
|
+
for (let i = 1; i < length; i++) {
|
|
4146
|
+
const byte = data[offset + i];
|
|
4147
|
+
value = value * 256 + byte;
|
|
4148
|
+
allOnes &&= byte === 255;
|
|
4149
|
+
}
|
|
4150
|
+
return {
|
|
4151
|
+
value,
|
|
4152
|
+
length,
|
|
4153
|
+
unknown: !keepMarker && allOnes
|
|
4154
|
+
};
|
|
4155
|
+
}
|
|
4156
|
+
function frameOf(block) {
|
|
4157
|
+
const track = readVint(block, 0, false);
|
|
4158
|
+
const flags = block[track.length + 2];
|
|
4159
|
+
if (flags === void 0) throw new Error("Not a WebM file: empty block");
|
|
4160
|
+
if (flags & 6) throw new Error("WebM lacing isn't supported");
|
|
4161
|
+
return block.subarray(track.length + 3);
|
|
4162
|
+
}
|
|
4163
|
+
function parse(data) {
|
|
4164
|
+
const packets = [];
|
|
4165
|
+
let offset = 0;
|
|
4166
|
+
while (offset < data.length) {
|
|
4167
|
+
let id;
|
|
4168
|
+
let size;
|
|
4169
|
+
try {
|
|
4170
|
+
id = readVint(data, offset, true);
|
|
4171
|
+
size = readVint(data, offset + id.length, false);
|
|
4172
|
+
} catch (error) {
|
|
4173
|
+
if (error instanceof CutShort) break;
|
|
4174
|
+
throw error;
|
|
4175
|
+
}
|
|
4176
|
+
const body = offset + id.length + size.length;
|
|
4177
|
+
if (CONTAINERS.has(id.value)) {
|
|
4178
|
+
offset = body;
|
|
4179
|
+
continue;
|
|
4180
|
+
}
|
|
4181
|
+
if (size.unknown) throw new Error("Not a WebM file: an element of unknown size");
|
|
4182
|
+
const end = body + size.value;
|
|
4183
|
+
if (end > data.length) break;
|
|
4184
|
+
if (id.value === SIMPLE_BLOCK || id.value === BLOCK) {
|
|
4185
|
+
const frame = frameOf(data.subarray(body, end));
|
|
4186
|
+
if (frame.length) packets.push(frame);
|
|
4187
|
+
}
|
|
4188
|
+
offset = end;
|
|
4189
|
+
}
|
|
4190
|
+
return {
|
|
4191
|
+
packets,
|
|
4192
|
+
consumed: offset
|
|
4193
|
+
};
|
|
4194
|
+
}
|
|
4195
|
+
var WebmOpusReader = class {
|
|
4196
|
+
#pending = Buffer.alloc(0);
|
|
4197
|
+
push(chunk) {
|
|
4198
|
+
this.#pending = Buffer.concat([this.#pending, chunk]);
|
|
4199
|
+
const { packets, consumed } = parse(this.#pending);
|
|
4200
|
+
this.#pending = this.#pending.subarray(consumed);
|
|
4201
|
+
return packets;
|
|
4202
|
+
}
|
|
4203
|
+
get complete() {
|
|
4204
|
+
return this.#pending.length === 0;
|
|
4205
|
+
}
|
|
4206
|
+
};
|
|
4207
|
+
//#endregion
|
|
4208
|
+
//#region ../charmd/src/voice/microsoft.ts
|
|
4209
|
+
const TRUSTED_CLIENT_TOKEN = "6A5AA1D4EAFF4E9FB37E23D68491D6F4";
|
|
4210
|
+
const SERVICE_URL = "wss://speech.platform.bing.com/consumer/speech/synthesize/readaloud/edge/v1";
|
|
4211
|
+
const GEC_VERSION = "1-143.0.3650.96";
|
|
4212
|
+
const USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/143.0.0.0 Safari/537.36 Edg/143.0.0.0";
|
|
4213
|
+
const ORIGIN = "chrome-extension://jdiccldimpdaibmpdkjnbmckianbfold";
|
|
4214
|
+
const OUTPUT_FORMAT = "webm-24khz-16bit-mono-opus";
|
|
4215
|
+
const DEFAULT_VOICES = {
|
|
4216
|
+
en: "en-US-AvaMultilingualNeural",
|
|
4217
|
+
it: "it-IT-IsabellaNeural",
|
|
4218
|
+
es: "es-ES-ElviraNeural",
|
|
4219
|
+
fr: "fr-FR-DeniseNeural",
|
|
4220
|
+
de: "de-DE-KatjaNeural",
|
|
4221
|
+
pt: "pt-BR-FranciscaNeural"
|
|
4222
|
+
};
|
|
4223
|
+
const WINDOWS_EPOCH_SECONDS = 11644473600;
|
|
4224
|
+
function secMsGec(nowMs = Date.now()) {
|
|
4225
|
+
const seconds = Math.floor(nowMs / 1e3) + WINDOWS_EPOCH_SECONDS;
|
|
4226
|
+
const ticks = BigInt(seconds - seconds % 300) * 10000000n;
|
|
4227
|
+
return createHash("sha256").update(`${ticks}${TRUSTED_CLIENT_TOKEN}`).digest("hex").toUpperCase();
|
|
4228
|
+
}
|
|
4229
|
+
function escapeXml(text) {
|
|
4230
|
+
return text.replace(/&/g, "&").replace(/</g, "<").replace(/>/g, ">").replace(/"/g, """).replace(/'/g, "'");
|
|
4231
|
+
}
|
|
4232
|
+
function ssml(text, voice) {
|
|
4233
|
+
return `<speak version='1.0' xmlns='http://www.w3.org/2001/10/synthesis' xml:lang='${voice.split("-").slice(0, 2).join("-")}'><voice name='${voice}'><prosody pitch='+0Hz' rate='+0%' volume='+0%'>${escapeXml(text)}</prosody></voice></speak>`;
|
|
4234
|
+
}
|
|
4235
|
+
function message(path, contentType, body, requestId) {
|
|
4236
|
+
return `X-Timestamp:${(/* @__PURE__ */ new Date()).toISOString()}\r\n` + (requestId ? `X-RequestId:${requestId}\r\n` : "") + `Content-Type:${contentType}\r\nPath:${path}\r\n\r\n${body}`;
|
|
4237
|
+
}
|
|
4238
|
+
function audioOf(frame) {
|
|
4239
|
+
if (frame.length < 2) return void 0;
|
|
4240
|
+
const headerLength = frame.readUInt16BE(0);
|
|
4241
|
+
return frame.subarray(2, 2 + headerLength).toString().includes("Path:audio") ? frame.subarray(2 + headerLength) : void 0;
|
|
4242
|
+
}
|
|
4243
|
+
function voiceFor(language, voices) {
|
|
4244
|
+
return voices[language] ?? voices.en ?? DEFAULT_VOICES.en;
|
|
4245
|
+
}
|
|
4246
|
+
function connect(text, language, voices, signal, options) {
|
|
4247
|
+
const timeoutMs = options.timeoutMs ?? 15e3;
|
|
4248
|
+
const connection = randomUUID().replace(/-/g, "");
|
|
4249
|
+
const url = `${options.url ?? SERVICE_URL}?TrustedClientToken=${TRUSTED_CLIENT_TOKEN}&Sec-MS-GEC=${secMsGec()}&Sec-MS-GEC-Version=${GEC_VERSION}&ConnectionId=${connection}`;
|
|
4250
|
+
const packets = createChannel();
|
|
4251
|
+
const reader = new WebmOpusReader();
|
|
4252
|
+
let sent = 0;
|
|
4253
|
+
let settled = false;
|
|
4254
|
+
const socket = new WebSocket(url, {
|
|
4255
|
+
headers: {
|
|
4256
|
+
"User-Agent": USER_AGENT,
|
|
4257
|
+
Origin: ORIGIN
|
|
4258
|
+
},
|
|
4259
|
+
handshakeTimeout: timeoutMs
|
|
4260
|
+
});
|
|
4261
|
+
const finish = (error) => {
|
|
4262
|
+
if (settled) return;
|
|
4263
|
+
settled = true;
|
|
4264
|
+
clearTimeout(timer);
|
|
4265
|
+
signal.removeEventListener("abort", onAbort);
|
|
4266
|
+
socket.terminate();
|
|
4267
|
+
if (error) packets.fail(error);
|
|
4268
|
+
else if (sent === 0) packets.fail(/* @__PURE__ */ new Error("Microsoft's voice sent no audio"));
|
|
4269
|
+
else packets.end();
|
|
4270
|
+
};
|
|
4271
|
+
const onAbort = () => finish(signal.reason instanceof Error ? signal.reason : /* @__PURE__ */ new Error("Aborted"));
|
|
4272
|
+
const timer = setTimeout(() => finish(/* @__PURE__ */ new Error("Microsoft's voice took too long")), timeoutMs);
|
|
4273
|
+
if (signal.aborted) onAbort();
|
|
4274
|
+
else signal.addEventListener("abort", onAbort, { once: true });
|
|
4275
|
+
socket.on("open", () => {
|
|
4276
|
+
socket.send(message("speech.config", "application/json; charset=utf-8", JSON.stringify({ context: { synthesis: { audio: {
|
|
4277
|
+
metadataoptions: {
|
|
4278
|
+
sentenceBoundaryEnabled: "false",
|
|
4279
|
+
wordBoundaryEnabled: "false"
|
|
4280
|
+
},
|
|
4281
|
+
outputFormat: OUTPUT_FORMAT
|
|
4282
|
+
} } } })));
|
|
4283
|
+
socket.send(message("ssml", "application/ssml+xml", ssml(text, voiceFor(language, voices)), connection));
|
|
4284
|
+
});
|
|
4285
|
+
socket.on("message", (data, isBinary) => {
|
|
4286
|
+
const frame = Buffer.isBuffer(data) ? data : Buffer.concat(data);
|
|
4287
|
+
if (!isBinary) {
|
|
4288
|
+
if (frame.toString().includes("Path:turn.end")) finish();
|
|
4289
|
+
return;
|
|
4290
|
+
}
|
|
4291
|
+
const audio = audioOf(frame);
|
|
4292
|
+
if (!audio?.length) return;
|
|
4293
|
+
try {
|
|
4294
|
+
for (const packet of reader.push(audio)) {
|
|
4295
|
+
sent += 1;
|
|
4296
|
+
packets.push(packet);
|
|
4297
|
+
}
|
|
4298
|
+
} catch (error) {
|
|
4299
|
+
finish(error instanceof Error ? error : new Error(String(error)));
|
|
4300
|
+
}
|
|
4301
|
+
});
|
|
4302
|
+
socket.on("unexpected-response", (_request, response) => finish(/* @__PURE__ */ new Error(`Microsoft's voice refused the request: ${response.statusCode}`)));
|
|
4303
|
+
socket.on("error", (error) => finish(/* @__PURE__ */ new Error(`Microsoft's voice failed: ${error.message}`)));
|
|
4304
|
+
socket.on("close", () => finish(/* @__PURE__ */ new Error("Microsoft's voice closed before it finished")));
|
|
4305
|
+
return packets;
|
|
4306
|
+
}
|
|
4307
|
+
function createMicrosoftSpeaker(options = {}) {
|
|
4308
|
+
const voices = {
|
|
4309
|
+
...DEFAULT_VOICES,
|
|
4310
|
+
...options.voices
|
|
4311
|
+
};
|
|
4312
|
+
const stream = (text, signal, language = "en") => connect(text, language, voices, signal, options);
|
|
4313
|
+
return {
|
|
4314
|
+
name: "microsoft",
|
|
4315
|
+
stream,
|
|
4316
|
+
async synthesize(text, signal, language = "en") {
|
|
4317
|
+
const packets = [];
|
|
4318
|
+
for await (const packet of stream(text, signal, language)) packets.push(packet);
|
|
4319
|
+
return writeOggOpus(packets, { inputSampleRate: 24e3 });
|
|
4320
|
+
}
|
|
4321
|
+
};
|
|
4322
|
+
}
|
|
4323
|
+
//#endregion
|
|
3523
4324
|
//#region ../charmd/src/voice/openai.ts
|
|
3524
4325
|
function createOpenAiVoice(options) {
|
|
3525
4326
|
const apiKey = options.apiKey ?? process.env.OPENAI_API_KEY;
|
|
@@ -3563,22 +4364,260 @@ function createOpenAiVoice(options) {
|
|
|
3563
4364
|
};
|
|
3564
4365
|
}
|
|
3565
4366
|
//#endregion
|
|
4367
|
+
//#region ../charmd/src/voice/sherpa.ts
|
|
4368
|
+
const SUPERTONIC_LANGUAGES = /* @__PURE__ */ new Set([
|
|
4369
|
+
"en",
|
|
4370
|
+
"it",
|
|
4371
|
+
"es",
|
|
4372
|
+
"fr",
|
|
4373
|
+
"de",
|
|
4374
|
+
"pt"
|
|
4375
|
+
]);
|
|
4376
|
+
const DEFAULT_SPEAKER = 5;
|
|
4377
|
+
const OUTPUT_RATE = 24e3;
|
|
4378
|
+
let sherpa;
|
|
4379
|
+
function loadSherpa() {
|
|
4380
|
+
if (sherpa) return sherpa;
|
|
4381
|
+
try {
|
|
4382
|
+
sherpa = createRequire(import.meta.url)("sherpa-onnx-node");
|
|
4383
|
+
return sherpa;
|
|
4384
|
+
} catch (error) {
|
|
4385
|
+
throw new Error(`The local voice can't run on this computer: sherpa-onnx didn't load (${error instanceof Error ? error.message.split("\n")[0] : String(error)})`, { cause: error });
|
|
4386
|
+
}
|
|
4387
|
+
}
|
|
4388
|
+
function ready(id, dir) {
|
|
4389
|
+
const state = modelState(id, dir);
|
|
4390
|
+
if (state.state === "ready") return join(dir, MODELS[id].folder);
|
|
4391
|
+
if (state.state === "failed" && Date.now() - state.at < 3e5) throw new VoiceNotReady(`Couldn't get ${MODELS[id].label}: ${state.error}`.slice(0, 160));
|
|
4392
|
+
if (state.state !== "downloading") installModel(id, { dir }).catch(() => void 0);
|
|
4393
|
+
const now = modelState(id, dir);
|
|
4394
|
+
const percent = now.state === "downloading" && now.total ? ` (${Math.floor(now.received * 100 / now.total)}%)` : "";
|
|
4395
|
+
throw new VoiceNotReady(`${MODELS[id].label} is still downloading${percent}`);
|
|
4396
|
+
}
|
|
4397
|
+
function toFloat(pcm) {
|
|
4398
|
+
const out = new Float32Array(pcm.length / 2);
|
|
4399
|
+
for (let i = 0; i < out.length; i++) out[i] = pcm.readInt16LE(i * 2) / 32768;
|
|
4400
|
+
return out;
|
|
4401
|
+
}
|
|
4402
|
+
function toPcm(samples) {
|
|
4403
|
+
const out = Buffer.alloc(samples.length * 2);
|
|
4404
|
+
for (let i = 0; i < samples.length; i++) out.writeInt16LE(Math.round(Math.max(-1, Math.min(1, samples[i] ?? 0)) * 32767), i * 2);
|
|
4405
|
+
return out;
|
|
4406
|
+
}
|
|
4407
|
+
function createParakeetListener(options = {}) {
|
|
4408
|
+
const dir = options.modelsDir ?? defaultModelsDir();
|
|
4409
|
+
let recognizer;
|
|
4410
|
+
const load = () => {
|
|
4411
|
+
const folder = ready("parakeet", dir);
|
|
4412
|
+
recognizer ??= loadSherpa().OfflineRecognizer.createAsync({
|
|
4413
|
+
featConfig: {
|
|
4414
|
+
sampleRate: 16e3,
|
|
4415
|
+
featureDim: 80
|
|
4416
|
+
},
|
|
4417
|
+
modelConfig: {
|
|
4418
|
+
transducer: {
|
|
4419
|
+
encoder: join(folder, "encoder.int8.onnx"),
|
|
4420
|
+
decoder: join(folder, "decoder.int8.onnx"),
|
|
4421
|
+
joiner: join(folder, "joiner.int8.onnx")
|
|
4422
|
+
},
|
|
4423
|
+
tokens: join(folder, "tokens.txt"),
|
|
4424
|
+
modelType: "nemo_transducer",
|
|
4425
|
+
numThreads: options.threads ?? 4,
|
|
4426
|
+
provider: "cpu"
|
|
4427
|
+
}
|
|
4428
|
+
}).catch((error) => {
|
|
4429
|
+
recognizer = void 0;
|
|
4430
|
+
throw error;
|
|
4431
|
+
});
|
|
4432
|
+
return recognizer;
|
|
4433
|
+
};
|
|
4434
|
+
return {
|
|
4435
|
+
name: "local",
|
|
4436
|
+
warm: async () => {
|
|
4437
|
+
await load();
|
|
4438
|
+
},
|
|
4439
|
+
async transcribe(ogg) {
|
|
4440
|
+
const engine = await load();
|
|
4441
|
+
const stream = engine.createStream();
|
|
4442
|
+
stream.acceptWaveform({
|
|
4443
|
+
samples: toFloat(oggOpusToPcm(ogg, 16e3)),
|
|
4444
|
+
sampleRate: 16e3
|
|
4445
|
+
});
|
|
4446
|
+
return ((await engine.decodeAsync(stream)).text ?? "").replace(/\s+/g, " ").trim();
|
|
4447
|
+
}
|
|
4448
|
+
};
|
|
4449
|
+
}
|
|
4450
|
+
function createSupertonicSpeaker(options = {}) {
|
|
4451
|
+
const dir = options.modelsDir ?? defaultModelsDir();
|
|
4452
|
+
let tts;
|
|
4453
|
+
const load = () => {
|
|
4454
|
+
const folder = ready("supertonic", dir);
|
|
4455
|
+
const file = (name) => join(folder, name);
|
|
4456
|
+
tts ??= loadSherpa().OfflineTts.createAsync({
|
|
4457
|
+
model: {
|
|
4458
|
+
supertonic: {
|
|
4459
|
+
durationPredictor: file("duration_predictor.int8.onnx"),
|
|
4460
|
+
textEncoder: file("text_encoder.int8.onnx"),
|
|
4461
|
+
vectorEstimator: file("vector_estimator.int8.onnx"),
|
|
4462
|
+
vocoder: file("vocoder.int8.onnx"),
|
|
4463
|
+
ttsJson: file("tts.json"),
|
|
4464
|
+
unicodeIndexer: file("unicode_indexer.bin"),
|
|
4465
|
+
voiceStyle: file("voice.bin")
|
|
4466
|
+
},
|
|
4467
|
+
numThreads: options.threads ?? 4,
|
|
4468
|
+
provider: "cpu"
|
|
4469
|
+
},
|
|
4470
|
+
maxNumSentences: 1
|
|
4471
|
+
}).catch((error) => {
|
|
4472
|
+
tts = void 0;
|
|
4473
|
+
throw error;
|
|
4474
|
+
});
|
|
4475
|
+
return tts;
|
|
4476
|
+
};
|
|
4477
|
+
return {
|
|
4478
|
+
name: "local",
|
|
4479
|
+
onDevice: true,
|
|
4480
|
+
warm: async () => {
|
|
4481
|
+
await load();
|
|
4482
|
+
},
|
|
4483
|
+
async synthesize(text, signal, language = "en") {
|
|
4484
|
+
const engine = await load();
|
|
4485
|
+
const sherpaModule = loadSherpa();
|
|
4486
|
+
const audio = await engine.generateAsync({
|
|
4487
|
+
text,
|
|
4488
|
+
generationConfig: new sherpaModule.GenerationConfig({
|
|
4489
|
+
sid: options.speaker ?? DEFAULT_SPEAKER,
|
|
4490
|
+
numSteps: 8,
|
|
4491
|
+
extra: { lang: SUPERTONIC_LANGUAGES.has(language) ? language : "en" }
|
|
4492
|
+
}),
|
|
4493
|
+
onProgress: () => signal.aborted ? 0 : 1
|
|
4494
|
+
});
|
|
4495
|
+
signal.throwIfAborted();
|
|
4496
|
+
return pcmToOggOpus(toPcm(audio.sampleRate === OUTPUT_RATE ? audio.samples : new sherpaModule.LinearResampler(audio.sampleRate, OUTPUT_RATE).flush(audio.samples)), OUTPUT_RATE);
|
|
4497
|
+
}
|
|
4498
|
+
};
|
|
4499
|
+
}
|
|
4500
|
+
//#endregion
|
|
4501
|
+
//#region ../charmd/src/voice/speak.ts
|
|
4502
|
+
const FIRST_AUDIO_MS = 3e3;
|
|
4503
|
+
const REST_MS = 6e4;
|
|
4504
|
+
async function firstWithin(packets, ms, name) {
|
|
4505
|
+
let timer;
|
|
4506
|
+
try {
|
|
4507
|
+
await Promise.race([packets.first(), new Promise((_, reject) => {
|
|
4508
|
+
timer = setTimeout(() => reject(/* @__PURE__ */ new Error(`${name} had no audio within ${ms} ms`)), ms);
|
|
4509
|
+
})]);
|
|
4510
|
+
} finally {
|
|
4511
|
+
clearTimeout(timer);
|
|
4512
|
+
}
|
|
4513
|
+
}
|
|
4514
|
+
function speakWithFallback(chain, options = {}) {
|
|
4515
|
+
const [first] = chain;
|
|
4516
|
+
if (!first) throw new Error("No voice to speak with");
|
|
4517
|
+
if (chain.length === 1) return first;
|
|
4518
|
+
const now = options.now ?? Date.now;
|
|
4519
|
+
const resting = /* @__PURE__ */ new Map();
|
|
4520
|
+
async function* stream(text, signal, language) {
|
|
4521
|
+
let lastError;
|
|
4522
|
+
for (const [index, speaker] of chain.entries()) {
|
|
4523
|
+
const last = index === chain.length - 1;
|
|
4524
|
+
if (!last && (resting.get(speaker) ?? 0) > now()) continue;
|
|
4525
|
+
const controller = new AbortController();
|
|
4526
|
+
const abort = () => controller.abort(signal.reason);
|
|
4527
|
+
signal.addEventListener("abort", abort, { once: true });
|
|
4528
|
+
const packets = new PacketStream(packetsOf(speaker, text, controller.signal, language));
|
|
4529
|
+
try {
|
|
4530
|
+
if (last || speaker.onDevice) await packets.first();
|
|
4531
|
+
else await firstWithin(packets, options.timeoutMs ?? FIRST_AUDIO_MS, speaker.name);
|
|
4532
|
+
} catch (error) {
|
|
4533
|
+
signal.removeEventListener("abort", abort);
|
|
4534
|
+
controller.abort(error);
|
|
4535
|
+
signal.throwIfAborted();
|
|
4536
|
+
lastError = error;
|
|
4537
|
+
resting.set(speaker, now() + (options.restMs ?? REST_MS));
|
|
4538
|
+
continue;
|
|
4539
|
+
}
|
|
4540
|
+
try {
|
|
4541
|
+
yield* packets;
|
|
4542
|
+
} finally {
|
|
4543
|
+
signal.removeEventListener("abort", abort);
|
|
4544
|
+
}
|
|
4545
|
+
return;
|
|
4546
|
+
}
|
|
4547
|
+
throw lastError instanceof Error ? lastError : new Error(String(lastError));
|
|
4548
|
+
}
|
|
4549
|
+
return {
|
|
4550
|
+
name: first.name,
|
|
4551
|
+
stream,
|
|
4552
|
+
async synthesize(text, signal, language) {
|
|
4553
|
+
const packets = [];
|
|
4554
|
+
for await (const packet of stream(text, signal, language)) packets.push(packet);
|
|
4555
|
+
return writeOggOpus(packets, { inputSampleRate: 24e3 });
|
|
4556
|
+
},
|
|
4557
|
+
warm: async () => {
|
|
4558
|
+
await Promise.allSettled(chain.map((speaker) => speaker.warm?.() ?? Promise.resolve()));
|
|
4559
|
+
}
|
|
4560
|
+
};
|
|
4561
|
+
}
|
|
4562
|
+
//#endregion
|
|
3566
4563
|
//#region ../charmd/src/wiring.ts
|
|
3567
4564
|
function key(config, fallbackEnv) {
|
|
3568
4565
|
const env = config.apiKeyEnv ?? fallbackEnv;
|
|
3569
4566
|
return config.apiKey ?? (env ? process.env[env] : void 0);
|
|
3570
4567
|
}
|
|
3571
|
-
function
|
|
3572
|
-
|
|
3573
|
-
|
|
3574
|
-
case "
|
|
3575
|
-
case "
|
|
4568
|
+
function createListener(config) {
|
|
4569
|
+
switch (config.provider) {
|
|
4570
|
+
case "fake": return createFakeVoice(config);
|
|
4571
|
+
case "local": return createParakeetListener();
|
|
4572
|
+
case "whisper": return createLocalVoice({
|
|
4573
|
+
...config.model ? { whisperModel: config.model } : {},
|
|
4574
|
+
...config.command ? { whisperCommand: config.command } : {}
|
|
4575
|
+
});
|
|
4576
|
+
case "openai": return createOpenAiVoice({
|
|
4577
|
+
...config,
|
|
4578
|
+
apiKey: key(config, "OPENAI_API_KEY"),
|
|
4579
|
+
...config.model ? { sttModel: config.model } : {}
|
|
4580
|
+
});
|
|
4581
|
+
}
|
|
4582
|
+
}
|
|
4583
|
+
function createSpeaker(config) {
|
|
4584
|
+
switch (config.provider) {
|
|
4585
|
+
case "fake": return createFakeVoice();
|
|
4586
|
+
case "microsoft": return createMicrosoftSpeaker(config.voices ? { voices: config.voices } : {});
|
|
4587
|
+
case "local": return createSupertonicSpeaker(config.speaker === void 0 ? {} : { speaker: config.speaker });
|
|
4588
|
+
case "system": return {
|
|
4589
|
+
...createLocalVoice(config.voice ? { sayVoice: config.voice } : {}),
|
|
4590
|
+
name: "system",
|
|
4591
|
+
onDevice: true
|
|
4592
|
+
};
|
|
3576
4593
|
case "openai": return createOpenAiVoice({
|
|
3577
|
-
...
|
|
3578
|
-
apiKey: key(
|
|
4594
|
+
...config,
|
|
4595
|
+
apiKey: key(config, "OPENAI_API_KEY"),
|
|
4596
|
+
...config.model ? { ttsModel: config.model } : {}
|
|
3579
4597
|
});
|
|
3580
4598
|
}
|
|
3581
4599
|
}
|
|
4600
|
+
function speakerChain(config) {
|
|
4601
|
+
if (config.provider === "fake" || config.provider === "system") return createSpeaker(config);
|
|
4602
|
+
const chain = [createSpeaker(config)];
|
|
4603
|
+
if (config.provider === "microsoft") chain.push(createSupertonicSpeaker());
|
|
4604
|
+
if (process.platform === "darwin") chain.push(createSpeaker({ provider: "system" }));
|
|
4605
|
+
return speakWithFallback(chain);
|
|
4606
|
+
}
|
|
4607
|
+
function createVoiceFromConfig(config) {
|
|
4608
|
+
const listener = createListener(config.voice.listen);
|
|
4609
|
+
const speaker = speakerChain(config.voice.speak);
|
|
4610
|
+
return {
|
|
4611
|
+
name: listener.name === speaker.name ? listener.name : `${listener.name}+${speaker.name}`,
|
|
4612
|
+
transcribe: listener.transcribe,
|
|
4613
|
+
synthesize: speaker.synthesize,
|
|
4614
|
+
...speaker.stream ? { stream: speaker.stream } : {},
|
|
4615
|
+
language: config.voice.language,
|
|
4616
|
+
warm: async () => {
|
|
4617
|
+
await Promise.allSettled([listener.warm?.(), speaker.warm?.()]);
|
|
4618
|
+
}
|
|
4619
|
+
};
|
|
4620
|
+
}
|
|
3582
4621
|
function charmToolsServers(config, tools) {
|
|
3583
4622
|
const agent = config.agent;
|
|
3584
4623
|
if (agent.adapter !== "acp" || !agent.charmTools || !tools) return [];
|
|
@@ -3631,6 +4670,7 @@ async function startDaemon(config, options = {}) {
|
|
|
3631
4670
|
});
|
|
3632
4671
|
const sessions = /* @__PURE__ */ new Map();
|
|
3633
4672
|
const voice = createVoiceFromConfig(config);
|
|
4673
|
+
voice.warm?.().catch(() => void 0);
|
|
3634
4674
|
const log = options.log ?? ((entry) => {
|
|
3635
4675
|
if (!options.quiet) console.log(JSON.stringify({
|
|
3636
4676
|
time: (/* @__PURE__ */ new Date()).toISOString(),
|
|
@@ -3650,7 +4690,15 @@ async function startDaemon(config, options = {}) {
|
|
|
3650
4690
|
look,
|
|
3651
4691
|
agentCanChangeLook,
|
|
3652
4692
|
identity: options.identity,
|
|
3653
|
-
starter: options.starter
|
|
4693
|
+
starter: options.starter,
|
|
4694
|
+
voice: () => {
|
|
4695
|
+
const downloading = ["parakeet", "supertonic"].flatMap((id) => {
|
|
4696
|
+
const state = modelState(id);
|
|
4697
|
+
if (state.state === "failed") return [`${MODELS[id].label} failed`];
|
|
4698
|
+
return state.state === "downloading" && state.total ? [`${MODELS[id].label} downloading ${Math.floor(state.received * 100 / state.total)}%`] : [];
|
|
4699
|
+
});
|
|
4700
|
+
return [voice.name, ...downloading].join(" · ");
|
|
4701
|
+
}
|
|
3654
4702
|
});
|
|
3655
4703
|
const adminServer = await startAdminServer(config.adminSocket, admin);
|
|
3656
4704
|
let websocketUrl = config.publicUrl ?? "";
|
|
@@ -3783,7 +4831,7 @@ function formatIdentity(unit, version, commit) {
|
|
|
3783
4831
|
let cached;
|
|
3784
4832
|
function identity() {
|
|
3785
4833
|
if (cached) return cached;
|
|
3786
|
-
const commit = "
|
|
4834
|
+
const commit = "7d76bc9";
|
|
3787
4835
|
cached = {
|
|
3788
4836
|
version: `cli@${version}`,
|
|
3789
4837
|
commit,
|
|
@@ -3832,6 +4880,45 @@ async function runServe(ctx, args) {
|
|
|
3832
4880
|
process.once("SIGTERM", stop);
|
|
3833
4881
|
}
|
|
3834
4882
|
//#endregion
|
|
4883
|
+
//#region src/commands/voice.ts
|
|
4884
|
+
const IDS = Object.keys(MODELS);
|
|
4885
|
+
function megabytes(bytes) {
|
|
4886
|
+
return `${Math.round(bytes / 1e6)} MB`;
|
|
4887
|
+
}
|
|
4888
|
+
async function runVoice(ctx, args, deps = {}) {
|
|
4889
|
+
const install = deps.install ?? installModel;
|
|
4890
|
+
const state = deps.state ?? modelState;
|
|
4891
|
+
const action = firstPositional(args) ?? "";
|
|
4892
|
+
if (action !== "" && action !== "install") {
|
|
4893
|
+
ctx.err.write(`Unknown voice command "${action}". Try: opencharm voice install\n`);
|
|
4894
|
+
process.exitCode = 1;
|
|
4895
|
+
return;
|
|
4896
|
+
}
|
|
4897
|
+
for (const id of IDS) {
|
|
4898
|
+
const model = MODELS[id];
|
|
4899
|
+
if (action === "") {
|
|
4900
|
+
const ready = state(id).state === "ready";
|
|
4901
|
+
ctx.out.write(`${model.label}: ${ready ? "installed" : `not downloaded yet (${megabytes(model.bytes)})`}\n`);
|
|
4902
|
+
continue;
|
|
4903
|
+
}
|
|
4904
|
+
let shown = -1;
|
|
4905
|
+
try {
|
|
4906
|
+
await install(id, { onProgress: (received, total) => {
|
|
4907
|
+
const percent = Math.floor(received * 100 / (total || model.bytes));
|
|
4908
|
+
if (percent >= shown + 10) {
|
|
4909
|
+
shown = percent - percent % 10;
|
|
4910
|
+
ctx.out.write(`${model.label}: ${shown}%\n`);
|
|
4911
|
+
}
|
|
4912
|
+
} });
|
|
4913
|
+
ctx.out.write(`${model.label}: installed\n`);
|
|
4914
|
+
} catch (error) {
|
|
4915
|
+
ctx.err.write(`${error instanceof Error ? error.message : String(error)}\n`);
|
|
4916
|
+
process.exitCode = 1;
|
|
4917
|
+
return;
|
|
4918
|
+
}
|
|
4919
|
+
}
|
|
4920
|
+
}
|
|
4921
|
+
//#endregion
|
|
3835
4922
|
//#region src/commands/setup.ts
|
|
3836
4923
|
const USER = "charmd";
|
|
3837
4924
|
const STATE_DIR = "/var/lib/charmd";
|
|
@@ -4206,6 +5293,9 @@ async function main(argv) {
|
|
|
4206
5293
|
case "serve":
|
|
4207
5294
|
await runServe(ctx, rest);
|
|
4208
5295
|
return;
|
|
5296
|
+
case "voice":
|
|
5297
|
+
await runVoice(ctx, rest);
|
|
5298
|
+
return;
|
|
4209
5299
|
case "mcp": {
|
|
4210
5300
|
const socket = flagValue(rest, "--socket") ?? loadConfig(flagValue(rest, "--config")).adminSocket;
|
|
4211
5301
|
await serveMcp(process.stdin, process.stdout, {
|