dsh-her 0.1.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/client.js +862 -506
- package/lib/core/character.js +1 -1
- package/lib/core/index.js +3 -1
- package/lib/core/sounds.js +31 -0
- package/lib/core/spoken-numbers.js +90 -0
- package/lib/core/voices.js +31 -1
- package/lib/dsh/api.js +53 -0
- package/lib/dsh/plugin.js +129 -14
- package/lib/types/core/character.d.ts +1 -1
- package/lib/types/core/index.d.ts +3 -1
- package/lib/types/core/sounds.d.ts +21 -0
- package/lib/types/core/spoken-numbers.d.ts +11 -0
- package/lib/types/core/voices.d.ts +18 -2
- package/lib/types/dsh/api.d.ts +37 -1
- package/lib/types/dsh/plugin.d.ts +2 -0
- package/lib/types/voice/ming-model.d.ts +62 -0
- package/lib/types/voice/ming-voices.d.ts +75 -0
- package/lib/types/voice/ming.d.ts +103 -0
- package/lib/types/voice/speech-queue.d.ts +4 -2
- package/lib/types/voice/uv.d.ts +12 -0
- package/lib/types/voice/voice.d.ts +14 -6
- package/lib/voice/edge.js +4 -48
- package/lib/voice/ming-model.js +30 -0
- package/lib/voice/ming-voices.js +181 -0
- package/lib/voice/ming.js +335 -0
- package/lib/voice/speech-queue.js +5 -3
- package/lib/voice/uv.js +60 -0
- package/lib/voice/voice.js +46 -10
- package/package.json +2 -1
- package/python/ming-requirements.in +1 -0
- package/python/ming-requirements.txt +887 -0
- package/python/ming_worker.py +77 -0
package/lib/types/dsh/api.d.ts
CHANGED
|
@@ -14,6 +14,8 @@ import type { ReportLine } from './import-status.ts';
|
|
|
14
14
|
import type { LookProject, WorkshopCard } from '../core/workshop.ts';
|
|
15
15
|
import type { ProjectStatus } from './workshop.ts';
|
|
16
16
|
import type { InstallState } from './body-install.ts';
|
|
17
|
+
import type { MingStatus } from '../voice/ming.ts';
|
|
18
|
+
import type { MingSettings, OwnVoice } from '../voice/ming-voices.ts';
|
|
17
19
|
export declare const API_BASE = "/api/dsh-her";
|
|
18
20
|
export interface StateView {
|
|
19
21
|
platform: string;
|
|
@@ -25,7 +27,7 @@ export interface StateView {
|
|
|
25
27
|
effort?: string;
|
|
26
28
|
chosen: boolean;
|
|
27
29
|
} | null;
|
|
28
|
-
/** status: listening (local recognition); speaking: her
|
|
30
|
+
/** status: listening (local recognition); speaking: the service of her voice; ming: the local Ming voice and her own voices. */
|
|
29
31
|
voice: {
|
|
30
32
|
status: string;
|
|
31
33
|
speaking: {
|
|
@@ -35,6 +37,11 @@ export interface StateView {
|
|
|
35
37
|
};
|
|
36
38
|
id: string;
|
|
37
39
|
speed: number;
|
|
40
|
+
ming: MingStatus & {
|
|
41
|
+
installed: boolean;
|
|
42
|
+
settings: MingSettings;
|
|
43
|
+
own: OwnVoice[];
|
|
44
|
+
};
|
|
38
45
|
};
|
|
39
46
|
body: {
|
|
40
47
|
connected: boolean;
|
|
@@ -73,6 +80,11 @@ export interface StateView {
|
|
|
73
80
|
text: string;
|
|
74
81
|
at: number;
|
|
75
82
|
};
|
|
83
|
+
/** Her last reply failed with no words (the model had no key, no quota …): why, and what to do. */
|
|
84
|
+
problem?: {
|
|
85
|
+
text: string;
|
|
86
|
+
at: number;
|
|
87
|
+
};
|
|
76
88
|
memory: {
|
|
77
89
|
facts: Fact[];
|
|
78
90
|
episodes: Episode[];
|
|
@@ -105,6 +117,30 @@ export interface ApiRuntime {
|
|
|
105
117
|
format: 'wav' | 'mp3';
|
|
106
118
|
durationMs: number;
|
|
107
119
|
}>;
|
|
120
|
+
/** The local Ming voice: install (progress shows in the state), uninstall, start again after errors, settings. */
|
|
121
|
+
mingInstall(): void;
|
|
122
|
+
mingUninstall(): void;
|
|
123
|
+
mingRestart(): Promise<void>;
|
|
124
|
+
mingSettings(patch: Partial<MingSettings>): void;
|
|
125
|
+
/** Three takes of a voice from its description, each kept as a draft to listen to. */
|
|
126
|
+
mingDesign(description: string): Promise<Array<{
|
|
127
|
+
id: string;
|
|
128
|
+
audio: string;
|
|
129
|
+
durationMs: number;
|
|
130
|
+
}>>;
|
|
131
|
+
/** A recording to clone (base64, its file name for the format): a draft, with what it says as recognized. */
|
|
132
|
+
mingClone(audio: string, name: string): Promise<{
|
|
133
|
+
id: string;
|
|
134
|
+
audio: string;
|
|
135
|
+
durationMs: number;
|
|
136
|
+
text: string;
|
|
137
|
+
}>;
|
|
138
|
+
/** Keep a draft as one of her voices (a clone needs consent and the recording's words). */
|
|
139
|
+
mingKeep(draft: string, name: string, change: {
|
|
140
|
+
text?: string;
|
|
141
|
+
consent?: boolean;
|
|
142
|
+
}): OwnVoice;
|
|
143
|
+
mingRemove(id: string): void;
|
|
108
144
|
startBody(): string;
|
|
109
145
|
/** Download and install the body app (or a newer one); progress shows in the state. */
|
|
110
146
|
installBody(): void;
|
|
@@ -31,4 +31,6 @@
|
|
|
31
31
|
*/
|
|
32
32
|
import type { Context } from '@deepseek-ai/cordis';
|
|
33
33
|
import type { HerConfig } from './config.ts';
|
|
34
|
+
/** What to tell you when her model failed: the likely fix, in plain words. */
|
|
35
|
+
export declare function explainModelError(message: string, model: string): string;
|
|
34
36
|
export declare function apply(ctx: Context, config: HerConfig): void;
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The Ming voice's model: mlx-community/Ming-3.0-tts-0.5B-8bit (MIT, from
|
|
3
|
+
* inclusionAI's Ming-3.0-tts-0.5B), pinned to one revision. A copy of the
|
|
4
|
+
* same files is on dsh-her's GitHub release `ming-model-1`, which is tried
|
|
5
|
+
* first (Hugging Face is slow or blocked in some places); each file is
|
|
6
|
+
* checked against its size and SHA-256 whichever source it came from.
|
|
7
|
+
*/
|
|
8
|
+
export declare const MING_MODEL: {
|
|
9
|
+
readonly repo: "mlx-community/Ming-3.0-tts-0.5B-8bit";
|
|
10
|
+
readonly revision: "d0d1da6f627e4b3f20cc2db6ba4aee86d45343ad";
|
|
11
|
+
readonly sources: readonly ["https://github.com/AIcivilization/dsh-her/releases/download/ming-model-1/", "https://huggingface.co/mlx-community/Ming-3.0-tts-0.5B-8bit/resolve/d0d1da6f627e4b3f20cc2db6ba4aee86d45343ad/"];
|
|
12
|
+
readonly files: readonly [{
|
|
13
|
+
readonly name: "LICENSE";
|
|
14
|
+
readonly size: 1067;
|
|
15
|
+
readonly sha256: "2872f1918dfc04fdeb7f5d66d8d1b773594c9375e7c8f7bda39780bb97ad4fba";
|
|
16
|
+
}, {
|
|
17
|
+
readonly name: "README.md";
|
|
18
|
+
readonly size: 4360;
|
|
19
|
+
readonly sha256: "0af63591f3ab08d8e057330477928692ce8a4234618aacd4f2a0095c5d50bda9";
|
|
20
|
+
}, {
|
|
21
|
+
readonly name: "added_tokens.json";
|
|
22
|
+
readonly size: 1785;
|
|
23
|
+
readonly sha256: "34394b1e16d7da2db16618bd43a0f970902ade0075480bcd812fe62d559ec319";
|
|
24
|
+
}, {
|
|
25
|
+
readonly name: "campplus.safetensors";
|
|
26
|
+
readonly size: 27699096;
|
|
27
|
+
readonly sha256: "a479586bd8f8f96cdb414eaa2fa07c6cd54d6d1636562e2d75a0211cb3b0a6fd";
|
|
28
|
+
}, {
|
|
29
|
+
readonly name: "config.json";
|
|
30
|
+
readonly size: 6087;
|
|
31
|
+
readonly sha256: "8ea3e685a870fd1a535d3b4a4c975fc20c4fca56e67acdef06fe40004064c73b";
|
|
32
|
+
}, {
|
|
33
|
+
readonly name: "merges.txt";
|
|
34
|
+
readonly size: 1671853;
|
|
35
|
+
readonly sha256: "8831e4f1a044471340f7c0a83d7bd71306a5b867e95fd870f74d0c5308a904d5";
|
|
36
|
+
}, {
|
|
37
|
+
readonly name: "model.safetensors";
|
|
38
|
+
readonly size: 1573366591;
|
|
39
|
+
readonly sha256: "e88a4b53cb5b705703e961389f3d7f0e8b72c240b4dad796bf9961cc018971fc";
|
|
40
|
+
}, {
|
|
41
|
+
readonly name: "model.safetensors.index.json";
|
|
42
|
+
readonly size: 201850;
|
|
43
|
+
readonly sha256: "f59991abebed1a4f798a9488568af496d925e6eea507450aa8051a06f475a2ce";
|
|
44
|
+
}, {
|
|
45
|
+
readonly name: "special_tokens_map.json";
|
|
46
|
+
readonly size: 2726;
|
|
47
|
+
readonly sha256: "2c2ec51b9da5db654ac20de8310350d0c9c1a65918ae0181b80dacd2919b4fc3";
|
|
48
|
+
}, {
|
|
49
|
+
readonly name: "tokenizer.json";
|
|
50
|
+
readonly size: 11431389;
|
|
51
|
+
readonly sha256: "87d54ca3b44ea2f822ba39a0a45cb1b61644bd6606d9d3536dea75367bffb06d";
|
|
52
|
+
}, {
|
|
53
|
+
readonly name: "tokenizer_config.json";
|
|
54
|
+
readonly size: 16260;
|
|
55
|
+
readonly sha256: "12d6162c815edef24f8606e65b60d43fe0d92b0a4e150a0db8fa01c523aaf36a";
|
|
56
|
+
}, {
|
|
57
|
+
readonly name: "vocab.json";
|
|
58
|
+
readonly size: 2776833;
|
|
59
|
+
readonly sha256: "ca10d7e9fb3ed18575dd1e277a2579c16d108e32f27439684afa0e10b1440910";
|
|
60
|
+
}];
|
|
61
|
+
};
|
|
62
|
+
export declare const MING_MODEL_BYTES: number;
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Her own Ming voices (docs/Ming本地声音设计.md §4), in `<home>/voices/ming/`:
|
|
3
|
+
* `<id>.wav` (the reference recording) and `<id>.json`. Ming says every
|
|
4
|
+
* sentence in such a voice by cloning that recording.
|
|
5
|
+
*
|
|
6
|
+
* - designed: made from a description. The same description gives a
|
|
7
|
+
* slightly different voice every time, so a few candidates are made, one is
|
|
8
|
+
* kept, and from then on she clones it;
|
|
9
|
+
* - cloned: from a recording you add, only with your word that it is your
|
|
10
|
+
* own voice or that you may use it. It never leaves this computer.
|
|
11
|
+
*
|
|
12
|
+
* Candidates not kept yet wait in `drafts/` and are cleared after a day.
|
|
13
|
+
* The voice settings (quality, moods, idle minutes) are in `settings.json`.
|
|
14
|
+
*/
|
|
15
|
+
export interface OwnVoice {
|
|
16
|
+
id: string;
|
|
17
|
+
name: string;
|
|
18
|
+
kind: 'design' | 'clone';
|
|
19
|
+
/** What the recording says (Ming needs it to clone). */
|
|
20
|
+
text: string;
|
|
21
|
+
/** A designed voice: the description it was made from. */
|
|
22
|
+
description?: string;
|
|
23
|
+
createdAt: string;
|
|
24
|
+
}
|
|
25
|
+
export interface MingSettings {
|
|
26
|
+
/** quality: 10 steps (about 2.9 s for 25 characters on an M1 Pro); speed: 5 steps (about 1.7 s). */
|
|
27
|
+
quality: 'quality' | 'speed';
|
|
28
|
+
/** Her moods change the voice. */
|
|
29
|
+
moods: boolean;
|
|
30
|
+
/** Minutes without a sentence before Ming stops to free its memory; 0: never. Unset: by the Mac's memory. */
|
|
31
|
+
idleMinutes?: number;
|
|
32
|
+
}
|
|
33
|
+
export declare const DEFAULT_MING_SETTINGS: MingSettings;
|
|
34
|
+
export declare const MING_STEPS: Record<MingSettings['quality'], number>;
|
|
35
|
+
/** A clone reference: long enough to carry the voice, short enough to keep each sentence quick. */
|
|
36
|
+
export declare const CLONE_MIN_S = 3;
|
|
37
|
+
export declare const CLONE_MAX_S = 15;
|
|
38
|
+
interface Draft {
|
|
39
|
+
kind: OwnVoice['kind'];
|
|
40
|
+
text: string;
|
|
41
|
+
description?: string;
|
|
42
|
+
}
|
|
43
|
+
export declare class OwnVoices {
|
|
44
|
+
private readonly dir;
|
|
45
|
+
constructor(dir: string);
|
|
46
|
+
list(): OwnVoice[];
|
|
47
|
+
/** The recording Ming clones for a voice. */
|
|
48
|
+
ref(id: string): {
|
|
49
|
+
wav: string;
|
|
50
|
+
text: string;
|
|
51
|
+
} | undefined;
|
|
52
|
+
/** Keep a candidate for a while (until it is kept, or a day). */
|
|
53
|
+
addDraft(wav: Uint8Array, draft: Draft): string;
|
|
54
|
+
/** Keep a candidate as one of her voices. A clone needs your consent and what the recording says. */
|
|
55
|
+
keep(draftId: string, name: string, change?: {
|
|
56
|
+
text?: string;
|
|
57
|
+
consent?: boolean;
|
|
58
|
+
}): OwnVoice;
|
|
59
|
+
remove(id: string): void;
|
|
60
|
+
settings(): MingSettings;
|
|
61
|
+
saveSettings(patch: Partial<MingSettings>): MingSettings;
|
|
62
|
+
}
|
|
63
|
+
/**
|
|
64
|
+
* A recording you add, made ready to clone: converted by macOS's afconvert
|
|
65
|
+
* (wav, m4a from Voice Memos, mp3, aiff …) to 44.1 kHz mono, quiet start and
|
|
66
|
+
* end cut off, at most CLONE_MAX_S. Also at 16 kHz, to recognize what it says.
|
|
67
|
+
*/
|
|
68
|
+
export declare function prepareRecording(bytes: Uint8Array, name: string): Promise<{
|
|
69
|
+
wav: Uint8Array;
|
|
70
|
+
pcm16k: Uint8Array;
|
|
71
|
+
seconds: number;
|
|
72
|
+
}>;
|
|
73
|
+
/** Cut quiet 10 ms frames off both ends, keeping 0.1 s around the speech. */
|
|
74
|
+
export declare function trimSilence(samples: Int16Array, rate: number): Int16Array;
|
|
75
|
+
export {};
|
|
@@ -0,0 +1,103 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Ming-3.0-tts on this computer (docs/Ming本地声音设计.md): offline, free,
|
|
3
|
+
* with moods, and voices of your own. Apple chips with macOS 14+ only (MLX).
|
|
4
|
+
*
|
|
5
|
+
* - Install (on request, from the "她" page): uv (voice/uv.ts) builds a
|
|
6
|
+
* Python 3.12 environment in `<home>/ming/env` and installs mlx-audio from
|
|
7
|
+
* python/ming-requirements.txt, every package pinned with its hash; the
|
|
8
|
+
* model (voice/ming-model.ts) is downloaded into `<home>/ming/model`, each
|
|
9
|
+
* file checked against its size and SHA-256. An interrupted download goes
|
|
10
|
+
* on where it stopped. Deleting `<home>/ming` uninstalls it.
|
|
11
|
+
* - Run: python/ming_worker.py, one process that keeps the model loaded
|
|
12
|
+
* (about 2 GB of memory); sentences go to it one at a time. It starts with
|
|
13
|
+
* the first sentence (or when you start talking, `warm`), and stops after
|
|
14
|
+
* some minutes without a sentence to give the memory back.
|
|
15
|
+
* - A worker that dies is started again with the next sentence; one that dies
|
|
16
|
+
* twice within five minutes is left stopped (`failed`) until `restart`.
|
|
17
|
+
*/
|
|
18
|
+
export type MingState = 'unsupported' | 'absent' | 'installing' | 'stopped' | 'loading' | 'ready' | 'failed';
|
|
19
|
+
export interface MingStatus {
|
|
20
|
+
state: MingState;
|
|
21
|
+
/** While installing: what is being done, and bytes of the model so far. */
|
|
22
|
+
step?: string;
|
|
23
|
+
done?: number;
|
|
24
|
+
total?: number;
|
|
25
|
+
error?: string;
|
|
26
|
+
}
|
|
27
|
+
export interface MingRequest {
|
|
28
|
+
text: string;
|
|
29
|
+
/** A built-in voice (灵风信_高兴 …), a recording to clone, or a description to design a voice from. */
|
|
30
|
+
voice?: string;
|
|
31
|
+
ref?: {
|
|
32
|
+
wav: string;
|
|
33
|
+
text: string;
|
|
34
|
+
};
|
|
35
|
+
design?: string;
|
|
36
|
+
emotion?: string;
|
|
37
|
+
speed?: string;
|
|
38
|
+
steps: number;
|
|
39
|
+
seed?: number;
|
|
40
|
+
}
|
|
41
|
+
export interface MingAudio {
|
|
42
|
+
/** WAV, 16-bit mono. */
|
|
43
|
+
audio: Uint8Array;
|
|
44
|
+
durationMs: number;
|
|
45
|
+
ms: number;
|
|
46
|
+
}
|
|
47
|
+
type Logger = {
|
|
48
|
+
info(message: string): void;
|
|
49
|
+
warn(message: string): void;
|
|
50
|
+
};
|
|
51
|
+
/** Why Ming cannot run here, or undefined when it can: Apple chips and macOS 14 (Darwin 23) or later. */
|
|
52
|
+
export declare function mingUnsupported(platform?: string, arch?: string, darwin?: string): string | undefined;
|
|
53
|
+
/** Minutes without a sentence before the worker stops: 10 on a Mac with 16 GB or less, else 30. */
|
|
54
|
+
export declare const defaultIdleMinutes: (memory?: number) => number;
|
|
55
|
+
export declare class MingVoice {
|
|
56
|
+
private readonly root;
|
|
57
|
+
private readonly uvHome;
|
|
58
|
+
private readonly logger;
|
|
59
|
+
private state;
|
|
60
|
+
private worker;
|
|
61
|
+
private starting;
|
|
62
|
+
private installing;
|
|
63
|
+
private readonly pending;
|
|
64
|
+
private queue;
|
|
65
|
+
private nextId;
|
|
66
|
+
private lastUse;
|
|
67
|
+
private crashes;
|
|
68
|
+
private stderr;
|
|
69
|
+
private idleMinutes;
|
|
70
|
+
private readonly idleTimer;
|
|
71
|
+
/** `root`: `<home>/ming`; `uvHome`: `<home>/uv`, shared with the Edge voice. `idleMinutes` 0: never stop. */
|
|
72
|
+
constructor(root: string, uvHome: string, logger: Logger, idleMinutes?: number);
|
|
73
|
+
get status(): MingStatus;
|
|
74
|
+
get installed(): boolean;
|
|
75
|
+
setIdleMinutes(minutes: number): void;
|
|
76
|
+
private get python();
|
|
77
|
+
private get modelDir();
|
|
78
|
+
/** Install everything (once; a second call joins the first). Progress shows in `status`. */
|
|
79
|
+
install(): Promise<void>;
|
|
80
|
+
private doInstall;
|
|
81
|
+
/** Every model file, checked; files already checked (listed in `.verified`) are kept. */
|
|
82
|
+
private downloadModel;
|
|
83
|
+
/** Stop and delete everything (about 2 GB). */
|
|
84
|
+
uninstall(): void;
|
|
85
|
+
/** You started talking: have the model loaded by the time she answers. */
|
|
86
|
+
warm(): void;
|
|
87
|
+
/** After `failed`: forget the crashes and start again. */
|
|
88
|
+
restart(): Promise<void>;
|
|
89
|
+
/** One sentence. Sentences are said one at a time, in the order asked. */
|
|
90
|
+
synthesize(request: MingRequest): Promise<MingAudio>;
|
|
91
|
+
stop(): void;
|
|
92
|
+
/** Stop for good (the plugin is unloading). */
|
|
93
|
+
close(): void;
|
|
94
|
+
private speakNow;
|
|
95
|
+
private ensure;
|
|
96
|
+
private start;
|
|
97
|
+
private send;
|
|
98
|
+
private onReply;
|
|
99
|
+
private onExit;
|
|
100
|
+
private failPending;
|
|
101
|
+
private stopWhenIdle;
|
|
102
|
+
}
|
|
103
|
+
export {};
|
|
@@ -36,8 +36,10 @@ export declare class SpeechQueue {
|
|
|
36
36
|
private readonly sent;
|
|
37
37
|
private readonly synth;
|
|
38
38
|
private readonly out;
|
|
39
|
-
|
|
40
|
-
|
|
39
|
+
/** `synth` gets what to say and the sentence's cues (its emotion colours the voice). */
|
|
40
|
+
constructor(synth: (text: string, cues: Cue[]) => Promise<Synthesized | undefined>, out: SpeechOut);
|
|
41
|
+
/** `spoken`: what to say when it differs from the subtitle `text` (sound markers for Ming). */
|
|
42
|
+
say(uid: string, seq: number, text: string, cues?: Cue[], spoken?: string): void;
|
|
41
43
|
end(uid: string): void;
|
|
42
44
|
cancel(uid: string): void;
|
|
43
45
|
/** Sentences of a reply that went out, in order. */
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Running Python tools without asking the computer for a Python: uv
|
|
3
|
+
* (https://github.com/astral-sh/uv), one pinned, checksummed download kept
|
|
4
|
+
* in `<home>/uv`, fetches a Python of its own into the same folder and builds
|
|
5
|
+
* virtual environments with it. No dialogs, nothing system-wide. Used by the
|
|
6
|
+
* Edge voice (when the computer has no usable Python) and the Ming voice.
|
|
7
|
+
*/
|
|
8
|
+
export declare function run(cmd: string, args: string[], timeout: number, env?: NodeJS.ProcessEnv): Promise<string>;
|
|
9
|
+
/** What uv needs to keep its Python and downloads inside `uvHome`. */
|
|
10
|
+
export declare function uvEnv(uvHome: string): NodeJS.ProcessEnv;
|
|
11
|
+
/** The uv binary in `uvHome`, downloaded and checked the first time. */
|
|
12
|
+
export declare function installUv(uvHome: string): Promise<string>;
|
|
@@ -5,8 +5,9 @@
|
|
|
5
5
|
* process, engine.ts). Its model is downloaded in the background on first
|
|
6
6
|
* use; until then she only reads what you type.
|
|
7
7
|
* - Speaking: the service of the voice she uses (core/voices.ts): Microsoft
|
|
8
|
-
* Edge voices through the edge-tts tool (edge.ts, the default),
|
|
9
|
-
* Cloud Model Studio CosyVoice (cosyvoice.ts)
|
|
8
|
+
* Edge voices through the edge-tts tool (edge.ts, the default), Alibaba
|
|
9
|
+
* Cloud Model Studio CosyVoice (cosyvoice.ts), or Ming on this computer
|
|
10
|
+
* (ming.ts; built-in voices or her own, with her moods). When the service cannot
|
|
10
11
|
* speak (tool not installed yet, no key, no quota, offline), a sentence goes
|
|
11
12
|
* out without audio and shows as text; a key added later is picked up
|
|
12
13
|
* without restarting.
|
|
@@ -14,6 +15,9 @@
|
|
|
14
15
|
* answers start at once.
|
|
15
16
|
*/
|
|
16
17
|
import { type VoiceService } from '../core/index.ts';
|
|
18
|
+
import type { Cue } from '../protocol/index.ts';
|
|
19
|
+
import type { MingVoice } from './ming.ts';
|
|
20
|
+
import { type OwnVoices } from './ming-voices.ts';
|
|
17
21
|
import type { Synthesized } from './speech-queue.ts';
|
|
18
22
|
export interface VoiceConfig {
|
|
19
23
|
/** false: she only writes (text only). */
|
|
@@ -40,12 +44,16 @@ export declare class Voice {
|
|
|
40
44
|
private keyCheckedAt;
|
|
41
45
|
private readonly cloud;
|
|
42
46
|
private readonly edge;
|
|
47
|
+
private readonly ming;
|
|
43
48
|
private readonly cache;
|
|
44
49
|
private readonly modelsDir;
|
|
45
50
|
private config;
|
|
46
51
|
private readonly logger;
|
|
47
|
-
/** `edgeDir`: where dsh-her installs its own edge-tts if none is found. */
|
|
48
|
-
constructor(modelsDir: string, edgeDir: string, config: VoiceConfig, logger: Logger
|
|
52
|
+
/** `edgeDir`: where dsh-her installs its own edge-tts if none is found. `ming`: the local Ming voice and her own voices. */
|
|
53
|
+
constructor(modelsDir: string, edgeDir: string, config: VoiceConfig, logger: Logger, ming?: {
|
|
54
|
+
voice: MingVoice;
|
|
55
|
+
own: OwnVoices;
|
|
56
|
+
});
|
|
49
57
|
/** Listening. */
|
|
50
58
|
get status(): VoiceStatus;
|
|
51
59
|
get speaking(): {
|
|
@@ -61,8 +69,8 @@ export declare class Voice {
|
|
|
61
69
|
setVoice(voice: string, speed: number): void;
|
|
62
70
|
/** One sentence in a given voice, for trying voices; not cached. Throws with a message for the page. */
|
|
63
71
|
preview(text: string, voice: string, speed: number): Promise<Synthesized>;
|
|
64
|
-
/** Her sentence as audio, or undefined when she cannot speak right now (it shows as text). */
|
|
65
|
-
synthesize(text: string): Promise<Synthesized | undefined>;
|
|
72
|
+
/** Her sentence as audio, or undefined when she cannot speak right now (it shows as text). Its emotion colours a Ming voice. */
|
|
73
|
+
synthesize(text: string, cues?: Cue[]): Promise<Synthesized | undefined>;
|
|
66
74
|
/** What you said, from 16-bit mono PCM. Empty when nothing was understood. */
|
|
67
75
|
recognize(pcm16: Uint8Array, sampleRate: number): Promise<{
|
|
68
76
|
text: string;
|
package/lib/voice/edge.js
CHANGED
|
@@ -19,26 +19,16 @@
|
|
|
19
19
|
* measured on 2026-10-29); the speech queue synthesizes ahead to hide it.
|
|
20
20
|
* Needs the internet; not an official Microsoft API, so it may stop working.
|
|
21
21
|
*/
|
|
22
|
-
import {
|
|
23
|
-
import { createHash, randomUUID } from 'node:crypto';
|
|
22
|
+
import { randomUUID } from 'node:crypto';
|
|
24
23
|
import { existsSync } from 'node:fs';
|
|
25
|
-
import {
|
|
24
|
+
import { readFile, rm } from 'node:fs/promises';
|
|
26
25
|
import { homedir, tmpdir } from 'node:os';
|
|
27
26
|
import { dirname, join } from 'node:path';
|
|
27
|
+
import { installUv, run, uvEnv } from "./uv.js";
|
|
28
28
|
/** audio-24khz-48kbitrate-mono-mp3: constant bitrate, so bytes give the duration. */
|
|
29
29
|
const MP3_BYTES_PER_MS = 48_000 / 8 / 1000;
|
|
30
30
|
const SPEAK_TIMEOUT_MS = 20_000;
|
|
31
31
|
const win = process.platform === 'win32';
|
|
32
|
-
function run(cmd, args, timeout, env) {
|
|
33
|
-
return new Promise((resolve, reject) => {
|
|
34
|
-
execFile(cmd, args, { timeout, maxBuffer: 8 << 20, ...(env === undefined ? {} : { env: { ...process.env, ...env } }) }, (error, stdout, stderr) => {
|
|
35
|
-
if (error !== null)
|
|
36
|
-
reject(new Error(String(stderr || error.message).trim().split('\n').slice(-2).join(' ')));
|
|
37
|
-
else
|
|
38
|
-
resolve(String(stdout));
|
|
39
|
-
});
|
|
40
|
-
});
|
|
41
|
-
}
|
|
42
32
|
const venvBin = (root, name) => join(root, win ? 'Scripts' : 'bin', win ? `${name}.exe` : name);
|
|
43
33
|
/** `--rate` for a speed factor: 1.2 → "+20%". */
|
|
44
34
|
export function edgeRate(speed) {
|
|
@@ -122,7 +112,7 @@ export class EdgeVoice {
|
|
|
122
112
|
// No Python here: uv brings one of its own, kept next to edge-tts.
|
|
123
113
|
const uvHome = join(dirname(this.ownDir), 'uv');
|
|
124
114
|
const uv = await installUv(uvHome);
|
|
125
|
-
const env =
|
|
115
|
+
const env = uvEnv(uvHome);
|
|
126
116
|
if (!existsSync(venvPython))
|
|
127
117
|
await run(uv, ['venv', '--quiet', '--python', '3.12', this.ownDir], 600_000, env);
|
|
128
118
|
await run(uv, ['pip', 'install', '--quiet', '--python', venvPython, '--upgrade', 'edge-tts'], 300_000, env);
|
|
@@ -163,37 +153,3 @@ async function findPython() {
|
|
|
163
153
|
}
|
|
164
154
|
return undefined;
|
|
165
155
|
}
|
|
166
|
-
/** uv, pinned: version and the SHA-256 of each download (from the release's .sha256 files). */
|
|
167
|
-
const UV_VERSION = '0.11.26';
|
|
168
|
-
const UV_ASSETS = {
|
|
169
|
-
'darwin-arm64': { file: 'uv-aarch64-apple-darwin.tar.gz', sha256: '8f7fbf1708399b921857bce71e1d60f0d3ccf52a30caebc1c1a2f175dce13ab6' },
|
|
170
|
-
'darwin-x64': { file: 'uv-x86_64-apple-darwin.tar.gz', sha256: '922b460202707dd5f4ccacbadbe7f6a546cc46e82a99bf50ca99a7977a78eddd' },
|
|
171
|
-
'win32-x64': { file: 'uv-x86_64-pc-windows-msvc.zip', sha256: '4e1278ede866be6c0bf32d2f466cc6de7a9fb399ecf20c9ce2d186e52424be47' },
|
|
172
|
-
};
|
|
173
|
-
/** The uv binary in `uvHome`, downloaded and checked the first time. */
|
|
174
|
-
async function installUv(uvHome) {
|
|
175
|
-
const key = `${process.platform}-${process.arch}`;
|
|
176
|
-
const asset = UV_ASSETS[key];
|
|
177
|
-
if (asset === undefined)
|
|
178
|
-
throw new Error(`这台电脑(${key})没有 Python,也没有现成的办法自动装:请先装 Python 3`);
|
|
179
|
-
const dir = join(uvHome, UV_VERSION);
|
|
180
|
-
const found = [join(dir, 'uv'), join(dir, 'uv.exe'), join(dir, asset.file.replace(/\.(tar\.gz|zip)$/, ''), 'uv')].find(p => existsSync(p));
|
|
181
|
-
if (found !== undefined)
|
|
182
|
-
return found;
|
|
183
|
-
const res = await fetch(`https://github.com/astral-sh/uv/releases/download/${UV_VERSION}/${asset.file}`, { redirect: 'follow', signal: AbortSignal.timeout(300_000) });
|
|
184
|
-
if (!res.ok)
|
|
185
|
-
throw new Error(`下载 uv 失败(${res.status})`);
|
|
186
|
-
const bytes = Buffer.from(await res.arrayBuffer());
|
|
187
|
-
if (createHash('sha256').update(bytes).digest('hex') !== asset.sha256)
|
|
188
|
-
throw new Error('下载的 uv 校验不对,没有使用');
|
|
189
|
-
await mkdir(dir, { recursive: true });
|
|
190
|
-
const archive = join(dir, asset.file);
|
|
191
|
-
await writeFile(archive, bytes);
|
|
192
|
-
// tar reads both the .tar.gz and (Windows 10+) the .zip.
|
|
193
|
-
await run('tar', ['-xf', archive, '-C', dir], 120_000);
|
|
194
|
-
await rm(archive, { force: true });
|
|
195
|
-
const uv = [join(dir, 'uv'), join(dir, 'uv.exe'), join(dir, asset.file.replace(/\.(tar\.gz|zip)$/, ''), 'uv')].find(p => existsSync(p));
|
|
196
|
-
if (uv === undefined)
|
|
197
|
-
throw new Error('uv 解压后找不到');
|
|
198
|
-
return uv;
|
|
199
|
-
}
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The Ming voice's model: mlx-community/Ming-3.0-tts-0.5B-8bit (MIT, from
|
|
3
|
+
* inclusionAI's Ming-3.0-tts-0.5B), pinned to one revision. A copy of the
|
|
4
|
+
* same files is on dsh-her's GitHub release `ming-model-1`, which is tried
|
|
5
|
+
* first (Hugging Face is slow or blocked in some places); each file is
|
|
6
|
+
* checked against its size and SHA-256 whichever source it came from.
|
|
7
|
+
*/
|
|
8
|
+
export const MING_MODEL = {
|
|
9
|
+
repo: 'mlx-community/Ming-3.0-tts-0.5B-8bit',
|
|
10
|
+
revision: 'd0d1da6f627e4b3f20cc2db6ba4aee86d45343ad',
|
|
11
|
+
sources: [
|
|
12
|
+
'https://github.com/AIcivilization/dsh-her/releases/download/ming-model-1/',
|
|
13
|
+
'https://huggingface.co/mlx-community/Ming-3.0-tts-0.5B-8bit/resolve/d0d1da6f627e4b3f20cc2db6ba4aee86d45343ad/',
|
|
14
|
+
],
|
|
15
|
+
files: [
|
|
16
|
+
{ name: 'LICENSE', size: 1067, sha256: '2872f1918dfc04fdeb7f5d66d8d1b773594c9375e7c8f7bda39780bb97ad4fba' },
|
|
17
|
+
{ name: 'README.md', size: 4360, sha256: '0af63591f3ab08d8e057330477928692ce8a4234618aacd4f2a0095c5d50bda9' },
|
|
18
|
+
{ name: 'added_tokens.json', size: 1785, sha256: '34394b1e16d7da2db16618bd43a0f970902ade0075480bcd812fe62d559ec319' },
|
|
19
|
+
{ name: 'campplus.safetensors', size: 27699096, sha256: 'a479586bd8f8f96cdb414eaa2fa07c6cd54d6d1636562e2d75a0211cb3b0a6fd' },
|
|
20
|
+
{ name: 'config.json', size: 6087, sha256: '8ea3e685a870fd1a535d3b4a4c975fc20c4fca56e67acdef06fe40004064c73b' },
|
|
21
|
+
{ name: 'merges.txt', size: 1671853, sha256: '8831e4f1a044471340f7c0a83d7bd71306a5b867e95fd870f74d0c5308a904d5' },
|
|
22
|
+
{ name: 'model.safetensors', size: 1573366591, sha256: 'e88a4b53cb5b705703e961389f3d7f0e8b72c240b4dad796bf9961cc018971fc' },
|
|
23
|
+
{ name: 'model.safetensors.index.json', size: 201850, sha256: 'f59991abebed1a4f798a9488568af496d925e6eea507450aa8051a06f475a2ce' },
|
|
24
|
+
{ name: 'special_tokens_map.json', size: 2726, sha256: '2c2ec51b9da5db654ac20de8310350d0c9c1a65918ae0181b80dacd2919b4fc3' },
|
|
25
|
+
{ name: 'tokenizer.json', size: 11431389, sha256: '87d54ca3b44ea2f822ba39a0a45cb1b61644bd6606d9d3536dea75367bffb06d' },
|
|
26
|
+
{ name: 'tokenizer_config.json', size: 16260, sha256: '12d6162c815edef24f8606e65b60d43fe0d92b0a4e150a0db8fa01c523aaf36a' },
|
|
27
|
+
{ name: 'vocab.json', size: 2776833, sha256: 'ca10d7e9fb3ed18575dd1e277a2579c16d108e32f27439684afa0e10b1440910' },
|
|
28
|
+
],
|
|
29
|
+
};
|
|
30
|
+
export const MING_MODEL_BYTES = MING_MODEL.files.reduce((sum, f) => sum + f.size, 0);
|