@tychenjiajun/pi-voice 0.7.0 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +23 -1
- package/catalog/catalog.json +1501 -0
- package/catalog/recommendations.json +67 -27
- package/models/README.md +25 -0
- package/models/silero-vad-v6.2-F32.gguf +0 -0
- package/package.json +8 -6
- package/src/audio-constants.ts +2 -0
- package/src/audio.ts +2 -5
- package/src/catalog.ts +6 -2
- package/src/chinese.ts +17 -6
- package/src/chunking.ts +319 -0
- package/src/dictation-controller.ts +61 -18
- package/src/file-audio.ts +2 -2
- package/src/file-transcription.ts +6 -3
- package/src/language-detection-policy.ts +127 -0
- package/src/languages.ts +4 -3
- package/src/model-activation.ts +50 -4
- package/src/model-cells.ts +28 -6
- package/src/model-picker.ts +29 -12
- package/src/model-selection-controller.ts +14 -9
- package/src/models.ts +15 -1
- package/src/onboarding.ts +13 -1
- package/src/pcm-chunker.ts +14 -4
- package/src/recommendation-picker.ts +9 -7
- package/src/recommendations.ts +4 -2
- package/src/runtime.ts +9 -1
- package/src/settings-menu.ts +40 -9
- package/src/settings.ts +35 -3
- package/src/transcription-service.ts +205 -37
- package/src/transcription.ts +72 -24
- package/src/try-it.ts +11 -2
- package/src/vad.ts +301 -0
- package/src/catalog.generated.ts +0 -1305
package/src/languages.ts
CHANGED
|
@@ -3,9 +3,10 @@ export function canonicalLanguage(language: string): string {
|
|
|
3
3
|
return language.trim().toLowerCase().split("-", 1)[0] ?? "";
|
|
4
4
|
}
|
|
5
5
|
|
|
6
|
-
// Model cards and
|
|
7
|
-
//
|
|
8
|
-
|
|
6
|
+
// Model cards, benchmarks, and the language detector use different ISO codes
|
|
7
|
+
// for these languages (iw/jw are legacy codes for Hebrew and Javanese). This
|
|
8
|
+
// identity is for matching, never a code to send directly to a backend.
|
|
9
|
+
const LANGUAGE_ALIASES: Readonly<Record<string, string>> = { tl: "fil", no: "nb", iw: "he", jw: "jv" };
|
|
9
10
|
|
|
10
11
|
export function languageIdentity(language: string): string {
|
|
11
12
|
const base = canonicalLanguage(language);
|
package/src/model-activation.ts
CHANGED
|
@@ -1,13 +1,17 @@
|
|
|
1
|
-
import type
|
|
2
|
-
import {
|
|
1
|
+
import { LANGUAGE_DETECTION_MODEL, displayLanguage, type CatalogModel } from "./catalog.js";
|
|
2
|
+
import { settingsLanguagePolicy } from "./language-detection-policy.js";
|
|
3
|
+
import { describeError, log } from "./log.js";
|
|
4
|
+
import { downloadCatalogModel, findCachedCatalogModel, pendingDownloads, type CachedCatalogModel } from "./models.js";
|
|
3
5
|
import { writeSettings, type TranscribeSettings } from "./settings.js";
|
|
4
6
|
|
|
7
|
+
export type ActivationProgress = { downloaded: number; total: number; message?: string };
|
|
8
|
+
|
|
5
9
|
export type CatalogModelActivation = (
|
|
6
10
|
model: CatalogModel,
|
|
7
11
|
options: {
|
|
8
12
|
cached: CachedCatalogModel | undefined;
|
|
9
13
|
signal: AbortSignal;
|
|
10
|
-
onProgress: (progress:
|
|
14
|
+
onProgress: (progress: ActivationProgress) => void;
|
|
11
15
|
},
|
|
12
16
|
) => Promise<{ path: string }>;
|
|
13
17
|
|
|
@@ -20,6 +24,7 @@ export type CatalogModelActivation = (
|
|
|
20
24
|
export function createModelActivation(options: {
|
|
21
25
|
buildSettings: (model: CatalogModel, path: string) => TranscribeSettings;
|
|
22
26
|
onCommitted: (settings: TranscribeSettings) => void;
|
|
27
|
+
onWarning: (message: string) => void;
|
|
23
28
|
}): {
|
|
24
29
|
activate: CatalogModelActivation;
|
|
25
30
|
/** Resolves once every commit enqueued so far has landed (or failed). */
|
|
@@ -32,6 +37,11 @@ export function createModelActivation(options: {
|
|
|
32
37
|
{ cached, signal, onProgress },
|
|
33
38
|
) => {
|
|
34
39
|
let path: string;
|
|
40
|
+
let detectorFailed = false;
|
|
41
|
+
const pending = pendingDownloads(model, Boolean(cached));
|
|
42
|
+
const detectorMissing = pending.includes(LANGUAGE_DETECTION_MODEL);
|
|
43
|
+
const asrBytes = cached ? 0 : model.size;
|
|
44
|
+
const total = pending.reduce((sum, item) => sum + item.size, 0);
|
|
35
45
|
if (cached) {
|
|
36
46
|
// The picker listed the cache when it opened; re-check so settings
|
|
37
47
|
// never point at a file that has since been evicted. Integrity is
|
|
@@ -44,9 +54,33 @@ export function createModelActivation(options: {
|
|
|
44
54
|
}
|
|
45
55
|
path = stillCached.path;
|
|
46
56
|
} else {
|
|
47
|
-
path = await downloadCatalogModel(model, {
|
|
57
|
+
path = await downloadCatalogModel(model, {
|
|
58
|
+
signal,
|
|
59
|
+
onProgress: (progress) => onProgress({ ...progress, total }),
|
|
60
|
+
});
|
|
48
61
|
}
|
|
49
62
|
|
|
63
|
+
if (detectorMissing) {
|
|
64
|
+
try {
|
|
65
|
+
onProgress({ downloaded: asrBytes, total, message: "Downloading language detector…" });
|
|
66
|
+
await downloadCatalogModel(LANGUAGE_DETECTION_MODEL, {
|
|
67
|
+
signal,
|
|
68
|
+
onProgress: (progress) => onProgress({
|
|
69
|
+
downloaded: asrBytes + progress.downloaded, total,
|
|
70
|
+
message: "Downloading language detector…",
|
|
71
|
+
}),
|
|
72
|
+
});
|
|
73
|
+
} catch (error) {
|
|
74
|
+
// Cancellation is never a settings commit. Other failures still
|
|
75
|
+
// select the model: without the detector, Auto falls back to the
|
|
76
|
+
// first preferred language, and the first take says how to fix it.
|
|
77
|
+
signal.throwIfAborted();
|
|
78
|
+
log.warn(`Language detector download failed: ${describeError(error)}`);
|
|
79
|
+
detectorFailed = true;
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
let committed: TranscribeSettings | undefined;
|
|
50
84
|
const commit = commitQueue.then(async () => {
|
|
51
85
|
// Skip the write when this selection was superseded or its download
|
|
52
86
|
// was cancelled while the commit sat in the queue.
|
|
@@ -60,6 +94,7 @@ export function createModelActivation(options: {
|
|
|
60
94
|
`Could not save settings: ${error instanceof Error ? error.message : String(error)}`,
|
|
61
95
|
);
|
|
62
96
|
}
|
|
97
|
+
committed = settings;
|
|
63
98
|
options.onCommitted(settings);
|
|
64
99
|
});
|
|
65
100
|
commitQueue = commit.then(
|
|
@@ -67,6 +102,17 @@ export function createModelActivation(options: {
|
|
|
67
102
|
() => undefined,
|
|
68
103
|
);
|
|
69
104
|
await commit;
|
|
105
|
+
// Only Auto detect uses the detector, so other settings need no warning.
|
|
106
|
+
const policy = committed && settingsLanguagePolicy(committed);
|
|
107
|
+
if (detectorFailed && policy?.kind === "detect") {
|
|
108
|
+
try {
|
|
109
|
+
options.onWarning(
|
|
110
|
+
`Selected ${model.name}, but the language detector didn't download. ` +
|
|
111
|
+
`Auto detect will use ${displayLanguage(policy.languages[0]!)} until it does. ` +
|
|
112
|
+
"To retry the download, choose Auto detect in /voice-settings.",
|
|
113
|
+
);
|
|
114
|
+
} catch { /* Notification is not part of the save. */ }
|
|
115
|
+
}
|
|
70
116
|
return { path };
|
|
71
117
|
};
|
|
72
118
|
|
package/src/model-cells.ts
CHANGED
|
@@ -16,6 +16,8 @@ import {
|
|
|
16
16
|
type AccuracyLetter,
|
|
17
17
|
type RecommendationRole,
|
|
18
18
|
} from "./recommendations.js";
|
|
19
|
+
import { requiresLanguageDetector, undetectablePreferences } from "./language-detection-policy.js";
|
|
20
|
+
import { findIncompleteDownload, pendingDownloads } from "./models.js";
|
|
19
21
|
import { LIST_PADDING, padToWidth, selectionMarker } from "./ui-components.js";
|
|
20
22
|
|
|
21
23
|
type UiTheme = ExtensionContext["ui"]["theme"];
|
|
@@ -181,18 +183,16 @@ export function modelDetailText(
|
|
|
181
183
|
width: number,
|
|
182
184
|
padding: number,
|
|
183
185
|
feedback?: { type: "success" | "error" | "muted"; text: string },
|
|
184
|
-
showManualSelection = false,
|
|
185
186
|
): string {
|
|
186
187
|
const canonicalLanguages = [...new Set(model.languages.map(canonicalLanguage))];
|
|
187
188
|
const features = [
|
|
188
189
|
canonicalLanguages.length === 1
|
|
189
190
|
? `${displayLanguage(canonicalLanguages[0]!)} only`
|
|
190
191
|
: `${canonicalLanguages.length} languages`,
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
: undefined,
|
|
192
|
+
// Native detection, or the shared detector for multilingual models.
|
|
193
|
+
model.capabilities.languageDetection || requiresLanguageDetector(model)
|
|
194
|
+
? "automatic language detection"
|
|
195
|
+
: undefined,
|
|
196
196
|
].filter((value): value is string => Boolean(value));
|
|
197
197
|
const description = truncateToWidth(
|
|
198
198
|
model.description,
|
|
@@ -205,6 +205,28 @@ export function modelDetailText(
|
|
|
205
205
|
return `${description}\n${theme.fg("dim", features.join(" · "))}${feedbackText}`;
|
|
206
206
|
}
|
|
207
207
|
|
|
208
|
+
/** A picker's action for a model: "choose", or what is left to download. */
|
|
209
|
+
export function downloadAction(model: CatalogModel, modelCached: boolean): string {
|
|
210
|
+
const pending = pendingDownloads(model, modelCached);
|
|
211
|
+
if (!pending.length) return "choose";
|
|
212
|
+
const partial = pending.map(findIncompleteDownload);
|
|
213
|
+
const bytes = pending.reduce((sum, item, index) => sum + item.size - (partial[index]?.bytes ?? 0), 0);
|
|
214
|
+
return partial.some(Boolean)
|
|
215
|
+
? `resume download (${formatBinarySize(bytes)} left)`
|
|
216
|
+
: `download ${formatBinarySize(bytes)}`;
|
|
217
|
+
}
|
|
218
|
+
|
|
219
|
+
/** Why Auto detect is missing for these preferences, if a language is undetectable. */
|
|
220
|
+
export function autoUnavailableNotice(
|
|
221
|
+
model: CatalogModel,
|
|
222
|
+
preferredLanguages: readonly string[],
|
|
223
|
+
): string | undefined {
|
|
224
|
+
const skipped = undetectablePreferences(model, preferredLanguages).map(displayLanguage);
|
|
225
|
+
if (!skipped.length) return undefined;
|
|
226
|
+
const languages = new Intl.ListFormat("en", { type: "disjunction" }).format(skipped);
|
|
227
|
+
return `Auto detect is unavailable for your preferred languages because it can't detect ${languages}.`;
|
|
228
|
+
}
|
|
229
|
+
|
|
208
230
|
/**
|
|
209
231
|
* The catalog's search matching, shared by every page that lists models.
|
|
210
232
|
* Each word of the query must appear literally in the model's id, name,
|
package/src/model-picker.ts
CHANGED
|
@@ -31,6 +31,8 @@ import {
|
|
|
31
31
|
type ModelBenchmark,
|
|
32
32
|
} from "./recommendations.js";
|
|
33
33
|
import {
|
|
34
|
+
autoUnavailableNotice,
|
|
35
|
+
downloadAction,
|
|
34
36
|
MANUAL_LANGUAGE_TAG,
|
|
35
37
|
ON_DISK_LABEL,
|
|
36
38
|
modelDetailText,
|
|
@@ -41,7 +43,7 @@ import {
|
|
|
41
43
|
import { ModelSelectionController } from "./model-selection-controller.js";
|
|
42
44
|
import { ModelRatingsHelp } from "./model-ratings-help.js";
|
|
43
45
|
import type { CatalogModelActivation } from "./model-activation.js";
|
|
44
|
-
import {
|
|
46
|
+
import { autoLanguagePolicy } from "./language-detection-policy.js";
|
|
45
47
|
import { VoiceKeys } from "./keybindings.js";
|
|
46
48
|
import type { TranscriptionLanguage } from "./settings.js";
|
|
47
49
|
import {
|
|
@@ -656,8 +658,11 @@ export class CatalogModelPicker extends Container implements Focusable {
|
|
|
656
658
|
const feedbackLines = this.selection.feedback
|
|
657
659
|
? new Text(this.selection.feedback.text, TEXT_PADDING, 0).render(width).length
|
|
658
660
|
: 0;
|
|
659
|
-
|
|
660
|
-
|
|
661
|
+
const model = this.highlightedModel();
|
|
662
|
+
const notices = model ? this.modelNotices(model) : "";
|
|
663
|
+
const noticeLines = notices ? new Text(notices, TEXT_PADDING, 0).render(width).length : 0;
|
|
664
|
+
// Description, features, and the undetectable-language warning.
|
|
665
|
+
return 2 + feedbackLines + noticeLines;
|
|
661
666
|
}
|
|
662
667
|
|
|
663
668
|
private refresh(): void {
|
|
@@ -783,12 +788,17 @@ export class CatalogModelPicker extends Container implements Focusable {
|
|
|
783
788
|
this.renderWidth,
|
|
784
789
|
TEXT_PADDING,
|
|
785
790
|
this.selection.feedback,
|
|
786
|
-
),
|
|
791
|
+
) + this.modelNotices(selected),
|
|
787
792
|
);
|
|
788
793
|
}
|
|
789
794
|
this.finishFooter(query);
|
|
790
795
|
}
|
|
791
796
|
|
|
797
|
+
private modelNotices(model: CatalogModel): string {
|
|
798
|
+
const notice = autoUnavailableNotice(model, this.preferredLanguages);
|
|
799
|
+
return notice ? `\n${this.theme.fg("warning", notice)}` : "";
|
|
800
|
+
}
|
|
801
|
+
|
|
792
802
|
private finishFooter(query: string): void {
|
|
793
803
|
const displayedId = this.selection.displayedModelId;
|
|
794
804
|
const total = CATALOG_MODELS.length;
|
|
@@ -803,10 +813,8 @@ export class CatalogModelPicker extends Container implements Focusable {
|
|
|
803
813
|
const highlighted = this.highlightedModel();
|
|
804
814
|
const confirmLabel = this.rows[this.selectedIndex]?.type === "fold"
|
|
805
815
|
? this.folded ? "show" : "hide"
|
|
806
|
-
: highlighted
|
|
807
|
-
?
|
|
808
|
-
? "resume download"
|
|
809
|
-
: `download ${formatBinarySize(highlighted.size)}`
|
|
816
|
+
: highlighted
|
|
817
|
+
? downloadAction(highlighted, this.selection.cachedById.has(highlighted.id))
|
|
810
818
|
: "choose";
|
|
811
819
|
this.footer.setText(
|
|
812
820
|
`${this.theme.fg("dim", shown)} ${statusLegend} ${this.keys.hint("voice.models.ratingsHelp", "rating guide")}\n${this.keys.navHint("navigate")} ${this.keys.hint("tui.select.confirm", confirmLabel)} ${this.keys.hint("tui.select.cancel", closeLabel)}`,
|
|
@@ -1003,10 +1011,19 @@ export function createTranscriptionLanguagePicker(
|
|
|
1003
1011
|
left.label.localeCompare(right.label) ||
|
|
1004
1012
|
left.value.localeCompare(right.value),
|
|
1005
1013
|
);
|
|
1014
|
+
const policy = autoLanguagePolicy(model, preferredLanguages);
|
|
1015
|
+
const autoAvailable = policy.kind !== "unsupported";
|
|
1016
|
+
const auto: SingleSelectChoice<TranscriptionLanguage> = policy.kind === "detect"
|
|
1017
|
+
? {
|
|
1018
|
+
value: "auto",
|
|
1019
|
+
label: "Auto detect",
|
|
1020
|
+
description: `Detects which of your preferred languages you're speaking (${new Intl.ListFormat("en", { type: "disjunction" }).format(
|
|
1021
|
+
policy.languages.map((language) => transcriptionLanguageName(language, model.languages)),
|
|
1022
|
+
)}) in each recording.`,
|
|
1023
|
+
}
|
|
1024
|
+
: { value: "auto", label: "Auto detect" };
|
|
1006
1025
|
const choices: SingleSelectChoice<TranscriptionLanguage>[] = [
|
|
1007
|
-
...(
|
|
1008
|
-
? [{ value: "auto", label: "Auto detect" }]
|
|
1009
|
-
: []),
|
|
1026
|
+
...(autoAvailable ? [auto] : []),
|
|
1010
1027
|
...languages,
|
|
1011
1028
|
];
|
|
1012
1029
|
return new SingleSelectPicker(
|
|
@@ -1017,7 +1034,7 @@ export function createTranscriptionLanguagePicker(
|
|
|
1017
1034
|
current,
|
|
1018
1035
|
{
|
|
1019
1036
|
title: "Choose transcription language",
|
|
1020
|
-
subtitle:
|
|
1037
|
+
subtitle: autoAvailable
|
|
1021
1038
|
? "Language expected in recordings, or automatic detection."
|
|
1022
1039
|
: "Language expected in recordings.",
|
|
1023
1040
|
searchable: true,
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import type { CatalogModel } from "./catalog.js";
|
|
2
2
|
import type { CatalogModelActivation } from "./model-activation.js";
|
|
3
|
-
import { findCachedCatalogModel, type CachedCatalogModel } from "./models.js";
|
|
3
|
+
import { findCachedCatalogModel, pendingDownloads, type CachedCatalogModel } from "./models.js";
|
|
4
4
|
|
|
5
5
|
export type DownloadState = {
|
|
6
6
|
readonly model: CatalogModel;
|
|
@@ -94,7 +94,8 @@ export class ModelSelectionController<R> {
|
|
|
94
94
|
select(model: CatalogModel): void {
|
|
95
95
|
if (!this.acceptsInput || this.download) return;
|
|
96
96
|
const cached = this.cachedById.get(model.id);
|
|
97
|
-
|
|
97
|
+
const downloads = pendingDownloads(model, Boolean(cached)).length > 0;
|
|
98
|
+
if (!downloads && !this.target && model.id === this.committedModelId &&
|
|
98
99
|
(this.selectedDuringSession || !this.options.advance)) {
|
|
99
100
|
this.selectedDuringSession = true;
|
|
100
101
|
if (this.options.advance) this.finish(this.options.completion);
|
|
@@ -107,9 +108,9 @@ export class ModelSelectionController<R> {
|
|
|
107
108
|
this.target = target;
|
|
108
109
|
this.feedback = undefined;
|
|
109
110
|
this.samples = [];
|
|
110
|
-
this.download =
|
|
111
|
+
this.download = downloads ? {
|
|
111
112
|
model, downloaded: 0, total: 0, message: "Connecting to Hugging Face…",
|
|
112
|
-
};
|
|
113
|
+
} : undefined;
|
|
113
114
|
this.changed();
|
|
114
115
|
|
|
115
116
|
// Catch synchronous implementations too. The real pipeline reports only
|
|
@@ -119,15 +120,15 @@ export class ModelSelectionController<R> {
|
|
|
119
120
|
work = this.activate(model, {
|
|
120
121
|
cached,
|
|
121
122
|
signal: target.controller.signal,
|
|
122
|
-
onProgress: ({ downloaded, total }) => {
|
|
123
|
+
onProgress: ({ downloaded, total, message }) => {
|
|
123
124
|
if (this.disposed || this.closed || this.target !== target || target.controller.signal.aborted) return;
|
|
124
125
|
if (!this.download) return;
|
|
125
126
|
const firstReport = this.download.total === 0 && total > 0;
|
|
126
127
|
this.download = {
|
|
127
128
|
model, downloaded, total,
|
|
128
|
-
message: firstReport
|
|
129
|
+
message: message ?? (firstReport
|
|
129
130
|
? downloaded > 0 ? "Resuming download from Hugging Face…" : "Downloading from Hugging Face…"
|
|
130
|
-
: this.download.message,
|
|
131
|
+
: this.download.message),
|
|
131
132
|
};
|
|
132
133
|
this.samples.push({ time: this.now(), bytes: downloaded });
|
|
133
134
|
if (this.samples.length > 64) this.samples.shift();
|
|
@@ -147,9 +148,13 @@ export class ModelSelectionController<R> {
|
|
|
147
148
|
if (this.target !== target) { this.changed(); return; }
|
|
148
149
|
this.target = undefined;
|
|
149
150
|
this.download = undefined;
|
|
150
|
-
|
|
151
|
+
// A failed detector download still selects the model; its own warning explains.
|
|
152
|
+
const detectorDownloaded = cached && !pendingDownloads(model, true).length;
|
|
153
|
+
this.feedback = !cached ? {
|
|
151
154
|
type: "success", text: `✓ Downloaded and selected ${model.name}`,
|
|
152
|
-
}
|
|
155
|
+
} : detectorDownloaded && downloads ? {
|
|
156
|
+
type: "success", text: `✓ Selected ${model.name} and downloaded the language detector`,
|
|
157
|
+
} : undefined;
|
|
153
158
|
if (this.pendingExit) this.finish(this.pendingExit.result);
|
|
154
159
|
else if (this.options.advance) this.finish(this.options.completion);
|
|
155
160
|
else this.changed();
|
package/src/models.ts
CHANGED
|
@@ -21,7 +21,8 @@ import {
|
|
|
21
21
|
} from "node:fs/promises";
|
|
22
22
|
import { dirname, join, relative } from "node:path";
|
|
23
23
|
import type { ReadableStream } from "node:stream/web";
|
|
24
|
-
import type
|
|
24
|
+
import { LANGUAGE_DETECTION_MODEL, type CatalogModel } from "./catalog.js";
|
|
25
|
+
import { requiresLanguageDetector } from "./language-detection-policy.js";
|
|
25
26
|
|
|
26
27
|
function repositoryCacheDirectory(model: CatalogModel): string {
|
|
27
28
|
return join(
|
|
@@ -261,3 +262,16 @@ export function findIncompleteDownload(
|
|
|
261
262
|
}
|
|
262
263
|
return undefined;
|
|
263
264
|
}
|
|
265
|
+
|
|
266
|
+
/**
|
|
267
|
+
* What choosing `model` still has to download: the model unless it is cached,
|
|
268
|
+
* plus the shared language detector when the model relies on it and it is
|
|
269
|
+
* not cached either.
|
|
270
|
+
*/
|
|
271
|
+
export function pendingDownloads(model: CatalogModel, modelCached: boolean): CatalogModel[] {
|
|
272
|
+
const pending = modelCached ? [] : [model];
|
|
273
|
+
if (requiresLanguageDetector(model) && !findCachedCatalogModel(LANGUAGE_DETECTION_MODEL)) {
|
|
274
|
+
pending.push(LANGUAGE_DETECTION_MODEL);
|
|
275
|
+
}
|
|
276
|
+
return pending;
|
|
277
|
+
}
|
package/src/onboarding.ts
CHANGED
|
@@ -21,6 +21,7 @@ import {
|
|
|
21
21
|
DEFAULT_MICROPHONE,
|
|
22
22
|
settingsForModel,
|
|
23
23
|
settingsRemoteOnly,
|
|
24
|
+
settingsWithPreferredLanguages,
|
|
24
25
|
writeSettings,
|
|
25
26
|
type BackendSettings,
|
|
26
27
|
type ChineseOutput,
|
|
@@ -47,10 +48,12 @@ type SettingsOptions = NonNullable<Parameters<typeof settingsForModel>[2]>;
|
|
|
47
48
|
function createSettingsActivation(
|
|
48
49
|
options: () => SettingsOptions,
|
|
49
50
|
onCommitted: (settings: TranscribeSettings) => void,
|
|
51
|
+
ctx: ExtensionContext,
|
|
50
52
|
) {
|
|
51
53
|
return createModelActivation({
|
|
52
54
|
buildSettings: (model, path) => settingsForModel(model.id, path, options()),
|
|
53
55
|
onCommitted,
|
|
56
|
+
onWarning: (message) => ctx.ui.notify(message, "warning"),
|
|
54
57
|
});
|
|
55
58
|
}
|
|
56
59
|
|
|
@@ -67,6 +70,8 @@ type ModelSelectionOptions = {
|
|
|
67
70
|
polishEnabled?: boolean;
|
|
68
71
|
/** Preserved across the model re-pick; absent means the agent's current model. */
|
|
69
72
|
polishModel?: PolishModelRef;
|
|
73
|
+
voiceActivityDetection?: boolean;
|
|
74
|
+
longformChunking?: boolean;
|
|
70
75
|
/** Persists language changes made before this flow activates a model. */
|
|
71
76
|
onPreferredLanguagesChange?: (languages: string[]) => Promise<void>;
|
|
72
77
|
postActivation?: CatalogModelPostActivation;
|
|
@@ -94,11 +99,14 @@ export async function runModelSelection(
|
|
|
94
99
|
backend: configured?.backend ?? options.backend,
|
|
95
100
|
polishEnabled: configured?.polishEnabled ?? options.polishEnabled ?? false,
|
|
96
101
|
polishModel: configured?.polishModel ?? options.polishModel,
|
|
102
|
+
voiceActivityDetection: configured?.voiceActivityDetection ?? options.voiceActivityDetection,
|
|
103
|
+
longformChunking: configured?.longformChunking ?? options.longformChunking,
|
|
97
104
|
}),
|
|
98
105
|
(settings) => {
|
|
99
106
|
configured = settings;
|
|
100
107
|
currentModelId = settings.model?.id;
|
|
101
108
|
},
|
|
109
|
+
ctx,
|
|
102
110
|
);
|
|
103
111
|
|
|
104
112
|
// One pane switches between the models on disk and downloads new ones.
|
|
@@ -123,7 +131,7 @@ export async function runModelSelection(
|
|
|
123
131
|
if (changed) {
|
|
124
132
|
try {
|
|
125
133
|
if (configured) {
|
|
126
|
-
const updated =
|
|
134
|
+
const updated = settingsWithPreferredLanguages(configured, changed.languages);
|
|
127
135
|
await writeSettings(updated);
|
|
128
136
|
configured = updated;
|
|
129
137
|
} else {
|
|
@@ -225,10 +233,13 @@ export async function changeOnboardingModel(
|
|
|
225
233
|
backend: current.backend,
|
|
226
234
|
polishEnabled: current.polishEnabled,
|
|
227
235
|
polishModel: current.polishModel,
|
|
236
|
+
voiceActivityDetection: current.voiceActivityDetection,
|
|
237
|
+
longformChunking: current.longformChunking,
|
|
228
238
|
}),
|
|
229
239
|
(settings) => {
|
|
230
240
|
chosen = settings;
|
|
231
241
|
},
|
|
242
|
+
ctx,
|
|
232
243
|
);
|
|
233
244
|
|
|
234
245
|
while (true) {
|
|
@@ -356,6 +367,7 @@ export async function runOnboarding(
|
|
|
356
367
|
(settings) => {
|
|
357
368
|
configured = settings;
|
|
358
369
|
},
|
|
370
|
+
ctx,
|
|
359
371
|
);
|
|
360
372
|
|
|
361
373
|
let changeLanguages = false;
|
package/src/pcm-chunker.ts
CHANGED
|
@@ -2,13 +2,18 @@ import { CAPTURE_SAMPLE_RATE } from "./audio-constants.js";
|
|
|
2
2
|
|
|
3
3
|
const STREAM_CHUNK_SAMPLES = CAPTURE_SAMPLE_RATE / 2;
|
|
4
4
|
|
|
5
|
-
/**
|
|
5
|
+
/**
|
|
6
|
+
* Coalesces recorder chunks into fresh Float32 PCM chunks without timers,
|
|
7
|
+
* along with their Silero frame probabilities: a chunk has them only when
|
|
8
|
+
* every piece of it did.
|
|
9
|
+
*/
|
|
6
10
|
export class PcmChunker {
|
|
7
11
|
private pieces: Float32Array[] = [];
|
|
12
|
+
private probs: number[] | undefined = [];
|
|
8
13
|
private sampleCount = 0;
|
|
9
14
|
|
|
10
15
|
constructor(
|
|
11
|
-
private readonly onChunk: (chunk: Float32Array) => void,
|
|
16
|
+
private readonly onChunk: (chunk: Float32Array, probs?: readonly number[]) => void,
|
|
12
17
|
private readonly targetSamples = STREAM_CHUNK_SAMPLES,
|
|
13
18
|
) {
|
|
14
19
|
if (!Number.isInteger(targetSamples) || targetSamples <= 0) {
|
|
@@ -16,9 +21,11 @@ export class PcmChunker {
|
|
|
16
21
|
}
|
|
17
22
|
}
|
|
18
23
|
|
|
19
|
-
push(samples: Float32Array): void {
|
|
24
|
+
push(samples: Float32Array, probs?: readonly number[]): void {
|
|
20
25
|
if (samples.length === 0) return;
|
|
21
26
|
this.pieces.push(samples);
|
|
27
|
+
if (probs && this.probs) this.probs.push(...probs);
|
|
28
|
+
else this.probs = undefined;
|
|
22
29
|
this.sampleCount += samples.length;
|
|
23
30
|
if (this.sampleCount >= this.targetSamples) this.emit();
|
|
24
31
|
}
|
|
@@ -29,6 +36,7 @@ export class PcmChunker {
|
|
|
29
36
|
|
|
30
37
|
discard(): void {
|
|
31
38
|
this.pieces = [];
|
|
39
|
+
this.probs = [];
|
|
32
40
|
this.sampleCount = 0;
|
|
33
41
|
}
|
|
34
42
|
|
|
@@ -39,8 +47,10 @@ export class PcmChunker {
|
|
|
39
47
|
pcm.set(piece, offset);
|
|
40
48
|
offset += piece.length;
|
|
41
49
|
}
|
|
50
|
+
const probs = this.probs;
|
|
42
51
|
this.pieces = [];
|
|
52
|
+
this.probs = [];
|
|
43
53
|
this.sampleCount = 0;
|
|
44
|
-
this.onChunk(pcm);
|
|
54
|
+
this.onChunk(pcm, probs);
|
|
45
55
|
}
|
|
46
56
|
}
|
|
@@ -13,6 +13,8 @@ import {
|
|
|
13
13
|
formatBinarySize,
|
|
14
14
|
} from "./catalog.js";
|
|
15
15
|
import type { CatalogModelActivation } from "./model-activation.js";
|
|
16
|
+
import { undetectablePreferences } from "./language-detection-policy.js";
|
|
17
|
+
import { autoUnavailableNotice, downloadAction } from "./model-cells.js";
|
|
16
18
|
import { ModelSelectionController } from "./model-selection-controller.js";
|
|
17
19
|
import { VoiceKeys } from "./keybindings.js";
|
|
18
20
|
import {
|
|
@@ -202,9 +204,10 @@ export class RecommendedModelPicker extends Container implements Focusable {
|
|
|
202
204
|
} else {
|
|
203
205
|
text = "A good balance of speed and accuracy.";
|
|
204
206
|
}
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
207
|
+
// Recommended models support every chosen language, so Auto covers
|
|
208
|
+
// them all unless the detector cannot identify one.
|
|
209
|
+
if (this.languages.length > 1 && undetectablePreferences(recommendation.model, this.languages).length === 0) {
|
|
210
|
+
text += " It detects which language you're speaking automatically.";
|
|
208
211
|
}
|
|
209
212
|
return text;
|
|
210
213
|
}
|
|
@@ -272,9 +275,8 @@ export class RecommendedModelPicker extends Container implements Focusable {
|
|
|
272
275
|
),
|
|
273
276
|
);
|
|
274
277
|
}
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
}
|
|
278
|
+
const notice = autoUnavailableNotice(recommendation.model, this.languages);
|
|
279
|
+
if (notice) details.addChild(new Text(this.theme.fg("warning", notice), LIST_PADDING + 2, 0));
|
|
278
280
|
return details;
|
|
279
281
|
}
|
|
280
282
|
|
|
@@ -387,7 +389,7 @@ export class RecommendedModelPicker extends Container implements Focusable {
|
|
|
387
389
|
if (!row) return "choose";
|
|
388
390
|
if (row.type !== "model") return this.rowLabel(row).action;
|
|
389
391
|
const model = row.recommendation.model;
|
|
390
|
-
return this.selection.cachedById.has(model.id)
|
|
392
|
+
return downloadAction(model, this.selection.cachedById.has(model.id));
|
|
391
393
|
}
|
|
392
394
|
|
|
393
395
|
override render(width: number): string[] {
|
package/src/recommendations.ts
CHANGED
|
@@ -7,6 +7,8 @@ import {
|
|
|
7
7
|
type CatalogModel,
|
|
8
8
|
} from "./catalog.js";
|
|
9
9
|
|
|
10
|
+
import { undetectablePreferences } from "./language-detection-policy.js";
|
|
11
|
+
|
|
10
12
|
export type MachineTier = "accelerated" | "cpu";
|
|
11
13
|
|
|
12
14
|
export type RecommendationRole = "best" | "fast" | "accurate";
|
|
@@ -211,7 +213,7 @@ function scoreModels(
|
|
|
211
213
|
worstCiLower: accuracyCells[worstIndex]!.ciLower,
|
|
212
214
|
waitSeconds,
|
|
213
215
|
cpuWaitSeconds: cpuXrt ? method.dictationSeconds / cpuXrt : undefined,
|
|
214
|
-
manual: wanted.length > 1 &&
|
|
216
|
+
manual: wanted.length > 1 && undetectablePreferences(model, languages).length > 0,
|
|
215
217
|
score: error + method.waitWeight * Math.max(0, waitSeconds - method.freeWaitSeconds),
|
|
216
218
|
});
|
|
217
219
|
}
|
|
@@ -284,7 +286,7 @@ function fallbackRecommendation(
|
|
|
284
286
|
(model) =>
|
|
285
287
|
wanted.every((language) =>
|
|
286
288
|
modelMatchesRecommendationLanguage(model, language),
|
|
287
|
-
) && (wanted.length === 1 || model.
|
|
289
|
+
) && (wanted.length === 1 || undetectablePreferences(model, languages).length === 0),
|
|
288
290
|
);
|
|
289
291
|
const model = rankCatalogModels(compatible.length ? compatible : models, wanted)[0];
|
|
290
292
|
return model
|
package/src/runtime.ts
CHANGED
|
@@ -15,6 +15,7 @@ import { polishTranscript } from "./polish.js";
|
|
|
15
15
|
import type { TranscribeSettings } from "./settings.js";
|
|
16
16
|
import { displayShortcut } from "./shortcut-core.js";
|
|
17
17
|
import { TranscriptionService } from "./transcription-service.js";
|
|
18
|
+
import { VoiceActivityDetection } from "./vad.js";
|
|
18
19
|
import type { RecordingMeter } from "./visualizer.js";
|
|
19
20
|
|
|
20
21
|
type ActiveRecording = {
|
|
@@ -67,7 +68,8 @@ export function createPiVoiceRuntime(
|
|
|
67
68
|
let settingsWarningShown = false;
|
|
68
69
|
let audioModulePromise: Promise<typeof import("./audio.js")> | undefined;
|
|
69
70
|
let visualizerModulePromise: Promise<typeof import("./visualizer.js")> | undefined;
|
|
70
|
-
const
|
|
71
|
+
const voiceActivity = new VoiceActivityDetection();
|
|
72
|
+
const transcriptionService = new TranscriptionService(undefined, (pcm, signal) => voiceActivity.scoreClip(pcm, signal));
|
|
71
73
|
|
|
72
74
|
function loadAudio(): Promise<typeof import("./audio.js")> {
|
|
73
75
|
return (audioModulePromise ??= import("./audio.js"));
|
|
@@ -150,6 +152,8 @@ export function createPiVoiceRuntime(
|
|
|
150
152
|
backend: previous.backend,
|
|
151
153
|
polishEnabled: previous.polishEnabled,
|
|
152
154
|
polishModel: previous.polishModel,
|
|
155
|
+
voiceActivityDetection: previous.voiceActivityDetection,
|
|
156
|
+
longformChunking: previous.longformChunking,
|
|
153
157
|
postActivation: "advance",
|
|
154
158
|
});
|
|
155
159
|
if (configured) rememberSettings(configured);
|
|
@@ -294,6 +298,7 @@ export function createPiVoiceRuntime(
|
|
|
294
298
|
try {
|
|
295
299
|
const result = await active.dictation.stop();
|
|
296
300
|
if (shuttingDown) return;
|
|
301
|
+
if (result?.warning) ctx.ui.notify(result.warning, "warning");
|
|
297
302
|
if (!result) {
|
|
298
303
|
await reportDictationError(ctx, active.dictation);
|
|
299
304
|
} else if (result.text) {
|
|
@@ -355,6 +360,7 @@ export function createPiVoiceRuntime(
|
|
|
355
360
|
openCapture: openMicrophone,
|
|
356
361
|
onAudio: (chunk) => meter.push(chunk),
|
|
357
362
|
onChange: updateMeter,
|
|
363
|
+
createSpeechFilter: (settings, onSpeech) => voiceActivity.createFilter(settings, onSpeech),
|
|
358
364
|
});
|
|
359
365
|
function updateMeter(): void {
|
|
360
366
|
meter.update({ modelState: controller.modelState, microphoneStopped: controller.interruption !== undefined });
|
|
@@ -514,6 +520,8 @@ export function createPiVoiceRuntime(
|
|
|
514
520
|
operation?.catch(() => undefined),
|
|
515
521
|
transcriptionService.shutdown().catch(() => undefined),
|
|
516
522
|
]);
|
|
523
|
+
// After the dictation, so no recording still holds a VAD session.
|
|
524
|
+
await voiceActivity.dispose().catch(() => undefined);
|
|
517
525
|
recording = undefined;
|
|
518
526
|
dictation = undefined;
|
|
519
527
|
if (visualizerModulePromise) {
|