@corenel/tools-web 0.4.6 → 0.4.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/core/protocol.d.ts +8 -0
- package/dist/core/protocol.d.ts.map +1 -1
- package/dist/runtime.d.ts.map +1 -1
- package/dist/runtime.js +53 -35
- package/dist/runtime.js.map +1 -1
- package/dist/speech/engine.d.ts +16 -0
- package/dist/speech/engine.d.ts.map +1 -0
- package/dist/speech/engine.js +115 -0
- package/dist/speech/engine.js.map +1 -0
- package/dist/speech/model.d.ts +40 -0
- package/dist/speech/model.d.ts.map +1 -0
- package/dist/speech/model.js +135 -0
- package/dist/speech/model.js.map +1 -0
- package/dist/speech/protocol.d.ts +38 -0
- package/dist/speech/protocol.d.ts.map +1 -0
- package/dist/speech/protocol.js +14 -0
- package/dist/speech/protocol.js.map +1 -0
- package/dist/speech/session.d.ts +34 -0
- package/dist/speech/session.d.ts.map +1 -0
- package/dist/speech/session.js +254 -0
- package/dist/speech/session.js.map +1 -0
- package/dist/speech/speechWorker.d.ts +2 -0
- package/dist/speech/speechWorker.d.ts.map +1 -0
- package/dist/speech/speechWorker.js +105 -0
- package/dist/speech/speechWorker.js.map +1 -0
- package/dist/workerClient.d.ts +6 -0
- package/dist/workerClient.d.ts.map +1 -1
- package/dist/workerClient.js +4 -2
- package/dist/workerClient.js.map +1 -1
- package/dist/workerHandler.d.ts.map +1 -1
- package/dist/workerHandler.js +19 -8
- package/dist/workerHandler.js.map +1 -1
- package/package.json +4 -3
|
@@ -0,0 +1,135 @@
|
|
|
1
|
+
/* The dictation model's lifecycle: is it here, get it here, remove it.
|
|
2
|
+
*
|
|
3
|
+
* Deliberately separate from session.ts because it has a SECOND CONSUMER that
|
|
4
|
+
* never opens a microphone -- the Settings row prefetches on wifi, reports what
|
|
5
|
+
* is stored, and clears it. Folding this into the session would make that row
|
|
6
|
+
* reach through a dictation object it has no use for.
|
|
7
|
+
*
|
|
8
|
+
* The two pieces of state here are different things and are stored differently:
|
|
9
|
+
*
|
|
10
|
+
* INTENT (localStorage) -- the user said yes to the download.
|
|
11
|
+
* PRESENCE (Cache Storage) -- the bytes are actually on this device.
|
|
12
|
+
*
|
|
13
|
+
* Intent deliberately outlives presence. Browsers evict Cache Storage under
|
|
14
|
+
* memory pressure without telling the page, and someone who has already
|
|
15
|
+
* consented must not be asked again because the browser reclaimed some space --
|
|
16
|
+
* the model just re-downloads on next use. */
|
|
17
|
+
import { SPEECH_MODEL_ID } from './protocol';
|
|
18
|
+
import { sharedSpeechEngine } from './engine';
|
|
19
|
+
export { SPEECH_MODEL_ID };
|
|
20
|
+
/** Where the user's consent is remembered. */
|
|
21
|
+
export const INTENT_KEY = 'prompd.web.speech.enabled';
|
|
22
|
+
/** transformers.js keeps downloaded weights here. */
|
|
23
|
+
const CACHE_NAME = 'transformers-cache';
|
|
24
|
+
export function createModelHost({ caches, storage, loadPipeline, onEvict }) {
|
|
25
|
+
/* The in-flight load, so three surfaces asking at once download once. Cleared
|
|
26
|
+
* on failure -- a rejected promise cached here would turn one offline blip
|
|
27
|
+
* into a permanently broken button. */
|
|
28
|
+
let loading = null;
|
|
29
|
+
/* The warmed pipeline. Presence in memory is NOT presence on disk: isCached()
|
|
30
|
+
* still probes, because this says nothing about a later eviction. */
|
|
31
|
+
let warmed = null;
|
|
32
|
+
const modelUrls = async () => {
|
|
33
|
+
const cache = await caches.open(CACHE_NAME);
|
|
34
|
+
const keys = await cache.keys();
|
|
35
|
+
return keys.map((k) => k.url).filter((u) => u.includes(SPEECH_MODEL_ID));
|
|
36
|
+
};
|
|
37
|
+
return {
|
|
38
|
+
isEnabled: () => storage.getItem(INTENT_KEY) === '1',
|
|
39
|
+
setEnabled: (on) => {
|
|
40
|
+
if (on)
|
|
41
|
+
storage.setItem(INTENT_KEY, '1');
|
|
42
|
+
else
|
|
43
|
+
storage.removeItem(INTENT_KEY);
|
|
44
|
+
},
|
|
45
|
+
/* A REAL PROBE EVERY TIME. The tempting optimisation -- remember the answer
|
|
46
|
+
* -- is the bug this module exists to avoid: a remembered `true` after an
|
|
47
|
+
* eviction makes the caller skip the download and then wait forever for a
|
|
48
|
+
* model that is not there. */
|
|
49
|
+
isCached: async () => {
|
|
50
|
+
try {
|
|
51
|
+
/* THE WEIGHTS, not merely "something for this model".
|
|
52
|
+
*
|
|
53
|
+
* Matching any cached URL containing the model id counted config.json,
|
|
54
|
+
* tokenizer_config.json and friends -- so a download that died after the
|
|
55
|
+
* small metadata files reported "Ready on this site", skipped the
|
|
56
|
+
* consent prompt, and left the button waiting on weights that were never
|
|
57
|
+
* there. Observed in a real cache: the metadata lands in milliseconds,
|
|
58
|
+
* the two .onnx files are ~28MB and take far longer, so the window where
|
|
59
|
+
* this lies is the entire download. */
|
|
60
|
+
const urls = await modelUrls();
|
|
61
|
+
const onnx = urls.filter((u) => u.endsWith('.onnx'));
|
|
62
|
+
return onnx.some((u) => /encoder/i.test(u)) && onnx.some((u) => /decoder/i.test(u));
|
|
63
|
+
}
|
|
64
|
+
catch {
|
|
65
|
+
// No Cache Storage (private mode, blocked site data): treat as absent
|
|
66
|
+
// rather than throwing. The model simply downloads again.
|
|
67
|
+
return false;
|
|
68
|
+
}
|
|
69
|
+
},
|
|
70
|
+
ensureModel: (onProgress) => {
|
|
71
|
+
if (warmed)
|
|
72
|
+
return Promise.resolve();
|
|
73
|
+
if (loading)
|
|
74
|
+
return loading;
|
|
75
|
+
loading = loadPipeline(onProgress)
|
|
76
|
+
.then((p) => { warmed = p; })
|
|
77
|
+
.catch((e) => {
|
|
78
|
+
// Let the next caller try again rather than inheriting this failure.
|
|
79
|
+
loading = null;
|
|
80
|
+
throw e;
|
|
81
|
+
});
|
|
82
|
+
return loading;
|
|
83
|
+
},
|
|
84
|
+
/* Deletes only OUR entries. The transformers cache is shared -- nuking the
|
|
85
|
+
* whole thing would evict any other model the app has downloaded. */
|
|
86
|
+
evict: async () => {
|
|
87
|
+
try {
|
|
88
|
+
const cache = await caches.open(CACHE_NAME);
|
|
89
|
+
const urls = await modelUrls();
|
|
90
|
+
await Promise.all(urls.map((u) => cache.delete(u)));
|
|
91
|
+
}
|
|
92
|
+
catch {
|
|
93
|
+
// Nothing to remove if the cache is unavailable.
|
|
94
|
+
}
|
|
95
|
+
warmed = null;
|
|
96
|
+
loading = null;
|
|
97
|
+
onEvict?.();
|
|
98
|
+
},
|
|
99
|
+
};
|
|
100
|
+
}
|
|
101
|
+
/* ONE HOST PER PAGE.
|
|
102
|
+
*
|
|
103
|
+
* The composer's mic button and the Settings row are two views of the same
|
|
104
|
+
* thing. Given separate hosts they would disagree: Settings clears the model,
|
|
105
|
+
* the button's own instance still holds a warmed pipeline and happily records;
|
|
106
|
+
* or Settings prefetches and the button still believes nothing is downloaded.
|
|
107
|
+
*
|
|
108
|
+
* Created LAZILY rather than at module scope -- touching `caches` or
|
|
109
|
+
* `localStorage` at import time runs in every test and every worker that pulls
|
|
110
|
+
* this module in transitively, which is exactly how import-time side effects
|
|
111
|
+
* poison things that never asked for them. */
|
|
112
|
+
let shared = null;
|
|
113
|
+
export function browserModelHost() {
|
|
114
|
+
if (!shared)
|
|
115
|
+
shared = createBrowserModelHost();
|
|
116
|
+
return shared;
|
|
117
|
+
}
|
|
118
|
+
/** The browser's real host.
|
|
119
|
+
*
|
|
120
|
+
* WARMS THE WORKER, NOT THE MAIN THREAD. An earlier version built a pipeline
|
|
121
|
+
* here, on the main thread, purely so it could watch a download -- then threw it
|
|
122
|
+
* away and let the worker build its own. That duplicated the expensive part
|
|
123
|
+
* (ONNX instantiation, graph compilation) and did it in the one place the worker
|
|
124
|
+
* exists to protect. Warming the shared engine means the thing that gets warmed
|
|
125
|
+
* is the thing that transcribes, and the Settings prefetch now genuinely makes
|
|
126
|
+
* the first dictation fast rather than only pre-filling the HTTP cache. */
|
|
127
|
+
export function createBrowserModelHost() {
|
|
128
|
+
return createModelHost({
|
|
129
|
+
caches: globalThis.caches,
|
|
130
|
+
storage: globalThis.localStorage,
|
|
131
|
+
loadPipeline: (onProgress) => sharedSpeechEngine().load(onProgress),
|
|
132
|
+
onEvict: () => sharedSpeechEngine().dispose(),
|
|
133
|
+
});
|
|
134
|
+
}
|
|
135
|
+
//# sourceMappingURL=model.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"model.js","sourceRoot":"","sources":["../../speech/model.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;8CAe8C;AAC9C,OAAO,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AAC7C,OAAO,EAAE,kBAAkB,EAAE,MAAM,UAAU,CAAC;AAE9C,OAAO,EAAE,eAAe,EAAE,CAAC;AAE3B,8CAA8C;AAC9C,MAAM,CAAC,MAAM,UAAU,GAAG,2BAA2B,CAAC;AAEtD,qDAAqD;AACrD,MAAM,UAAU,GAAG,oBAAoB,CAAC;AA4BxC,MAAM,UAAU,eAAe,CAAC,EAAE,MAAM,EAAE,OAAO,EAAE,YAAY,EAAE,OAAO,EAAiB;IACvF;;2CAEuC;IACvC,IAAI,OAAO,GAAyB,IAAI,CAAC;IACzC;yEACqE;IACrE,IAAI,MAAM,GAAY,IAAI,CAAC;IAE3B,MAAM,SAAS,GAAG,KAAK,IAAuB,EAAE;QAC9C,MAAM,KAAK,GAAG,MAAM,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QAC5C,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,IAAI,EAAE,CAAC;QAChC,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,eAAe,CAAC,CAAC,CAAC;IAC3E,CAAC,CAAC;IAEF,OAAO;QACL,SAAS,EAAE,GAAG,EAAE,CAAC,OAAO,CAAC,OAAO,CAAC,UAAU,CAAC,KAAK,GAAG;QAEpD,UAAU,EAAE,CAAC,EAAE,EAAE,EAAE;YACjB,IAAI,EAAE;gBAAE,OAAO,CAAC,OAAO,CAAC,UAAU,EAAE,GAAG,CAAC,CAAC;;gBACpC,OAAO,CAAC,UAAU,CAAC,UAAU,CAAC,CAAC;QACtC,CAAC;QAED;;;sCAG8B;QAC9B,QAAQ,EAAE,KAAK,IAAI,EAAE;YACnB,IAAI,CAAC;gBACH;;;;;;;;uDAQuC;gBACvC,MAAM,IAAI,GAAG,MAAM,SAAS,EAAE,CAAC;gBAC/B,MAAM,IAAI,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC;gBACrD,OAAO,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;YACtF,CAAC;YAAC,MAAM,CAAC;gBACP,sEAAsE;gBACtE,0DAA0D;gBAC1D,OAAO,KAAK,CAAC;YACf,CAAC;QACH,CAAC;QAED,WAAW,EAAE,CAAC,UAAU,EAAE,EAAE;YAC1B,IAAI,MAAM;gBAAE,OAAO,OAAO,CAAC,OAAO,EAAE,CAAC;YACrC,IAAI,OAAO;gBAAE,OAAO,OAAO,CAAC;YAC5B,OAAO,GAAG,YAAY,CAAC,UAAU,CAAC;iBAC/B,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,GAAG,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;iBAC5B,KAAK,CAAC,CAAC,CAAU,EAAE,EAAE;gBACpB,qEAAqE;gBACrE,OAAO,GAAG,IAAI,CAAC;gBACf,MAAM,CAAC,CAAC;YACV,CAAC,CAAC,CAAC;YACL,OAAO,OAAO,CAAC;QACjB,CAAC;QAED;6EACqE;QACrE,KAAK,EAAE,KAAK,IAAI,EAAE;YAChB,IAAI,CAAC;gBACH,MAAM,KAAK,GAAG,MAAM,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;gBAC5C,MAAM,IAAI,GAAG,MAAM,SAAS,EAAE,CAAC;gBAC/B,MAAM,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YACtD,CAAC;YAAC,MAAM,CAAC;gBACP,iDAAiD;YACnD,CAAC;YACD,MAAM,GAAG,IAAI,CAAC;YACd,OAAO,GAAG,IAAI,CAAC;YACf,OAAO,EAAE,EAAE,CAAC;QACd,CAAC;KACF,CAAC;AACJ,CAAC;AAED;;;;;;;;;;8CAU8C;AAC9C,IAAI,MAAM,GAAqB,IAAI,CAAC;AAEpC,MAAM,UAAU,gBAAgB;IAC9B,IAAI,CAAC,MAAM;QAAE,MAAM,GAAG,sBAAsB,EAAE,CAAC;IAC/C,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;;;;;;;2EAQ2E;AAC3E,MAAM,UAAU,sBAAsB;IACpC,OAAO,eAAe,CAAC;QACrB,MAAM,EAAE,UAAU,CAAC,MAAM;QACzB,OAAO,EAAE,UAAU,CAAC,YAAY;QAChC,YAAY,EAAE,CAAC,UAAU,EAAE,EAAE,CAAC,kBAAkB,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC;QACnE,OAAO,EAAE,GAAG,EAAE,CAAC,kBAAkB,EAAE,CAAC,OAAO,EAAE;KAC9C,CAAC,CAAC;AACL,CAAC"}
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
/** The model the browser downloads and runs. English-only by design: Moonshine's
|
|
2
|
+
* other languages are separate monolingual models, i.e. separate downloads. */
|
|
3
|
+
export declare const SPEECH_MODEL_ID = "onnx-community/moonshine-tiny-ONNX";
|
|
4
|
+
/** The pipeline's input contract: mono PCM at this rate. Both the capture and
|
|
5
|
+
* the worker resample to it, so it lives where both can see it. */
|
|
6
|
+
export declare const SAMPLE_RATE = 16000;
|
|
7
|
+
export type MainToWorker = {
|
|
8
|
+
kind: 'load';
|
|
9
|
+
} | {
|
|
10
|
+
kind: 'audio';
|
|
11
|
+
pcm: Float32Array;
|
|
12
|
+
} | {
|
|
13
|
+
kind: 'flush';
|
|
14
|
+
};
|
|
15
|
+
export type WorkerToMain = {
|
|
16
|
+
kind: 'ready';
|
|
17
|
+
}
|
|
18
|
+
/** Download progress, whole percent. Reported from the WORKER because the
|
|
19
|
+
* worker is what fetches the weights -- the main thread no longer builds a
|
|
20
|
+
* pipeline of its own just to observe a download. */
|
|
21
|
+
| {
|
|
22
|
+
kind: 'progress';
|
|
23
|
+
pct: number;
|
|
24
|
+
}
|
|
25
|
+
/** A revisable guess. Never reaches the draft -- see session.ts. */
|
|
26
|
+
| {
|
|
27
|
+
kind: 'interim';
|
|
28
|
+
text: string;
|
|
29
|
+
}
|
|
30
|
+
/** Committed text, safe to append to what the user is writing. */
|
|
31
|
+
| {
|
|
32
|
+
kind: 'final';
|
|
33
|
+
text: string;
|
|
34
|
+
} | {
|
|
35
|
+
kind: 'error';
|
|
36
|
+
message: string;
|
|
37
|
+
};
|
|
38
|
+
//# sourceMappingURL=protocol.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"protocol.d.ts","sourceRoot":"","sources":["../../speech/protocol.ts"],"names":[],"mappings":"AAQA;+EAC+E;AAC/E,eAAO,MAAM,eAAe,uCAAuC,CAAC;AAEpE;mEACmE;AACnE,eAAO,MAAM,WAAW,QAAQ,CAAC;AAEjC,MAAM,MAAM,YAAY,GACpB;IAAE,IAAI,EAAE,MAAM,CAAA;CAAE,GAChB;IAAE,IAAI,EAAE,OAAO,CAAC;IAAC,GAAG,EAAE,YAAY,CAAA;CAAE,GACpC;IAAE,IAAI,EAAE,OAAO,CAAA;CAAE,CAAC;AAEtB,MAAM,MAAM,YAAY,GACpB;IAAE,IAAI,EAAE,OAAO,CAAA;CAAE;AACnB;;qDAEqD;GACnD;IAAE,IAAI,EAAE,UAAU,CAAC;IAAC,GAAG,EAAE,MAAM,CAAA;CAAE;AACnC,oEAAoE;GAClE;IAAE,IAAI,EAAE,SAAS,CAAC;IAAC,IAAI,EAAE,MAAM,CAAA;CAAE;AACnC,kEAAkE;GAChE;IAAE,IAAI,EAAE,OAAO,CAAC;IAAC,IAAI,EAAE,MAAM,CAAA;CAAE,GAC/B;IAAE,IAAI,EAAE,OAAO,CAAC;IAAC,OAAO,EAAE,MAAM,CAAA;CAAE,CAAC"}
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
/* The wire between the main thread and the dictation worker.
|
|
2
|
+
*
|
|
3
|
+
* Its own module because BOTH sides import it and neither may import the other:
|
|
4
|
+
* the worker must not drag the session's DOM code (getUserMedia, MediaStream)
|
|
5
|
+
* into a worker scope that has none, and the session must not drag the
|
|
6
|
+
* transformers pipeline onto the main thread, which is the whole reason the
|
|
7
|
+
* worker exists. */
|
|
8
|
+
/** The model the browser downloads and runs. English-only by design: Moonshine's
|
|
9
|
+
* other languages are separate monolingual models, i.e. separate downloads. */
|
|
10
|
+
export const SPEECH_MODEL_ID = 'onnx-community/moonshine-tiny-ONNX';
|
|
11
|
+
/** The pipeline's input contract: mono PCM at this rate. Both the capture and
|
|
12
|
+
* the worker resample to it, so it lives where both can see it. */
|
|
13
|
+
export const SAMPLE_RATE = 16000;
|
|
14
|
+
//# sourceMappingURL=protocol.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"protocol.js","sourceRoot":"","sources":["../../speech/protocol.ts"],"names":[],"mappings":"AAAA;;;;;;oBAMoB;AAEpB;+EAC+E;AAC/E,MAAM,CAAC,MAAM,eAAe,GAAG,oCAAoC,CAAC;AAEpE;mEACmE;AACnE,MAAM,CAAC,MAAM,WAAW,GAAG,KAAK,CAAC"}
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
import { type SpeechEngine } from './engine';
|
|
2
|
+
export type SpeechErrorKind = 'permission' | 'unsupported' | 'inference';
|
|
3
|
+
export interface SpeechError {
|
|
4
|
+
kind: SpeechErrorKind;
|
|
5
|
+
message: string;
|
|
6
|
+
}
|
|
7
|
+
/** Pulls PCM off a live stream. Returned handle stops the pull. */
|
|
8
|
+
export interface Capture {
|
|
9
|
+
stop(): void;
|
|
10
|
+
}
|
|
11
|
+
export type CreateCapture = (stream: MediaStream, onChunk: (pcm: Float32Array) => void) => Capture;
|
|
12
|
+
export interface DictationOpts {
|
|
13
|
+
onInterim(text: string): void;
|
|
14
|
+
onFinal(text: string): void;
|
|
15
|
+
onError(e: SpeechError): void;
|
|
16
|
+
/** Injected in tests; defaults to the page's shared engine. */
|
|
17
|
+
engine?: SpeechEngine;
|
|
18
|
+
getMedia?: () => Promise<MediaStream>;
|
|
19
|
+
createCapture?: CreateCapture;
|
|
20
|
+
/** How long stop() waits for the tail transcript before giving up. A dead
|
|
21
|
+
* worker must not be able to wedge stop(); the microphone is already released
|
|
22
|
+
* by then, so the only thing at risk is the last sentence. */
|
|
23
|
+
flushTimeoutMs?: number;
|
|
24
|
+
}
|
|
25
|
+
export interface Dictation {
|
|
26
|
+
/** Acquire the mic, warm the engine, begin listening. Never rejects: a denial
|
|
27
|
+
* is a state the UI renders, not a crash it has to catch. */
|
|
28
|
+
start(): Promise<void>;
|
|
29
|
+
/** Flush the tail, release the mic. Idempotent, safe before start(). */
|
|
30
|
+
stop(): Promise<void>;
|
|
31
|
+
}
|
|
32
|
+
export declare const defaultCreateCapture: CreateCapture;
|
|
33
|
+
export declare function createDictation(opts: DictationOpts): Dictation;
|
|
34
|
+
//# sourceMappingURL=session.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"session.d.ts","sourceRoot":"","sources":["../../speech/session.ts"],"names":[],"mappings":"AAgBA,OAAO,EAAsB,KAAK,YAAY,EAAE,MAAM,UAAU,CAAC;AAEjE,MAAM,MAAM,eAAe,GAAG,YAAY,GAAG,aAAa,GAAG,WAAW,CAAC;AACzE,MAAM,WAAW,WAAW;IAAG,IAAI,EAAE,eAAe,CAAC;IAAC,OAAO,EAAE,MAAM,CAAA;CAAE;AAEvE,mEAAmE;AACnE,MAAM,WAAW,OAAO;IAAG,IAAI,IAAI,IAAI,CAAA;CAAE;AACzC,MAAM,MAAM,aAAa,GAAG,CAAC,MAAM,EAAE,WAAW,EAAE,OAAO,EAAE,CAAC,GAAG,EAAE,YAAY,KAAK,IAAI,KAAK,OAAO,CAAC;AAEnG,MAAM,WAAW,aAAa;IAC5B,SAAS,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B,OAAO,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,OAAO,CAAC,CAAC,EAAE,WAAW,GAAG,IAAI,CAAC;IAC9B,+DAA+D;IAC/D,MAAM,CAAC,EAAE,YAAY,CAAC;IACtB,QAAQ,CAAC,EAAE,MAAM,OAAO,CAAC,WAAW,CAAC,CAAC;IACtC,aAAa,CAAC,EAAE,aAAa,CAAC;IAC9B;;kEAE8D;IAC9D,cAAc,CAAC,EAAE,MAAM,CAAC;CACzB;AAED,MAAM,WAAW,SAAS;IACxB;iEAC6D;IAC7D,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IACvB,wEAAwE;IACxE,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CACvB;AAyBD,eAAO,MAAM,oBAAoB,EAAE,aAsBlC,CAAC;AAcF,wBAAgB,eAAe,CAAC,IAAI,EAAE,aAAa,GAAG,SAAS,CAgK9D"}
|
|
@@ -0,0 +1,254 @@
|
|
|
1
|
+
/* One dictation session: microphone in, text out.
|
|
2
|
+
*
|
|
3
|
+
* THE INVARIANT: the microphone is released on every terminal path. Stop,
|
|
4
|
+
* permission denial, a failed load, a worker crash, an inference error -- all of
|
|
5
|
+
* them run the same teardown. On-device dictation's whole claim is that nothing
|
|
6
|
+
* leaves the machine, and a mic left hot after a crash makes that claim false in
|
|
7
|
+
* a way the user cannot see.
|
|
8
|
+
*
|
|
9
|
+
* THE SECOND RULE: interim text never reaches the caller's draft. The worker
|
|
10
|
+
* revises its guesses as more audio arrives; only `final` is committed.
|
|
11
|
+
*
|
|
12
|
+
* WHAT THIS MODULE DOES NOT OWN: the worker. It borrows the page's shared
|
|
13
|
+
* SpeechEngine and hands it back on stop. An earlier version built and
|
|
14
|
+
* terminated a worker per dictation, which re-instantiated the ONNX runtime on
|
|
15
|
+
* every single click. Runs are cheap now; only the first one pays. */
|
|
16
|
+
import { SAMPLE_RATE } from './protocol';
|
|
17
|
+
import { sharedSpeechEngine } from './engine';
|
|
18
|
+
const defaultGetMedia = () => navigator.mediaDevices.getUserMedia({ audio: true });
|
|
19
|
+
/* Real capture: resample to the pipeline's contract (mono, 16kHz) and hand up
|
|
20
|
+
* Float32 frames. ScriptProcessor is deprecated but is the only node available
|
|
21
|
+
* without shipping a separate worklet file; the buffer is small and the work per
|
|
22
|
+
* frame is a copy.
|
|
23
|
+
*
|
|
24
|
+
* Exported so the two things below can actually be tested. Both were silent
|
|
25
|
+
* failures found from use, not from the suite.
|
|
26
|
+
*
|
|
27
|
+
* RESUME, ALWAYS. This context is constructed after two awaits -- the permission
|
|
28
|
+
* prompt and the engine warm-up -- so it is created well outside the user
|
|
29
|
+
* gesture that started dictation, and Chrome hands back a SUSPENDED context.
|
|
30
|
+
* A suspended context never fires onaudioprocess, so not one sample reaches the
|
|
31
|
+
* worker: the button says "Stop dictating", the microphone light is on, and
|
|
32
|
+
* nothing is ever transcribed.
|
|
33
|
+
*
|
|
34
|
+
* AND THE GRAPH MUST NOT END AT THE SPEAKERS. A ScriptProcessor only pulls while
|
|
35
|
+
* it is connected to something live, but connecting it to ctx.destination pipes
|
|
36
|
+
* the microphone straight back out of the speakers -- you hear yourself, with
|
|
37
|
+
* feedback if you are not on headphones. A zero-gain node keeps the graph pulling
|
|
38
|
+
* and outputs silence. */
|
|
39
|
+
export const defaultCreateCapture = (stream, onChunk) => {
|
|
40
|
+
const ctx = new AudioContext({ sampleRate: SAMPLE_RATE });
|
|
41
|
+
// Fire-and-forget: resume() rejects only if the context is already closed,
|
|
42
|
+
// and a failure here must not take down a dictation that may still work.
|
|
43
|
+
if (ctx.state === 'suspended')
|
|
44
|
+
void ctx.resume().catch(() => { });
|
|
45
|
+
const source = ctx.createMediaStreamSource(stream);
|
|
46
|
+
const node = ctx.createScriptProcessor(4096, 1, 1);
|
|
47
|
+
const mute = ctx.createGain();
|
|
48
|
+
mute.gain.value = 0;
|
|
49
|
+
node.onaudioprocess = (e) => { onChunk(new Float32Array(e.inputBuffer.getChannelData(0))); };
|
|
50
|
+
source.connect(node);
|
|
51
|
+
node.connect(mute);
|
|
52
|
+
mute.connect(ctx.destination);
|
|
53
|
+
return {
|
|
54
|
+
stop: () => {
|
|
55
|
+
node.onaudioprocess = null;
|
|
56
|
+
node.disconnect();
|
|
57
|
+
mute.disconnect();
|
|
58
|
+
source.disconnect();
|
|
59
|
+
void ctx.close().catch(() => { });
|
|
60
|
+
},
|
|
61
|
+
};
|
|
62
|
+
};
|
|
63
|
+
/** Map a getUserMedia rejection onto something the UI can explain. */
|
|
64
|
+
function mediaError(e) {
|
|
65
|
+
const name = e.name;
|
|
66
|
+
if (name === 'NotAllowedError' || name === 'PermissionDeniedError') {
|
|
67
|
+
return { kind: 'permission', message: 'Microphone permission was denied.' };
|
|
68
|
+
}
|
|
69
|
+
if (name === 'NotFoundError' || name === 'DevicesNotFoundError') {
|
|
70
|
+
return { kind: 'unsupported', message: 'No microphone was found.' };
|
|
71
|
+
}
|
|
72
|
+
return { kind: 'unsupported', message: 'The microphone is unavailable in this browser.' };
|
|
73
|
+
}
|
|
74
|
+
export function createDictation(opts) {
|
|
75
|
+
const engine = opts.engine ?? sharedSpeechEngine();
|
|
76
|
+
const getMedia = opts.getMedia ?? defaultGetMedia;
|
|
77
|
+
const createCapture = opts.createCapture ?? defaultCreateCapture;
|
|
78
|
+
const flushTimeoutMs = opts.flushTimeoutMs ?? 4000;
|
|
79
|
+
let stream = null;
|
|
80
|
+
let capture = null;
|
|
81
|
+
let unsubscribe = null;
|
|
82
|
+
let listening = false;
|
|
83
|
+
let running = false;
|
|
84
|
+
/* THE RUN GENERATION -- and the reason it is a counter rather than a boolean.
|
|
85
|
+
*
|
|
86
|
+
* A `disposed` flag cannot express "a stop arrived while start() was still
|
|
87
|
+
* awaiting the permission prompt". In that window there is nothing to tear
|
|
88
|
+
* down AND no way to say "do not finish starting either", so start() would
|
|
89
|
+
* resume after the user answered, open the microphone, and leave it live with
|
|
90
|
+
* no reachable handle to close it -- clicking again would begin a second,
|
|
91
|
+
* independent session rather than reaching the orphan.
|
|
92
|
+
*
|
|
93
|
+
* So every run claims a generation, every await in start() re-checks it, and
|
|
94
|
+
* stop() bumps it UNCONDITIONALLY. */
|
|
95
|
+
let gen = 0;
|
|
96
|
+
/** Release whatever a run holds. Never throws. Safe with partial state. */
|
|
97
|
+
function release(s, c) {
|
|
98
|
+
try {
|
|
99
|
+
c?.stop();
|
|
100
|
+
}
|
|
101
|
+
catch { /* releasing anyway */ }
|
|
102
|
+
// The microphone itself. This is the line that matters.
|
|
103
|
+
try {
|
|
104
|
+
s?.getTracks().forEach((t) => t.stop());
|
|
105
|
+
}
|
|
106
|
+
catch { /* releasing anyway */ }
|
|
107
|
+
}
|
|
108
|
+
/** The one teardown every terminal path runs. Bumping `gen` also cancels a
|
|
109
|
+
* start() sitting between its awaits. The ENGINE IS NOT DISPOSED -- it belongs
|
|
110
|
+
* to the page, and the next dictation reuses it warm. */
|
|
111
|
+
function teardown() {
|
|
112
|
+
gen += 1;
|
|
113
|
+
listening = false;
|
|
114
|
+
running = false;
|
|
115
|
+
unsubscribe?.();
|
|
116
|
+
unsubscribe = null;
|
|
117
|
+
const s = stream;
|
|
118
|
+
const c = capture;
|
|
119
|
+
stream = null;
|
|
120
|
+
capture = null;
|
|
121
|
+
release(s, c);
|
|
122
|
+
}
|
|
123
|
+
return {
|
|
124
|
+
async start() {
|
|
125
|
+
if (running)
|
|
126
|
+
return;
|
|
127
|
+
running = true;
|
|
128
|
+
const myGen = ++gen;
|
|
129
|
+
const stale = () => myGen !== gen;
|
|
130
|
+
// Mic FIRST. A denial must not warm an engine for a run that never starts.
|
|
131
|
+
let s;
|
|
132
|
+
try {
|
|
133
|
+
s = await getMedia();
|
|
134
|
+
}
|
|
135
|
+
catch (e) {
|
|
136
|
+
running = false;
|
|
137
|
+
opts.onError(mediaError(e));
|
|
138
|
+
return;
|
|
139
|
+
}
|
|
140
|
+
/* The window the generation exists for: stop() ran while the permission
|
|
141
|
+
* prompt was up. This run owns `s` and nothing else can see it. */
|
|
142
|
+
if (stale()) {
|
|
143
|
+
release(s, null);
|
|
144
|
+
return;
|
|
145
|
+
}
|
|
146
|
+
stream = s;
|
|
147
|
+
// Warm on the first dictation only; later ones resolve immediately.
|
|
148
|
+
try {
|
|
149
|
+
await engine.load();
|
|
150
|
+
}
|
|
151
|
+
catch (e) {
|
|
152
|
+
if (stale())
|
|
153
|
+
return;
|
|
154
|
+
opts.onError({ kind: 'inference', message: e.message });
|
|
155
|
+
teardown();
|
|
156
|
+
return;
|
|
157
|
+
}
|
|
158
|
+
if (stale())
|
|
159
|
+
return;
|
|
160
|
+
unsubscribe = engine.subscribe((m) => {
|
|
161
|
+
if (myGen !== gen)
|
|
162
|
+
return;
|
|
163
|
+
if (m.kind === 'interim') {
|
|
164
|
+
opts.onInterim(m.text);
|
|
165
|
+
return;
|
|
166
|
+
}
|
|
167
|
+
if (m.kind === 'final') {
|
|
168
|
+
opts.onFinal(m.text);
|
|
169
|
+
return;
|
|
170
|
+
}
|
|
171
|
+
if (m.kind === 'error') {
|
|
172
|
+
opts.onError({ kind: 'inference', message: m.message });
|
|
173
|
+
teardown();
|
|
174
|
+
}
|
|
175
|
+
}, () => {
|
|
176
|
+
if (myGen !== gen)
|
|
177
|
+
return;
|
|
178
|
+
opts.onError({ kind: 'inference', message: 'The speech engine stopped unexpectedly.' });
|
|
179
|
+
teardown();
|
|
180
|
+
});
|
|
181
|
+
listening = true;
|
|
182
|
+
capture = createCapture(s, (pcm) => {
|
|
183
|
+
if (!listening || myGen !== gen)
|
|
184
|
+
return;
|
|
185
|
+
engine.post({ kind: 'audio', pcm });
|
|
186
|
+
});
|
|
187
|
+
},
|
|
188
|
+
async stop() {
|
|
189
|
+
/* Nothing listening -- but STILL tear down, because this is what cancels a
|
|
190
|
+
* start() stuck at the permission prompt. A bare `return` here is what once
|
|
191
|
+
* left a microphone open with no handle to close it. */
|
|
192
|
+
if (!listening) {
|
|
193
|
+
teardown();
|
|
194
|
+
return;
|
|
195
|
+
}
|
|
196
|
+
// Stop feeding first, so the flush covers a fixed buffer rather than
|
|
197
|
+
// racing new frames into it.
|
|
198
|
+
listening = false;
|
|
199
|
+
try {
|
|
200
|
+
capture?.stop();
|
|
201
|
+
}
|
|
202
|
+
catch { /* tearing down anyway */ }
|
|
203
|
+
capture = null;
|
|
204
|
+
/* THE MICROPHONE GOES NOW, not after the tail arrives. Transcribing the
|
|
205
|
+
* last sentence takes a moment, and leaving the OS recording indicator lit
|
|
206
|
+
* for that long after the user pressed stop is the one thing this feature
|
|
207
|
+
* cannot do. The tail is computed from audio already captured; it does not
|
|
208
|
+
* need the device. */
|
|
209
|
+
const s = stream;
|
|
210
|
+
stream = null;
|
|
211
|
+
try {
|
|
212
|
+
s?.getTracks().forEach((t) => t.stop());
|
|
213
|
+
}
|
|
214
|
+
catch { /* releasing anyway */ }
|
|
215
|
+
/* Hand the tail to a DEDICATED subscriber. The run's own subscriber is
|
|
216
|
+
* dropped first so the final cannot be delivered twice, and this one
|
|
217
|
+
* unsubscribes the moment it has what it came for.
|
|
218
|
+
*
|
|
219
|
+
* stop() used to post the flush and tear down in the same tick, so the
|
|
220
|
+
* final came back to nobody -- every dictation silently discarded the last
|
|
221
|
+
* thing you said. */
|
|
222
|
+
await new Promise((resolve) => {
|
|
223
|
+
unsubscribe?.();
|
|
224
|
+
unsubscribe = null;
|
|
225
|
+
let settled = false;
|
|
226
|
+
let off = null;
|
|
227
|
+
let timer = null;
|
|
228
|
+
const finish = () => {
|
|
229
|
+
if (settled)
|
|
230
|
+
return;
|
|
231
|
+
settled = true;
|
|
232
|
+
if (timer)
|
|
233
|
+
clearTimeout(timer);
|
|
234
|
+
off?.();
|
|
235
|
+
resolve();
|
|
236
|
+
};
|
|
237
|
+
off = engine.subscribe((m) => {
|
|
238
|
+
if (m.kind !== 'final')
|
|
239
|
+
return;
|
|
240
|
+
// An empty flush is what a stop with no speech in it produces; it is
|
|
241
|
+
// not a transcript and must not reach the draft.
|
|
242
|
+
const text = m.text.trim();
|
|
243
|
+
if (text)
|
|
244
|
+
opts.onFinal(text);
|
|
245
|
+
finish();
|
|
246
|
+
}, () => finish());
|
|
247
|
+
timer = setTimeout(finish, flushTimeoutMs);
|
|
248
|
+
engine.post({ kind: 'flush' });
|
|
249
|
+
});
|
|
250
|
+
teardown();
|
|
251
|
+
},
|
|
252
|
+
};
|
|
253
|
+
}
|
|
254
|
+
//# sourceMappingURL=session.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"session.js","sourceRoot":"","sources":["../../speech/session.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;sEAcsE;AACtE,OAAO,EAAE,WAAW,EAAqB,MAAM,YAAY,CAAC;AAC5D,OAAO,EAAE,kBAAkB,EAAqB,MAAM,UAAU,CAAC;AA+BjE,MAAM,eAAe,GAAG,GAAyB,EAAE,CACjD,SAAS,CAAC,YAAY,CAAC,YAAY,CAAC,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;AAEvD;;;;;;;;;;;;;;;;;;;0BAmB0B;AAC1B,MAAM,CAAC,MAAM,oBAAoB,GAAkB,CAAC,MAAM,EAAE,OAAO,EAAE,EAAE;IACrE,MAAM,GAAG,GAAG,IAAI,YAAY,CAAC,EAAE,UAAU,EAAE,WAAW,EAAE,CAAC,CAAC;IAC1D,2EAA2E;IAC3E,yEAAyE;IACzE,IAAI,GAAG,CAAC,KAAK,KAAK,WAAW;QAAE,KAAK,GAAG,CAAC,MAAM,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;IACjE,MAAM,MAAM,GAAG,GAAG,CAAC,uBAAuB,CAAC,MAAM,CAAC,CAAC;IACnD,MAAM,IAAI,GAAG,GAAG,CAAC,qBAAqB,CAAC,IAAI,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;IACnD,MAAM,IAAI,GAAG,GAAG,CAAC,UAAU,EAAE,CAAC;IAC9B,IAAI,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC;IACpB,IAAI,CAAC,cAAc,GAAG,CAAC,CAAC,EAAE,EAAE,GAAG,OAAO,CAAC,IAAI,YAAY,CAAC,CAAC,CAAC,WAAW,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAC7F,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;IACrB,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;IACnB,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC;IAC9B,OAAO;QACL,IAAI,EAAE,GAAG,EAAE;YACT,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC;YAC3B,IAAI,CAAC,UAAU,EAAE,CAAC;YAClB,IAAI,CAAC,UAAU,EAAE,CAAC;YAClB,MAAM,CAAC,UAAU,EAAE,CAAC;YACpB,KAAK,GAAG,CAAC,KAAK,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;QACnC,CAAC;KACF,CAAC;AACJ,CAAC,CAAC;AAEF,sEAAsE;AACtE,SAAS,UAAU,CAAC,CAAU;IAC5B,MAAM,IAAI,GAAI,CAAuB,CAAC,IAAI,CAAC;IAC3C,IAAI,IAAI,KAAK,iBAAiB,IAAI,IAAI,KAAK,uBAAuB,EAAE,CAAC;QACnE,OAAO,EAAE,IAAI,EAAE,YAAY,EAAE,OAAO,EAAE,mCAAmC,EAAE,CAAC;IAC9E,CAAC;IACD,IAAI,IAAI,KAAK,eAAe,IAAI,IAAI,KAAK,sBAAsB,EAAE,CAAC;QAChE,OAAO,EAAE,IAAI,EAAE,aAAa,EAAE,OAAO,EAAE,0BAA0B,EAAE,CAAC;IACtE,CAAC;IACD,OAAO,EAAE,IAAI,EAAE,aAAa,EAAE,OAAO,EAAE,gDAAgD,EAAE,CAAC;AAC5F,CAAC;AAED,MAAM,UAAU,eAAe,CAAC,IAAmB;IACjD,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,IAAI,kBAAkB,EAAE,CAAC;IACnD,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,IAAI,eAAe,CAAC;IAClD,MAAM,aAAa,GAAG,IAAI,CAAC,aAAa,IAAI,oBAAoB,CAAC;IACjE,MAAM,cAAc,GAAG,IAAI,CAAC,cAAc,IAAI,IAAI,CAAC;IAEnD,IAAI,MAAM,GAAuB,IAAI,CAAC;IACtC,IAAI,OAAO,GAAmB,IAAI,CAAC;IACnC,IAAI,WAAW,GAAwB,IAAI,CAAC;IAC5C,IAAI,SAAS,GAAG,KAAK,CAAC;IACtB,IAAI,OAAO,GAAG,KAAK,CAAC;IACpB;;;;;;;;;;0CAUsC;IACtC,IAAI,GAAG,GAAG,CAAC,CAAC;IAEZ,2EAA2E;IAC3E,SAAS,OAAO,CAAC,CAAqB,EAAE,CAAiB;QACvD,IAAI,CAAC;YAAC,CAAC,EAAE,IAAI,EAAE,CAAC;QAAC,CAAC;QAAC,MAAM,CAAC,CAAC,sBAAsB,CAAC,CAAC;QACnD,wDAAwD;QACxD,IAAI,CAAC;YAAC,CAAC,EAAE,SAAS,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;QAAC,CAAC;QAAC,MAAM,CAAC,CAAC,sBAAsB,CAAC,CAAC;IACnF,CAAC;IAED;;6DAEyD;IACzD,SAAS,QAAQ;QACf,GAAG,IAAI,CAAC,CAAC;QACT,SAAS,GAAG,KAAK,CAAC;QAClB,OAAO,GAAG,KAAK,CAAC;QAChB,WAAW,EAAE,EAAE,CAAC;QAChB,WAAW,GAAG,IAAI,CAAC;QACnB,MAAM,CAAC,GAAG,MAAM,CAAC;QAAC,MAAM,CAAC,GAAG,OAAO,CAAC;QACpC,MAAM,GAAG,IAAI,CAAC;QAAC,OAAO,GAAG,IAAI,CAAC;QAC9B,OAAO,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IAChB,CAAC;IAED,OAAO;QACL,KAAK,CAAC,KAAK;YACT,IAAI,OAAO;gBAAE,OAAO;YACpB,OAAO,GAAG,IAAI,CAAC;YACf,MAAM,KAAK,GAAG,EAAE,GAAG,CAAC;YACpB,MAAM,KAAK,GAAG,GAAY,EAAE,CAAC,KAAK,KAAK,GAAG,CAAC;YAE3C,2EAA2E;YAC3E,IAAI,CAAc,CAAC;YACnB,IAAI,CAAC;gBACH,CAAC,GAAG,MAAM,QAAQ,EAAE,CAAC;YACvB,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACX,OAAO,GAAG,KAAK,CAAC;gBAChB,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC;gBAC5B,OAAO;YACT,CAAC;YACD;+EACmE;YACnE,IAAI,KAAK,EAAE,EAAE,CAAC;gBAAC,OAAO,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;gBAAC,OAAO;YAAC,CAAC;YAC1C,MAAM,GAAG,CAAC,CAAC;YAEX,oEAAoE;YACpE,IAAI,CAAC;gBACH,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;YACtB,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACX,IAAI,KAAK,EAAE;oBAAE,OAAO;gBACpB,IAAI,CAAC,OAAO,CAAC,EAAE,IAAI,EAAE,WAAW,EAAE,OAAO,EAAG,CAAW,CAAC,OAAO,EAAE,CAAC,CAAC;gBACnE,QAAQ,EAAE,CAAC;gBACX,OAAO;YACT,CAAC;YACD,IAAI,KAAK,EAAE;gBAAE,OAAO;YAEpB,WAAW,GAAG,MAAM,CAAC,SAAS,CAC5B,CAAC,CAAC,EAAE,EAAE;gBACJ,IAAI,KAAK,KAAK,GAAG;oBAAE,OAAO;gBAC1B,IAAI,CAAC,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;oBAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;oBAAC,OAAO;gBAAC,CAAC;gBAC7D,IAAI,CAAC,CAAC,IAAI,KAAK,OAAO,EAAE,CAAC;oBAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;oBAAC,OAAO;gBAAC,CAAC;gBACzD,IAAI,CAAC,CAAC,IAAI,KAAK,OAAO,EAAE,CAAC;oBACvB,IAAI,CAAC,OAAO,CAAC,EAAE,IAAI,EAAE,WAAW,EAAE,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC;oBACxD,QAAQ,EAAE,CAAC;gBACb,CAAC;YACH,CAAC,EACD,GAAG,EAAE;gBACH,IAAI,KAAK,KAAK,GAAG;oBAAE,OAAO;gBAC1B,IAAI,CAAC,OAAO,CAAC,EAAE,IAAI,EAAE,WAAW,EAAE,OAAO,EAAE,yCAAyC,EAAE,CAAC,CAAC;gBACxF,QAAQ,EAAE,CAAC;YACb,CAAC,CACF,CAAC;YAEF,SAAS,GAAG,IAAI,CAAC;YACjB,OAAO,GAAG,aAAa,CAAC,CAAC,EAAE,CAAC,GAAG,EAAE,EAAE;gBACjC,IAAI,CAAC,SAAS,IAAI,KAAK,KAAK,GAAG;oBAAE,OAAO;gBACxC,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,GAAG,EAAyB,CAAC,CAAC;YAC7D,CAAC,CAAC,CAAC;QACL,CAAC;QAED,KAAK,CAAC,IAAI;YACR;;oEAEwD;YACxD,IAAI,CAAC,SAAS,EAAE,CAAC;gBAAC,QAAQ,EAAE,CAAC;gBAAC,OAAO;YAAC,CAAC;YAEvC,qEAAqE;YACrE,6BAA6B;YAC7B,SAAS,GAAG,KAAK,CAAC;YAClB,IAAI,CAAC;gBAAC,OAAO,EAAE,IAAI,EAAE,CAAC;YAAC,CAAC;YAAC,MAAM,CAAC,CAAC,yBAAyB,CAAC,CAAC;YAC5D,OAAO,GAAG,IAAI,CAAC;YAEf;;;;kCAIsB;YACtB,MAAM,CAAC,GAAG,MAAM,CAAC;YACjB,MAAM,GAAG,IAAI,CAAC;YACd,IAAI,CAAC;gBAAC,CAAC,EAAE,SAAS,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;YAAC,CAAC;YAAC,MAAM,CAAC,CAAC,sBAAsB,CAAC,CAAC;YAEjF;;;;;;iCAMqB;YACrB,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE;gBAClC,WAAW,EAAE,EAAE,CAAC;gBAChB,WAAW,GAAG,IAAI,CAAC;gBACnB,IAAI,OAAO,GAAG,KAAK,CAAC;gBACpB,IAAI,GAAG,GAAwB,IAAI,CAAC;gBACpC,IAAI,KAAK,GAAyC,IAAI,CAAC;gBACvD,MAAM,MAAM,GAAG,GAAS,EAAE;oBACxB,IAAI,OAAO;wBAAE,OAAO;oBACpB,OAAO,GAAG,IAAI,CAAC;oBACf,IAAI,KAAK;wBAAE,YAAY,CAAC,KAAK,CAAC,CAAC;oBAC/B,GAAG,EAAE,EAAE,CAAC;oBACR,OAAO,EAAE,CAAC;gBACZ,CAAC,CAAC;gBACF,GAAG,GAAG,MAAM,CAAC,SAAS,CACpB,CAAC,CAAC,EAAE,EAAE;oBACJ,IAAI,CAAC,CAAC,IAAI,KAAK,OAAO;wBAAE,OAAO;oBAC/B,qEAAqE;oBACrE,iDAAiD;oBACjD,MAAM,IAAI,GAAG,CAAC,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;oBAC3B,IAAI,IAAI;wBAAE,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;oBAC7B,MAAM,EAAE,CAAC;gBACX,CAAC,EACD,GAAG,EAAE,CAAC,MAAM,EAAE,CACf,CAAC;gBACF,KAAK,GAAG,UAAU,CAAC,MAAM,EAAE,cAAc,CAAC,CAAC;gBAC3C,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,OAAO,EAAyB,CAAC,CAAC;YACxD,CAAC,CAAC,CAAC;YAEH,QAAQ,EAAE,CAAC;QACb,CAAC;KACF,CAAC;AACJ,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"speechWorker.d.ts","sourceRoot":"","sources":["../../speech/speechWorker.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
/* The dictation worker: ONNX inference off the main thread.
|
|
2
|
+
*
|
|
3
|
+
* Inference on the main thread would jank the composer while the user types --
|
|
4
|
+
* which is the one place this feature must not be felt, since the user is
|
|
5
|
+
* looking at the text as it lands.
|
|
6
|
+
*
|
|
7
|
+
* Interim vs final is decided HERE, and the rule is simple: everything produced
|
|
8
|
+
* while audio is still arriving is a revisable guess (`interim`); the flush at
|
|
9
|
+
* the end of a dictation produces the committed text (`final`). The main thread
|
|
10
|
+
* trusts that distinction absolutely -- only `final` reaches the user's draft. */
|
|
11
|
+
import { SPEECH_MODEL_ID, SAMPLE_RATE } from './protocol';
|
|
12
|
+
/* STATIC, not dynamic -- and that is deliberate.
|
|
13
|
+
*
|
|
14
|
+
* THIS FILE IS ITSELF THE LAZY BOUNDARY: the worker is constructed only when a
|
|
15
|
+
* dictation starts, so nothing here is fetched until then. A dynamic import
|
|
16
|
+
* inside it buys no extra laziness and costs the build outright -- it forces
|
|
17
|
+
* code-splitting in the worker bundle, and Vite emits workers as `iife` by
|
|
18
|
+
* default, which cannot code-split ("Invalid value 'iife' for option
|
|
19
|
+
* output.format"). The alternative was setting worker.format = 'es' globally in
|
|
20
|
+
* vite.shared.ts, which changes worker output for both apps and the existing
|
|
21
|
+
* agent worker to fix one module's mistake. */
|
|
22
|
+
import { pipeline } from '@huggingface/transformers';
|
|
23
|
+
let transcriber = null;
|
|
24
|
+
/** Everything heard since the last flush, at SAMPLE_RATE. */
|
|
25
|
+
let buffer = new Float32Array(0);
|
|
26
|
+
/** Guards against a second inference starting while one is running -- they would
|
|
27
|
+
* queue up behind each other and interim text would fall further behind speech. */
|
|
28
|
+
let busy = false;
|
|
29
|
+
const post = (m) => { self.postMessage(m); };
|
|
30
|
+
/** Re-transcribe roughly this often. Moonshine is built for short utterances, so
|
|
31
|
+
* re-running the whole buffer is cheap and avoids stitching partial windows. */
|
|
32
|
+
const INTERIM_EVERY_SAMPLES = SAMPLE_RATE; // ~1s of speech
|
|
33
|
+
let sinceInterim = 0;
|
|
34
|
+
function append(pcm) {
|
|
35
|
+
const next = new Float32Array(buffer.length + pcm.length);
|
|
36
|
+
next.set(buffer, 0);
|
|
37
|
+
next.set(pcm, buffer.length);
|
|
38
|
+
buffer = next;
|
|
39
|
+
}
|
|
40
|
+
function textOf(out) {
|
|
41
|
+
const one = Array.isArray(out) ? out[0] : out;
|
|
42
|
+
return (one?.text ?? '').trim();
|
|
43
|
+
}
|
|
44
|
+
async function load() {
|
|
45
|
+
if (transcriber) {
|
|
46
|
+
post({ kind: 'ready' });
|
|
47
|
+
return;
|
|
48
|
+
}
|
|
49
|
+
try {
|
|
50
|
+
const p = await pipeline('automatic-speech-recognition', SPEECH_MODEL_ID, {
|
|
51
|
+
dtype: 'q8',
|
|
52
|
+
progress_callback: (x) => {
|
|
53
|
+
const pct = x.progress;
|
|
54
|
+
if (typeof pct === 'number')
|
|
55
|
+
post({ kind: 'progress', pct: Math.round(pct) });
|
|
56
|
+
},
|
|
57
|
+
});
|
|
58
|
+
transcriber = p;
|
|
59
|
+
post({ kind: 'ready' });
|
|
60
|
+
}
|
|
61
|
+
catch (e) {
|
|
62
|
+
post({ kind: 'error', message: `The speech engine failed to load: ${e.message}` });
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
async function transcribe(final) {
|
|
66
|
+
if (!transcriber || buffer.length === 0) {
|
|
67
|
+
if (final)
|
|
68
|
+
post({ kind: 'final', text: '' });
|
|
69
|
+
return;
|
|
70
|
+
}
|
|
71
|
+
try {
|
|
72
|
+
const text = textOf(await transcriber(buffer));
|
|
73
|
+
if (final) {
|
|
74
|
+
post({ kind: 'final', text });
|
|
75
|
+
buffer = new Float32Array(0);
|
|
76
|
+
sinceInterim = 0;
|
|
77
|
+
}
|
|
78
|
+
else if (text) {
|
|
79
|
+
post({ kind: 'interim', text });
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
catch (e) {
|
|
83
|
+
post({ kind: 'error', message: `Transcription failed: ${e.message}` });
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
self.onmessage = (ev) => {
|
|
87
|
+
const m = ev.data;
|
|
88
|
+
if (m.kind === 'load') {
|
|
89
|
+
void load();
|
|
90
|
+
return;
|
|
91
|
+
}
|
|
92
|
+
if (m.kind === 'audio') {
|
|
93
|
+
append(m.pcm);
|
|
94
|
+
sinceInterim += m.pcm.length;
|
|
95
|
+
if (sinceInterim >= INTERIM_EVERY_SAMPLES && !busy) {
|
|
96
|
+
sinceInterim = 0;
|
|
97
|
+
busy = true;
|
|
98
|
+
void transcribe(false).finally(() => { busy = false; });
|
|
99
|
+
}
|
|
100
|
+
return;
|
|
101
|
+
}
|
|
102
|
+
if (m.kind === 'flush')
|
|
103
|
+
void transcribe(true);
|
|
104
|
+
};
|
|
105
|
+
//# sourceMappingURL=speechWorker.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"speechWorker.js","sourceRoot":"","sources":["../../speech/speechWorker.ts"],"names":[],"mappings":"AAAA;;;;;;;;;kFASkF;AAClF,OAAO,EAAE,eAAe,EAAE,WAAW,EAAwC,MAAM,YAAY,CAAC;AAChG;;;;;;;;;+CAS+C;AAC/C,OAAO,EAAE,QAAQ,EAAE,MAAM,2BAA2B,CAAC;AAIrD,IAAI,WAAW,GAAuB,IAAI,CAAC;AAC3C,6DAA6D;AAC7D,IAAI,MAAM,GAAG,IAAI,YAAY,CAAC,CAAC,CAAC,CAAC;AACjC;mFACmF;AACnF,IAAI,IAAI,GAAG,KAAK,CAAC;AAEjB,MAAM,IAAI,GAAG,CAAC,CAAe,EAAQ,EAAE,GAAI,IAA0B,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AAExF;gFACgF;AAChF,MAAM,qBAAqB,GAAG,WAAW,CAAC,CAAC,gBAAgB;AAE3D,IAAI,YAAY,GAAG,CAAC,CAAC;AAErB,SAAS,MAAM,CAAC,GAAiB;IAC/B,MAAM,IAAI,GAAG,IAAI,YAAY,CAAC,MAAM,CAAC,MAAM,GAAG,GAAG,CAAC,MAAM,CAAC,CAAC;IAC1D,IAAI,CAAC,GAAG,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;IACpB,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC;IAC7B,MAAM,GAAG,IAAI,CAAC;AAChB,CAAC;AAED,SAAS,MAAM,CAAC,GAAqC;IACnD,MAAM,GAAG,GAAG,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC;IAC9C,OAAO,CAAC,GAAG,EAAE,IAAI,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;AAClC,CAAC;AAED,KAAK,UAAU,IAAI;IACjB,IAAI,WAAW,EAAE,CAAC;QAAC,IAAI,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC,CAAC;QAAC,OAAO;IAAC,CAAC;IACrD,IAAI,CAAC;QACH,MAAM,CAAC,GAAG,MAAM,QAAQ,CAAC,8BAA8B,EAAE,eAAe,EAAE;YACxE,KAAK,EAAE,IAAI;YACX,iBAAiB,EAAE,CAAC,CAAU,EAAE,EAAE;gBAChC,MAAM,GAAG,GAAI,CAA2B,CAAC,QAAQ,CAAC;gBAClD,IAAI,OAAO,GAAG,KAAK,QAAQ;oBAAE,IAAI,CAAC,EAAE,IAAI,EAAE,UAAU,EAAE,GAAG,EAAE,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YAChF,CAAC;SACF,CAAC,CAAC;QACH,WAAW,GAAG,CAA2B,CAAC;QAC1C,IAAI,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC,CAAC;IAC1B,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,IAAI,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,OAAO,EAAE,qCAAsC,CAAW,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;IAChG,CAAC;AACH,CAAC;AAED,KAAK,UAAU,UAAU,CAAC,KAAc;IACtC,IAAI,CAAC,WAAW,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACxC,IAAI,KAAK;YAAE,IAAI,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC,CAAC;QAC7C,OAAO;IACT,CAAC;IACD,IAAI,CAAC;QACH,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC;QAC/C,IAAI,KAAK,EAAE,CAAC;YACV,IAAI,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC;YAC9B,MAAM,GAAG,IAAI,YAAY,CAAC,CAAC,CAAC,CAAC;YAC7B,YAAY,GAAG,CAAC,CAAC;QACnB,CAAC;aAAM,IAAI,IAAI,EAAE,CAAC;YAChB,IAAI,CAAC,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAClC,CAAC;IACH,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,IAAI,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,OAAO,EAAE,yBAA0B,CAAW,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;IACpF,CAAC;AACH,CAAC;AAED,IAAI,CAAC,SAAS,GAAG,CAAC,EAA8B,EAAQ,EAAE;IACxD,MAAM,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC;IAClB,IAAI,CAAC,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;QAAC,KAAK,IAAI,EAAE,CAAC;QAAC,OAAO;IAAC,CAAC;IAE/C,IAAI,CAAC,CAAC,IAAI,KAAK,OAAO,EAAE,CAAC;QACvB,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;QACd,YAAY,IAAI,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC;QAC7B,IAAI,YAAY,IAAI,qBAAqB,IAAI,CAAC,IAAI,EAAE,CAAC;YACnD,YAAY,GAAG,CAAC,CAAC;YACjB,IAAI,GAAG,IAAI,CAAC;YACZ,KAAK,UAAU,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC,GAAG,EAAE,GAAG,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;QAC1D,CAAC;QACD,OAAO;IACT,CAAC;IAED,IAAI,CAAC,CAAC,IAAI,KAAK,OAAO;QAAE,KAAK,UAAU,CAAC,IAAI,CAAC,CAAC;AAChD,CAAC,CAAC"}
|
package/dist/workerClient.d.ts
CHANGED
|
@@ -10,6 +10,7 @@ import type { FileService } from '@corenel/protocol';
|
|
|
10
10
|
import type { ToolDecl } from '@corenel/protocol';
|
|
11
11
|
import type { IMemoryProvider } from '@corenel/protocol';
|
|
12
12
|
import type { RunConstraints, Breach, BudgetState } from '@corenel/harness/core/budget';
|
|
13
|
+
import type { ModelRouting } from '@corenel/harness/providers/gateway';
|
|
13
14
|
/** Options for runNode — like RunOpts but the node's internal turns arrive via
|
|
14
15
|
* onNodeEvent (the node-tagged stream); onEvent only ever sees a terminal error. */
|
|
15
16
|
export interface RunNodeOpts {
|
|
@@ -141,6 +142,11 @@ export interface WorkerAgentOpts {
|
|
|
141
142
|
* run. A getter so connecting/disconnecting a sidecar is picked up per run
|
|
142
143
|
* without recreating the worker. Mirrors getFileService. */
|
|
143
144
|
getRemoteTools?: () => ToolDecl[];
|
|
145
|
+
/** Model routing to carry to the worker per run (the setting and the custom
|
|
146
|
+
* providers live in localStorage/sessionStorage, which the worker cannot
|
|
147
|
+
* read). A getter so a Settings change applies to the next run without
|
|
148
|
+
* recreating the worker. Mirrors getGraphConfig. */
|
|
149
|
+
getModelRouting?: () => ModelRouting | undefined;
|
|
144
150
|
/** Execute a sidecar tool by name on the main thread (which holds the WS
|
|
145
151
|
* connection + governs). Answers the worker's tool-request. `signal` aborts
|
|
146
152
|
* when the calling RUN aborts or ends — long host-side executions
|