@try-works/tool-mode-runtime-onnx 0.0.0-stage → 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,172 @@
1
+ /**
2
+ * Processor `clear/stft-v1`: DeepFilterNet3 speech enhancement (Phase 3, after uhm).
3
+ *
4
+ * The graph takes HOST-COMPUTED features (`spec`, `feat_erb`, `feat_spec`) and returns
5
+ * `spec_enhanced`; the DSP that computes and inverts them is `clear-dsp.ts`, pinned to the
6
+ * primary source. This module is the pipeline: window the spectrum, run the model per
7
+ * 200-frame chunk, overlap-add, master, encode a WAV under the host's result budget.
8
+ *
9
+ * OUTPUT IS BYTES, NOT A PATH. InvocationContextV1 has no writable path (resolvePath
10
+ * grants reads), so the enhanced audio travels back as base64 WAV and the caller writes it.
11
+ *
12
+ * @module clear-processor
13
+ */
14
+ import { Effect } from 'effect';
15
+ import { CLEAR_CHUNK_FRAMES, CLEAR_N_DF, CLEAR_N_ERB, CLEAR_N_FREQ, CLEAR_SAMPLE_RATE, ClearStft, clearChunkInputs, clearFeatures, encodePcm16Wav, maxSamplesForResultBudget, normalizeLoudnessInPlace, scatterEnhanced, } from "./clear-dsp.js";
16
+ import { InferenceFailed, InvalidInput, describeCause } from "./errors.js";
17
+ /** The recipe processor id this module implements. */
18
+ export const CLEAR_STFT_PROCESSOR = 'clear/stft-v1';
19
+ const PROVIDER = 'onnx';
20
+ /** The three input tensor names the pinned graph exposes (verified on the artifact). */
21
+ const REQUIRED_INPUTS = ['spec', 'feat_erb', 'feat_spec'];
22
+ /** The output tensor name the pinned graph exposes. */
23
+ const REQUIRED_OUTPUT = 'spec_enhanced';
24
+ /**
25
+ * The clear processor definition.
26
+ *
27
+ * @returns the definition the provider factory dispatches to
28
+ */
29
+ export function clearProcessor() {
30
+ return {
31
+ id: CLEAR_STFT_PROCESSOR,
32
+ sampleRate: CLEAR_SAMPLE_RATE,
33
+ loadSidecar: async () => undefined,
34
+ bindGraph: (names) => {
35
+ const missing = REQUIRED_INPUTS.filter((name) => !names.inputs.includes(name));
36
+ if (missing.length > 0) {
37
+ return {
38
+ ok: false,
39
+ reason: 'the clear graph is missing the input tensor(s) '
40
+ + missing.join(', ')
41
+ + '; it exposes '
42
+ + (names.inputs.join(', ') || 'none'),
43
+ };
44
+ }
45
+ if (!names.outputs.includes(REQUIRED_OUTPUT)) {
46
+ return {
47
+ ok: false,
48
+ reason: 'the clear graph has no ' + REQUIRED_OUTPUT + ' output; it exposes ' + (names.outputs.join(', ') || 'none'),
49
+ };
50
+ }
51
+ return { ok: true, value: { spec: 'spec', featErb: 'feat_erb', featSpec: 'feat_spec', output: REQUIRED_OUTPUT } };
52
+ },
53
+ invoke: (request) => Effect.gen(function* () {
54
+ const audio = request.audio.samples;
55
+ if (audio.length === 0) {
56
+ return yield* Effect.fail(new InvalidInput({ field: 'audio_path', message: 'the audio has no samples to enhance' }));
57
+ }
58
+ // Bound the work by the HOST's result budget, not by an invented fixed duration:
59
+ // base64 is 4/3 of the WAV, and the response must serialize within maxResultBytes.
60
+ const maxSamples = maxSamplesForResultBudget(request.context.limits.maxResultBytes);
61
+ if (maxSamples < 1) {
62
+ return yield* Effect.fail(new InvalidInput({
63
+ field: 'input',
64
+ message: 'the host result budget of '
65
+ + String(request.context.limits.maxResultBytes)
66
+ + ' bytes cannot hold an enhanced WAV',
67
+ }));
68
+ }
69
+ const processedSamples = Math.min(audio.length, maxSamples);
70
+ const stft = new ClearStft();
71
+ const spectrum = stft.forward(audio.subarray(0, processedSamples));
72
+ const features = clearFeatures(spectrum.real, spectrum.imag, spectrum.nFrames);
73
+ const outReal = new Float32Array(spectrum.real.length);
74
+ const outImag = new Float32Array(spectrum.imag.length);
75
+ const chunkCount = Math.ceil(spectrum.nFrames / CLEAR_CHUNK_FRAMES);
76
+ for (let chunk = 0; chunk < chunkCount; chunk += 1) {
77
+ const start = chunk * CLEAR_CHUNK_FRAMES;
78
+ const inputs = clearChunkInputs({
79
+ specReal: spectrum.real,
80
+ specImag: spectrum.imag,
81
+ featErb: features.featErb,
82
+ featSpecReal: features.featSpecReal,
83
+ featSpecImag: features.featSpecImag,
84
+ nFrames: spectrum.nFrames,
85
+ start,
86
+ });
87
+ const outputs = yield* request.engine.run({
88
+ [request.binding.spec]: {
89
+ data: inputs.spec,
90
+ dims: [1, 1, CLEAR_CHUNK_FRAMES, CLEAR_N_FREQ, 2],
91
+ },
92
+ [request.binding.featErb]: {
93
+ data: inputs.featErb,
94
+ dims: [1, 1, CLEAR_CHUNK_FRAMES, CLEAR_N_ERB],
95
+ },
96
+ [request.binding.featSpec]: {
97
+ data: inputs.featSpec,
98
+ dims: [1, 1, CLEAR_CHUNK_FRAMES, CLEAR_N_DF, 2],
99
+ },
100
+ });
101
+ const output = outputs[request.binding.output];
102
+ if (output === undefined) {
103
+ return yield* Effect.fail(new InferenceFailed({
104
+ provider: PROVIDER,
105
+ phase: 'post',
106
+ message: 'the graph produced no output named ' + JSON.stringify(request.binding.output),
107
+ }));
108
+ }
109
+ if (!(output.data instanceof Float32Array)) {
110
+ return yield* Effect.fail(new InferenceFailed({
111
+ provider: PROVIDER,
112
+ phase: 'post',
113
+ message: 'the graph output ' + JSON.stringify(request.binding.output) + ' is not a float32 tensor',
114
+ }));
115
+ }
116
+ if (output.data.length < CLEAR_CHUNK_FRAMES * CLEAR_N_FREQ * 2) {
117
+ return yield* Effect.fail(new InferenceFailed({
118
+ provider: PROVIDER,
119
+ phase: 'post',
120
+ message: 'the graph output ' + JSON.stringify(request.binding.output) + ' has '
121
+ + String(output.data.length) + ' values; a full window needs ' + String(CLEAR_CHUNK_FRAMES * CLEAR_N_FREQ * 2),
122
+ }));
123
+ }
124
+ scatterEnhanced({
125
+ enhanced: output.data,
126
+ chunkFrames: CLEAR_CHUNK_FRAMES,
127
+ start,
128
+ validFrames: inputs.validFrames,
129
+ outReal,
130
+ outImag,
131
+ });
132
+ }
133
+ const enhanced = stft.inverse(outReal, outImag, spectrum.nFrames);
134
+ const delivered = enhanced.subarray(0, processedSamples);
135
+ // The loudness stage needs a measurable programme; a short or silent signal is left
136
+ // as the model returned it (the reference behaves the same).
137
+ yield* Effect.try({
138
+ try: () => {
139
+ normalizeLoudnessInPlace(delivered);
140
+ },
141
+ catch: (cause) => new InferenceFailed({ provider: PROVIDER, phase: 'post', message: describeCause(cause) }),
142
+ });
143
+ const wav = encodePcm16Wav(delivered, CLEAR_SAMPLE_RATE);
144
+ const durationSeconds = processedSamples / CLEAR_SAMPLE_RATE;
145
+ const coverageBase = {
146
+ complete: processedSamples === audio.length,
147
+ processedSeconds: durationSeconds,
148
+ totalSeconds: request.audio.sourceDurationSeconds,
149
+ };
150
+ const audioBase64 = Buffer.from(wav).toString('base64');
151
+ if (processedSamples === audio.length) {
152
+ return { audio_base64: audioBase64, sampleRate: CLEAR_SAMPLE_RATE, durationSeconds, coverage: coverageBase };
153
+ }
154
+ return {
155
+ audio_base64: audioBase64,
156
+ sampleRate: CLEAR_SAMPLE_RATE,
157
+ durationSeconds,
158
+ coverage: {
159
+ ...coverageBase,
160
+ reason: 'the '
161
+ + String(request.context.limits.maxResultBytes)
162
+ + '-byte result budget caps the enhanced audio at '
163
+ + String(processedSamples)
164
+ + ' of '
165
+ + String(audio.length)
166
+ + ' samples',
167
+ },
168
+ };
169
+ }),
170
+ };
171
+ }
172
+ //# sourceMappingURL=clear-processor.js.map
@@ -0,0 +1,14 @@
1
+ /**
2
+ * Typed runtime failures, re-exported from the shared contract package.
3
+ *
4
+ * The classes are the SAME VALUES the whole runtime layer uses, so `instanceof` and
5
+ * `catchTag` in any consumer matches the error it saw. `describeCause` is a message
6
+ * helper, not part of the error protocol, and stays local.
7
+ *
8
+ * @module errors
9
+ */
10
+ import { InferenceFailed, InvalidInput, RuntimeUnavailable } from '@try-works/tool-mode-runtime-core';
11
+ export { InferenceFailed, InvalidInput, RuntimeUnavailable };
12
+ /** A short, always-defined description of an unknown thrown value. */
13
+ export declare const describeCause: (cause: unknown) => string;
14
+ //# sourceMappingURL=errors.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"errors.d.ts","sourceRoot":"","sources":["../src/errors.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAIH,OAAO,EAAE,eAAe,EAAE,YAAY,EAAE,kBAAkB,EAAE,MAAM,mCAAmC,CAAA;AAErG,OAAO,EAAE,eAAe,EAAE,YAAY,EAAE,kBAAkB,EAAE,CAAA;AAE5D,sEAAsE;AACtE,eAAO,MAAM,aAAa,GAAI,OAAO,OAAO,KAAG,MAU9C,CAAA"}
package/lib/errors.js ADDED
@@ -0,0 +1,29 @@
1
+ /**
2
+ * Typed runtime failures, re-exported from the shared contract package.
3
+ *
4
+ * The classes are the SAME VALUES the whole runtime layer uses, so `instanceof` and
5
+ * `catchTag` in any consumer matches the error it saw. `describeCause` is a message
6
+ * helper, not part of the error protocol, and stays local.
7
+ *
8
+ * @module errors
9
+ */
10
+ import { Schema } from 'effect';
11
+ import { InferenceFailed, InvalidInput, RuntimeUnavailable } from '@try-works/tool-mode-runtime-core';
12
+ export { InferenceFailed, InvalidInput, RuntimeUnavailable };
13
+ /** A short, always-defined description of an unknown thrown value. */
14
+ export const describeCause = (cause) => {
15
+ if (cause instanceof Error)
16
+ return cause.message === '' ? cause.name : cause.message;
17
+ if (Schema.is(Schema.String)(cause))
18
+ return cause;
19
+ if (cause === undefined || cause === null)
20
+ return String(cause);
21
+ try {
22
+ const encoded = JSON.stringify(cause);
23
+ return encoded ?? String(cause);
24
+ }
25
+ catch {
26
+ return String(cause);
27
+ }
28
+ };
29
+ //# sourceMappingURL=errors.js.map
package/lib/fft.d.ts ADDED
@@ -0,0 +1,86 @@
1
+ /**
2
+ * The 960-point DFT the clear graph was trained on.
3
+ *
4
+ * WHY NOT 1024. DeepFilterNet3 frames with n_fft = 960 and 481 bins (libDF). A power-of-two
5
+ * FFT of 1024 would be the obvious engineering choice and would produce a DIFFERENT
6
+ * spectrogram, silently out of distribution for the model. So the transform here is an
7
+ * exact 960-point DFT, implemented as Bluestein's algorithm over a radix-2 FFT of
8
+ * 2048 points: mathematically identical to the naive sum, and far cheaper.
9
+ *
10
+ * CONVENTIONS, stated once because two of them are easy to get backwards:
11
+ * - forward: X[k] = sum x[n] e^{-2*pi*i*k*n/N} (the sign vDSP's forward DFT uses);
12
+ * - `inverseUnscaled`: sum X[k] e^{+2*pi*i*k*n/N}, i.e. N times the mathematical IDFT.
13
+ * The clear STFT wants exactly this, because libDF/vDSP's inverse DFT is UNSCALED and
14
+ * the forward pass carries the compensating 1/n_fft gain.
15
+ *
16
+ * Instances hold scratch buffers, so one instance is used sequentially (the STFT does
17
+ * that). Nothing here is thread safe.
18
+ *
19
+ * @module fft
20
+ */
21
+ /** A complex FFT of one fixed power-of-two size. */
22
+ export declare class Radix2Fft {
23
+ /** The transform size. */
24
+ readonly n: number;
25
+ private readonly cos;
26
+ private readonly sin;
27
+ private readonly reverse;
28
+ /**
29
+ * @param n - the transform size, a power of two
30
+ */
31
+ constructor(n: number);
32
+ /**
33
+ * Transform in place.
34
+ *
35
+ * @param real - the real part, length `n`
36
+ * @param imag - the imaginary part, length `n`
37
+ * @param inverse - true for the inverse convention, which also divides by `n`
38
+ */
39
+ transform(real: Float64Array, imag: Float64Array, inverse: boolean): void;
40
+ }
41
+ /** An exact 960-point DFT (Bluestein over a radix-2 FFT). */
42
+ export declare class Dft960 {
43
+ private readonly fft;
44
+ private readonly chirpReal;
45
+ private readonly chirpImag;
46
+ private readonly kernelReal;
47
+ private readonly kernelImag;
48
+ private readonly scratchReal;
49
+ private readonly scratchImag;
50
+ private readonly zeroImag;
51
+ constructor();
52
+ /**
53
+ * Transform a complex frame with Bluestein.
54
+ *
55
+ * @param inputReal - the real part of the input
56
+ * @param inputImag - the imaginary part of the input
57
+ * @param offset - where the 960-sample frame starts
58
+ * @param outReal - the real output buffer
59
+ * @param outImag - the imaginary output buffer
60
+ * @param outOffset - where the 960 bins start in the output buffers
61
+ */
62
+ private forwardComplex;
63
+ /**
64
+ * Forward DFT of a REAL frame (the STFT analysis case).
65
+ *
66
+ * @param real - the real input buffer
67
+ * @param offset - where the 960-sample frame starts
68
+ * @param outReal - the real output buffer
69
+ * @param outImag - the imaginary output buffer
70
+ * @param outOffset - where the 960 bins start in the output buffers
71
+ */
72
+ forward(real: ArrayLike<number>, offset: number, outReal: Float32Array, outImag: Float32Array, outOffset: number): void;
73
+ /**
74
+ * The UNSCALED inverse DFT: sum X[k] e^{+2*pi*i*k*n/N}, i.e. N times the mathematical
75
+ * IDFT. conj(F(conj(X))) computes it with one forward transform.
76
+ *
77
+ * @param real - the real spectrum buffer
78
+ * @param imag - the imaginary spectrum buffer
79
+ * @param offset - where the 960 bins start
80
+ * @param outReal - the real time-domain output
81
+ * @param outImag - the imaginary time-domain output
82
+ * @param outOffset - where the 960 samples start in the output buffers
83
+ */
84
+ inverseUnscaled(real: ArrayLike<number>, imag: ArrayLike<number>, offset: number, outReal: Float32Array, outImag: Float32Array, outOffset: number): void;
85
+ }
86
+ //# sourceMappingURL=fft.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"fft.d.ts","sourceRoot":"","sources":["../src/fft.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;GAmBG;AAEH,oDAAoD;AACpD,qBAAa,SAAS;IACpB,0BAA0B;IAC1B,QAAQ,CAAC,CAAC,EAAE,MAAM,CAAA;IAClB,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAc;IAClC,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAc;IAClC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAa;IAErC;;OAEG;gBACS,CAAC,EAAE,MAAM;IA0BrB;;;;;;OAMG;IACH,SAAS,CAAC,IAAI,EAAE,YAAY,EAAE,IAAI,EAAE,YAAY,EAAE,OAAO,EAAE,OAAO,GAAG,IAAI;CA+C1E;AAQD,6DAA6D;AAC7D,qBAAa,MAAM;IACjB,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAgC;IACpD,OAAO,CAAC,QAAQ,CAAC,SAAS,CAA6B;IACvD,OAAO,CAAC,QAAQ,CAAC,SAAS,CAA6B;IACvD,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAmC;IAC9D,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAmC;IAC9D,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAmC;IAC/D,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAmC;IAC/D,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAA6B;;IA4BtD;;;;;;;;;OASG;IACH,OAAO,CAAC,cAAc;IA0CtB;;;;;;;;OAQG;IACH,OAAO,CACL,IAAI,EAAE,SAAS,CAAC,MAAM,CAAC,EACvB,MAAM,EAAE,MAAM,EACd,OAAO,EAAE,YAAY,EACrB,OAAO,EAAE,YAAY,EACrB,SAAS,EAAE,MAAM,GAChB,IAAI;IAIP;;;;;;;;;;OAUG;IACH,eAAe,CACb,IAAI,EAAE,SAAS,CAAC,MAAM,CAAC,EACvB,IAAI,EAAE,SAAS,CAAC,MAAM,CAAC,EACvB,MAAM,EAAE,MAAM,EACd,OAAO,EAAE,YAAY,EACrB,OAAO,EAAE,YAAY,EACrB,SAAS,EAAE,MAAM,GAChB,IAAI;CAYR"}
package/lib/fft.js ADDED
@@ -0,0 +1,231 @@
1
+ /**
2
+ * The 960-point DFT the clear graph was trained on.
3
+ *
4
+ * WHY NOT 1024. DeepFilterNet3 frames with n_fft = 960 and 481 bins (libDF). A power-of-two
5
+ * FFT of 1024 would be the obvious engineering choice and would produce a DIFFERENT
6
+ * spectrogram, silently out of distribution for the model. So the transform here is an
7
+ * exact 960-point DFT, implemented as Bluestein's algorithm over a radix-2 FFT of
8
+ * 2048 points: mathematically identical to the naive sum, and far cheaper.
9
+ *
10
+ * CONVENTIONS, stated once because two of them are easy to get backwards:
11
+ * - forward: X[k] = sum x[n] e^{-2*pi*i*k*n/N} (the sign vDSP's forward DFT uses);
12
+ * - `inverseUnscaled`: sum X[k] e^{+2*pi*i*k*n/N}, i.e. N times the mathematical IDFT.
13
+ * The clear STFT wants exactly this, because libDF/vDSP's inverse DFT is UNSCALED and
14
+ * the forward pass carries the compensating 1/n_fft gain.
15
+ *
16
+ * Instances hold scratch buffers, so one instance is used sequentially (the STFT does
17
+ * that). Nothing here is thread safe.
18
+ *
19
+ * @module fft
20
+ */
21
+ /** A complex FFT of one fixed power-of-two size. */
22
+ export class Radix2Fft {
23
+ /** The transform size. */
24
+ n;
25
+ cos;
26
+ sin;
27
+ reverse;
28
+ /**
29
+ * @param n - the transform size, a power of two
30
+ */
31
+ constructor(n) {
32
+ if (!Number.isInteger(n) || n < 2 || (n & (n - 1)) !== 0) {
33
+ throw new Error('the FFT size must be a power of two');
34
+ }
35
+ this.n = n;
36
+ const half = n / 2;
37
+ this.cos = new Float64Array(half);
38
+ this.sin = new Float64Array(half);
39
+ for (let k = 0; k < half; k += 1) {
40
+ const angle = (2 * Math.PI * k) / n;
41
+ this.cos[k] = Math.cos(angle);
42
+ this.sin[k] = Math.sin(angle);
43
+ }
44
+ const bits = Math.round(Math.log2(n));
45
+ this.reverse = new Uint32Array(n);
46
+ for (let i = 0; i < n; i += 1) {
47
+ let value = i;
48
+ let reversed = 0;
49
+ for (let bit = 0; bit < bits; bit += 1) {
50
+ reversed = (reversed << 1) | (value & 1);
51
+ value >>= 1;
52
+ }
53
+ this.reverse[i] = reversed;
54
+ }
55
+ }
56
+ /**
57
+ * Transform in place.
58
+ *
59
+ * @param real - the real part, length `n`
60
+ * @param imag - the imaginary part, length `n`
61
+ * @param inverse - true for the inverse convention, which also divides by `n`
62
+ */
63
+ transform(real, imag, inverse) {
64
+ const n = this.n;
65
+ if (real.length !== n || imag.length !== n)
66
+ throw new Error('the FFT buffers must match the transform size');
67
+ for (let i = 0; i < n; i += 1) {
68
+ const j = this.reverse[i] ?? i;
69
+ if (i < j) {
70
+ const tr = real[i] ?? 0;
71
+ real[i] = real[j] ?? 0;
72
+ real[j] = tr;
73
+ const ti = imag[i] ?? 0;
74
+ imag[i] = imag[j] ?? 0;
75
+ imag[j] = ti;
76
+ }
77
+ }
78
+ const sign = inverse ? 1 : -1;
79
+ for (let length = 2; length <= n; length <<= 1) {
80
+ const half = length >> 1;
81
+ const step = n / length;
82
+ for (let start = 0; start < n; start += length) {
83
+ let twiddle = 0;
84
+ for (let j = 0; j < half; j += 1) {
85
+ const c = this.cos[twiddle] ?? 1;
86
+ const s = sign * (this.sin[twiddle] ?? 0);
87
+ const a = start + j;
88
+ const b = a + half;
89
+ const br = real[b] ?? 0;
90
+ const bi = imag[b] ?? 0;
91
+ const tr = br * c - bi * s;
92
+ const ti = br * s + bi * c;
93
+ const ar = real[a] ?? 0;
94
+ const ai = imag[a] ?? 0;
95
+ real[b] = ar - tr;
96
+ imag[b] = ai - ti;
97
+ real[a] = ar + tr;
98
+ imag[a] = ai + ti;
99
+ twiddle += step;
100
+ }
101
+ }
102
+ }
103
+ if (inverse) {
104
+ const scale = 1 / n;
105
+ for (let i = 0; i < n; i += 1) {
106
+ real[i] = (real[i] ?? 0) * scale;
107
+ imag[i] = (imag[i] ?? 0) * scale;
108
+ }
109
+ }
110
+ }
111
+ }
112
+ /** The DFT size the clear graph was trained on. */
113
+ const DFT_SIZE = 960;
114
+ /** A power of two at least 2*960-1, as Bluestein requires. */
115
+ const BLUESTEIN_SIZE = 2048;
116
+ /** An exact 960-point DFT (Bluestein over a radix-2 FFT). */
117
+ export class Dft960 {
118
+ fft = new Radix2Fft(BLUESTEIN_SIZE);
119
+ chirpReal = new Float64Array(DFT_SIZE);
120
+ chirpImag = new Float64Array(DFT_SIZE);
121
+ kernelReal = new Float64Array(BLUESTEIN_SIZE);
122
+ kernelImag = new Float64Array(BLUESTEIN_SIZE);
123
+ scratchReal = new Float64Array(BLUESTEIN_SIZE);
124
+ scratchImag = new Float64Array(BLUESTEIN_SIZE);
125
+ zeroImag = new Float64Array(DFT_SIZE);
126
+ constructor() {
127
+ // w[k] = exp(-i*pi*k^2/N), with k^2 reduced modulo 2N for precision (as the Swift
128
+ // reference does: exp(-i*pi*(k*k mod 2N)/N)).
129
+ for (let k = 0; k < DFT_SIZE; k += 1) {
130
+ const reduced = (k * k) % (2 * DFT_SIZE);
131
+ const angle = (-Math.PI * reduced) / DFT_SIZE;
132
+ this.chirpReal[k] = Math.cos(angle);
133
+ this.chirpImag[k] = Math.sin(angle);
134
+ }
135
+ const bReal = new Float64Array(BLUESTEIN_SIZE);
136
+ const bImag = new Float64Array(BLUESTEIN_SIZE);
137
+ bReal[0] = this.chirpReal[0] ?? 1;
138
+ bImag[0] = -(this.chirpImag[0] ?? 0);
139
+ for (let k = 1; k < DFT_SIZE; k += 1) {
140
+ const re = this.chirpReal[k] ?? 0;
141
+ const im = -(this.chirpImag[k] ?? 0);
142
+ bReal[k] = re;
143
+ bImag[k] = im;
144
+ bReal[BLUESTEIN_SIZE - k] = re;
145
+ bImag[BLUESTEIN_SIZE - k] = im;
146
+ }
147
+ this.fft.transform(bReal, bImag, false);
148
+ this.kernelReal.set(bReal);
149
+ this.kernelImag.set(bImag);
150
+ }
151
+ /**
152
+ * Transform a complex frame with Bluestein.
153
+ *
154
+ * @param inputReal - the real part of the input
155
+ * @param inputImag - the imaginary part of the input
156
+ * @param offset - where the 960-sample frame starts
157
+ * @param outReal - the real output buffer
158
+ * @param outImag - the imaginary output buffer
159
+ * @param outOffset - where the 960 bins start in the output buffers
160
+ */
161
+ forwardComplex(inputReal, inputImag, offset, outReal, outImag, outOffset) {
162
+ const scratchReal = this.scratchReal;
163
+ const scratchImag = this.scratchImag;
164
+ for (let n = 0; n < DFT_SIZE; n += 1) {
165
+ const xr = inputReal[offset + n] ?? 0;
166
+ const xi = inputImag[offset + n] ?? 0;
167
+ const wr = this.chirpReal[n] ?? 0;
168
+ const wi = this.chirpImag[n] ?? 0;
169
+ scratchReal[n] = xr * wr - xi * wi;
170
+ scratchImag[n] = xr * wi + xi * wr;
171
+ }
172
+ for (let n = DFT_SIZE; n < BLUESTEIN_SIZE; n += 1) {
173
+ scratchReal[n] = 0;
174
+ scratchImag[n] = 0;
175
+ }
176
+ this.fft.transform(scratchReal, scratchImag, false);
177
+ for (let n = 0; n < BLUESTEIN_SIZE; n += 1) {
178
+ const ar = scratchReal[n] ?? 0;
179
+ const ai = scratchImag[n] ?? 0;
180
+ const br = this.kernelReal[n] ?? 0;
181
+ const bi = this.kernelImag[n] ?? 0;
182
+ scratchReal[n] = ar * br - ai * bi;
183
+ scratchImag[n] = ar * bi + ai * br;
184
+ }
185
+ this.fft.transform(scratchReal, scratchImag, true);
186
+ for (let k = 0; k < DFT_SIZE; k += 1) {
187
+ const cr = scratchReal[k] ?? 0;
188
+ const ci = scratchImag[k] ?? 0;
189
+ const wr = this.chirpReal[k] ?? 0;
190
+ const wi = this.chirpImag[k] ?? 0;
191
+ outReal[outOffset + k] = cr * wr - ci * wi;
192
+ outImag[outOffset + k] = cr * wi + ci * wr;
193
+ }
194
+ }
195
+ /**
196
+ * Forward DFT of a REAL frame (the STFT analysis case).
197
+ *
198
+ * @param real - the real input buffer
199
+ * @param offset - where the 960-sample frame starts
200
+ * @param outReal - the real output buffer
201
+ * @param outImag - the imaginary output buffer
202
+ * @param outOffset - where the 960 bins start in the output buffers
203
+ */
204
+ forward(real, offset, outReal, outImag, outOffset) {
205
+ this.forwardComplex(real, this.zeroImag, offset, outReal, outImag, outOffset);
206
+ }
207
+ /**
208
+ * The UNSCALED inverse DFT: sum X[k] e^{+2*pi*i*k*n/N}, i.e. N times the mathematical
209
+ * IDFT. conj(F(conj(X))) computes it with one forward transform.
210
+ *
211
+ * @param real - the real spectrum buffer
212
+ * @param imag - the imaginary spectrum buffer
213
+ * @param offset - where the 960 bins start
214
+ * @param outReal - the real time-domain output
215
+ * @param outImag - the imaginary time-domain output
216
+ * @param outOffset - where the 960 samples start in the output buffers
217
+ */
218
+ inverseUnscaled(real, imag, offset, outReal, outImag, outOffset) {
219
+ const conjReal = new Float64Array(DFT_SIZE);
220
+ const conjImag = new Float64Array(DFT_SIZE);
221
+ for (let k = 0; k < DFT_SIZE; k += 1) {
222
+ conjReal[k] = real[offset + k] ?? 0;
223
+ conjImag[k] = -(imag[offset + k] ?? 0);
224
+ }
225
+ this.forwardComplex(conjReal, conjImag, 0, outReal, outImag, outOffset);
226
+ for (let n = 0; n < DFT_SIZE; n += 1) {
227
+ outImag[outOffset + n] = -(outImag[outOffset + n] ?? 0);
228
+ }
229
+ }
230
+ }
231
+ //# sourceMappingURL=fft.js.map
@@ -0,0 +1,99 @@
1
+ /**
2
+ * Frames, spans and coverage (PHASE3 processor uhm/waveform-v1, steps 3-5).
3
+ *
4
+ * The model emits one probability row per 20 ms frame. Turning 1499 rows per window into
5
+ * spans that a caller can cut a recording with is where the honest-timeline decisions live:
6
+ *
7
+ * - **Every absolute frame is decided ONCE.** Windows overlap by a second, so the last
8
+ * 1450 frames of one window are the first frames of the next. A frame is a FILLER if
9
+ * either window calls it one at or above the threshold — the overlap exists so a word
10
+ * seen at a window edge by one window is not vetoed by the other — and between two
11
+ * filler calls the higher probability wins; an exact tie keeps the earlier window.
12
+ * Averaging instead would invent a confidence neither window produced.
13
+ * - **A filler frame is `class != 0 && probability >= 0.5`.** Class 0 is `not_filler`,
14
+ * and a filler word that the model is not sure about is not worth cutting out.
15
+ * - **One frame gap is tolerated.** A single non-filler frame between two filler frames of
16
+ * the same class is a boundary artefact, not two separate fillers.
17
+ * - **Spans are clipped to the audio length.** The final window is zero padded to 30 s, so
18
+ * frames past the real audio carry no evidence and are dropped; a frame that straddles
19
+ * the end is clipped.
20
+ *
21
+ * @module frames
22
+ */
23
+ import { type AudioWindow } from './wav.ts';
24
+ /** The labels file the pinned asset set carries next to the model. */
25
+ export declare const UHM_LABELS_FILE = "labels.json";
26
+ /** The number of classes the pinned model head emits. */
27
+ export declare const UHM_CLASS_COUNT = 6;
28
+ /** A frame is a filler only at or above this probability. */
29
+ export declare const UHM_FILLER_THRESHOLD = 0.5;
30
+ /** The filler classes the output contract can name (class 0 is not a filler). */
31
+ export declare const UHM_FILLER_TYPES: readonly ["uh", "um", "hmm", "and", "other"];
32
+ /** One filler class name. */
33
+ export type UhmFillerType = (typeof UHM_FILLER_TYPES)[number];
34
+ /** What {@link parseUhmLabels} returns. */
35
+ export type UhmLabelsParseResult = {
36
+ readonly ok: true;
37
+ readonly labels: readonly string[];
38
+ } | {
39
+ readonly ok: false;
40
+ readonly reason: string;
41
+ };
42
+ /** One window's model output, as the session returned it. */
43
+ export interface UhmWindowRun {
44
+ readonly window: AudioWindow;
45
+ /** Frame-major probabilities, length `frames * classes`. */
46
+ readonly data: Float32Array;
47
+ /** The model output dims, either `[1, frames, classes]` or `[frames, classes]`. */
48
+ readonly dims: readonly number[];
49
+ }
50
+ /** One merged filler span, in milliseconds. */
51
+ export interface UhmSpan {
52
+ readonly startMs: number;
53
+ readonly endMs: number;
54
+ readonly type: UhmFillerType;
55
+ readonly confidence: number;
56
+ }
57
+ /** How much of the audio was analysed. */
58
+ export interface UhmCoverage {
59
+ readonly complete: boolean;
60
+ readonly processedSeconds: number;
61
+ readonly totalSeconds: number;
62
+ readonly reason?: string;
63
+ }
64
+ /** What the uhm processor returns. */
65
+ export interface UhmResult {
66
+ readonly fillers: readonly UhmSpan[];
67
+ readonly coverage: UhmCoverage;
68
+ }
69
+ /** The input {@link uhmResultFromWindows} needs. */
70
+ export interface UhmResultInput {
71
+ readonly runs: readonly UhmWindowRun[];
72
+ /** The decoded 16 kHz mono sample count, i.e. the real audio length. */
73
+ readonly totalSamples: number;
74
+ readonly labels: readonly string[];
75
+ }
76
+ /**
77
+ * Parse the pinned `labels.json` into the class order the head emits.
78
+ *
79
+ * The ORDER is the contract, not any name lookup: frame class 1 must be `uh` because the
80
+ * head was trained that way. The names are validated too, because the output contract can
81
+ * only name the five filler classes — a drifted labels file must fail loudly rather than
82
+ * emit a `type` the schema does not allow.
83
+ *
84
+ * @param raw - the parsed JSON document
85
+ * @returns the labels in class order, or the reason they cannot be used
86
+ */
87
+ export declare function parseUhmLabels(raw: unknown): UhmLabelsParseResult;
88
+ /**
89
+ * Turn window outputs into merged filler spans and honest coverage.
90
+ *
91
+ * `totalSamples` is the REAL audio length, so zero padded frames are dropped and spans are
92
+ * clipped; it is never the padded window length.
93
+ *
94
+ * @param input - the window runs, the real sample count and the class order
95
+ * @returns the filler spans and coverage
96
+ * @throws Error when the model output is malformed (the provider maps this to InferenceFailed)
97
+ */
98
+ export declare function uhmResultFromWindows(input: UhmResultInput): UhmResult;
99
+ //# sourceMappingURL=frames.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"frames.d.ts","sourceRoot":"","sources":["../src/frames.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;GAqBG;AAUH,OAAO,EAAsC,KAAK,WAAW,EAAE,MAAM,UAAU,CAAA;AAE/E,sEAAsE;AACtE,eAAO,MAAM,eAAe,gBAAgB,CAAA;AAE5C,yDAAyD;AACzD,eAAO,MAAM,eAAe,IAAI,CAAA;AAEhC,6DAA6D;AAC7D,eAAO,MAAM,oBAAoB,MAAM,CAAA;AAEvC,iFAAiF;AACjF,eAAO,MAAM,gBAAgB,8CAA+C,CAAA;AAE5E,6BAA6B;AAC7B,MAAM,MAAM,aAAa,GAAG,CAAC,OAAO,gBAAgB,CAAC,CAAC,MAAM,CAAC,CAAA;AAE7D,2CAA2C;AAC3C,MAAM,MAAM,oBAAoB,GAC5B;IAAE,QAAQ,CAAC,EAAE,EAAE,IAAI,CAAC;IAAC,QAAQ,CAAC,MAAM,EAAE,SAAS,MAAM,EAAE,CAAA;CAAE,GACzD;IAAE,QAAQ,CAAC,EAAE,EAAE,KAAK,CAAC;IAAC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;CAAE,CAAA;AAEnD,6DAA6D;AAC7D,MAAM,WAAW,YAAY;IAC3B,QAAQ,CAAC,MAAM,EAAE,WAAW,CAAA;IAC5B,4DAA4D;IAC5D,QAAQ,CAAC,IAAI,EAAE,YAAY,CAAA;IAC3B,mFAAmF;IACnF,QAAQ,CAAC,IAAI,EAAE,SAAS,MAAM,EAAE,CAAA;CACjC;AAED,+CAA+C;AAC/C,MAAM,WAAW,OAAO;IACtB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAA;IACxB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;IACtB,QAAQ,CAAC,IAAI,EAAE,aAAa,CAAA;IAC5B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAA;CAC5B;AAED,0CAA0C;AAC1C,MAAM,WAAW,WAAW;IAC1B,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAA;IAC1B,QAAQ,CAAC,gBAAgB,EAAE,MAAM,CAAA;IACjC,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAA;IAC7B,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAA;CACzB;AAED,sCAAsC;AACtC,MAAM,WAAW,SAAS;IACxB,QAAQ,CAAC,OAAO,EAAE,SAAS,OAAO,EAAE,CAAA;IACpC,QAAQ,CAAC,QAAQ,EAAE,WAAW,CAAA;CAC/B;AAED,oDAAoD;AACpD,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,IAAI,EAAE,SAAS,YAAY,EAAE,CAAA;IACtC,wEAAwE;IACxE,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAA;IAC7B,QAAQ,CAAC,MAAM,EAAE,SAAS,MAAM,EAAE,CAAA;CACnC;AAiDD;;;;;;;;;;GAUG;AACH,wBAAgB,cAAc,CAAC,GAAG,EAAE,OAAO,GAAG,oBAAoB,CA4BjE;AAgFD;;;;;;;;;GASG;AACH,wBAAgB,oBAAoB,CAAC,KAAK,EAAE,cAAc,GAAG,SAAS,CAsGrE"}