@try-works/tool-mode-runtime-litert 0.0.0-stage → 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/ear-frontend.d.ts +113 -0
- package/lib/ear-frontend.d.ts.map +1 -0
- package/lib/ear-frontend.js +258 -0
- package/lib/ear.d.ts +51 -0
- package/lib/ear.d.ts.map +1 -0
- package/lib/ear.js +296 -0
- package/lib/emo-tokenizer.d.ts +87 -0
- package/lib/emo-tokenizer.d.ts.map +1 -0
- package/lib/emo-tokenizer.js +365 -0
- package/lib/emo.d.ts +53 -0
- package/lib/emo.d.ts.map +1 -0
- package/lib/emo.js +218 -0
- package/lib/errors.d.ts +14 -0
- package/lib/errors.d.ts.map +1 -0
- package/lib/errors.js +29 -0
- package/lib/index.d.ts +28 -0
- package/lib/index.d.ts.map +1 -0
- package/lib/index.js +34 -0
- package/lib/model.d.ts +103 -0
- package/lib/model.d.ts.map +1 -0
- package/lib/model.js +112 -0
- package/lib/moderator.d.ts +59 -0
- package/lib/moderator.d.ts.map +1 -0
- package/lib/moderator.js +189 -0
- package/lib/pillow.d.ts +75 -0
- package/lib/pillow.d.ts.map +1 -0
- package/lib/pillow.js +187 -0
- package/lib/png.d.ts +27 -0
- package/lib/png.d.ts.map +1 -0
- package/lib/png.js +176 -0
- package/lib/processor.d.ts +43 -0
- package/lib/processor.d.ts.map +1 -0
- package/lib/processor.js +48 -0
- package/lib/provider-factory.d.ts +44 -0
- package/lib/provider-factory.d.ts.map +1 -0
- package/lib/provider-factory.js +219 -0
- package/lib/runtime.d.ts +41 -0
- package/lib/runtime.d.ts.map +1 -0
- package/lib/runtime.js +66 -0
- package/lib/shapes.d.ts +28 -0
- package/lib/shapes.d.ts.map +1 -0
- package/lib/shapes.js +187 -0
- package/lib/shim.d.ts +45 -0
- package/lib/shim.d.ts.map +1 -0
- package/lib/shim.js +215 -0
- package/lib/wav.d.ts +39 -0
- package/lib/wav.d.ts.map +1 -0
- package/lib/wav.js +134 -0
- package/package.json +47 -4
- package/README.md +0 -3
|
@@ -0,0 +1,113 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The ear log-mel front end and window selection, ported from the primary source.
|
|
3
|
+
*
|
|
4
|
+
* PROVENANCE (Desert-Ant-Labs/desert-ant-core @ baf4e6e3b4655c8e54aa8d5a25215b5f35f899a7):
|
|
5
|
+
* - Sources/Ear/Frontend.swift:25-40 geometry (and windowSamples = frames * hop)
|
|
6
|
+
* - Sources/Ear/Frontend.swift:48-63 Hann periodic + center STFT + filterbank size checks
|
|
7
|
+
* - Sources/Ear/Frontend.swift:65-106 features: truncate/pad to one window, STFT, mel energies,
|
|
8
|
+
* log10(clamp), floor relative to the peak, affine
|
|
9
|
+
* - Sources/Ear/Frontend.swift:137-221 windowOffsets + modulationScores
|
|
10
|
+
* - Sources/Ear/Frontend.swift:223-242 mel filterbank parse (uint32 LE mels/bins then f32 LE)
|
|
11
|
+
* - Sources/AudioDSP/STFT.swift:68-106 (bins = nFFT/2 + 1), :109-158 (REFLECT pad, frames)
|
|
12
|
+
* - Sources/AudioDSP/Windowing.swift:21-27 Hann periodic, :30-41 reflect padding
|
|
13
|
+
* - .assets/ear/ear_meta.json: 16 kHz, n_fft 400, hop 160, mels 80, frames 3000, Hann periodic,
|
|
14
|
+
* log10, clamp 1e-10, floor 8 decades, affine (x + 4) / 4.
|
|
15
|
+
*
|
|
16
|
+
* nFFT is not a power of two, so the source's portable path is an exact real-DFT matmul
|
|
17
|
+
* (STFT.swift:142-158). This module does the same transform, in double precision, then stores
|
|
18
|
+
* the features as float32 -- the tensor the model consumes.
|
|
19
|
+
*/
|
|
20
|
+
/** The model's sample rate. */
|
|
21
|
+
export declare const EAR_SAMPLE_RATE = 16000;
|
|
22
|
+
/** Analysis window length. */
|
|
23
|
+
export declare const EAR_N_FFT = 400;
|
|
24
|
+
/** Hop between analysis frames. */
|
|
25
|
+
export declare const EAR_HOP = 160;
|
|
26
|
+
/** Mel bands. */
|
|
27
|
+
export declare const EAR_MELS = 80;
|
|
28
|
+
/** Frames the model consumes. */
|
|
29
|
+
export declare const EAR_FRAMES = 3000;
|
|
30
|
+
/** Spectrum bins: nFFT / 2 + 1. */
|
|
31
|
+
export declare const EAR_BINS: number;
|
|
32
|
+
/** Samples in one 30 s analysis window. */
|
|
33
|
+
export declare const EAR_WINDOW_SAMPLES: number;
|
|
34
|
+
/** Floor applied to mel energies before the logarithm. */
|
|
35
|
+
export declare const EAR_CLAMP_MIN = 1e-10;
|
|
36
|
+
/** How far below the loudest bin the features are floored, in decades. */
|
|
37
|
+
export declare const EAR_FLOOR_DECADES = 8;
|
|
38
|
+
/** Affine add. */
|
|
39
|
+
export declare const EAR_AFFINE_ADD = 4;
|
|
40
|
+
/** Affine divide. */
|
|
41
|
+
export declare const EAR_AFFINE_DIVIDE = 4;
|
|
42
|
+
/** Windows the SDK listens to by default (Ear.swift:113). */
|
|
43
|
+
export declare const EAR_DEFAULT_WINDOWS = 3;
|
|
44
|
+
/** The periodic Hann window (Windowing.swift:21-27). */
|
|
45
|
+
export declare const hannPeriodic: (count: number) => Float32Array;
|
|
46
|
+
/** The 400-sample periodic Hann window. */
|
|
47
|
+
export declare const EAR_HANN: Float32Array<ArrayBufferLike>;
|
|
48
|
+
/**
|
|
49
|
+
* Reflect-pad by pad samples on each side, without repeating the edge sample, matching
|
|
50
|
+
* torch.stft(center=True) (Windowing.swift:34-41).
|
|
51
|
+
*/
|
|
52
|
+
export declare const reflectPad: (samples: Float32Array, pad: number) => Float32Array;
|
|
53
|
+
/** A parsed mel filterbank: mels x bins, mel-major. */
|
|
54
|
+
export interface MelFilterbank {
|
|
55
|
+
readonly mels: number;
|
|
56
|
+
readonly bins: number;
|
|
57
|
+
readonly values: Float32Array;
|
|
58
|
+
}
|
|
59
|
+
/**
|
|
60
|
+
* Parse the filterbank asset: uint32 LE mels, uint32 LE bins, then mels*bins float32 LE,
|
|
61
|
+
* mel-major (Frontend.swift:223-242).
|
|
62
|
+
*
|
|
63
|
+
* @param bytes - the whole asset
|
|
64
|
+
* @returns the parsed bank
|
|
65
|
+
* @throws Error when the document is truncated or its header disagrees with its size
|
|
66
|
+
*/
|
|
67
|
+
export declare const parseMelFilters: (bytes: Uint8Array) => MelFilterbank;
|
|
68
|
+
/** A dense power spectrogram: frames rows of bins, frame-major. */
|
|
69
|
+
export interface PowerSpectrum {
|
|
70
|
+
readonly frames: number;
|
|
71
|
+
readonly bins: number;
|
|
72
|
+
readonly power: Float64Array;
|
|
73
|
+
}
|
|
74
|
+
/**
|
|
75
|
+
* Squared-magnitude STFT of a signal: reflect pad nFFT/2, window each frame, exact real DFT
|
|
76
|
+
* (STFT.swift:109-158). Frames = 1 + (padded - nFFT) / hop.
|
|
77
|
+
*/
|
|
78
|
+
export declare const stftPower: (samples: Float32Array) => PowerSpectrum;
|
|
79
|
+
/**
|
|
80
|
+
* The model's log-mel features: [mels * frames], mel-major (Frontend.swift:65-106).
|
|
81
|
+
*
|
|
82
|
+
* Audio is truncated or zero-padded to exactly one 30 s window first; only the first
|
|
83
|
+
* 'frames' STFT frames are used (the transform computes 3001 for 480000 samples).
|
|
84
|
+
*
|
|
85
|
+
* @param samples - any-length mono samples at 16 kHz
|
|
86
|
+
* @param filters - the parsed mel bank
|
|
87
|
+
* @returns float32 features, ready to reshape to [1, mels, frames]
|
|
88
|
+
*/
|
|
89
|
+
export declare const logMelFeatures: (samples: Float32Array, filters: MelFilterbank) => Float32Array;
|
|
90
|
+
/** One candidate window and its syllable-modulation score. */
|
|
91
|
+
export interface ModulationScore {
|
|
92
|
+
readonly offset: number;
|
|
93
|
+
readonly score: number;
|
|
94
|
+
}
|
|
95
|
+
/**
|
|
96
|
+
* Per-window syllable-modulation scores (Frontend.swift:163-221).
|
|
97
|
+
*
|
|
98
|
+
* RMS envelope at 'hop', then the difference of one-pole 8 Hz and 2 Hz low-passes as a
|
|
99
|
+
* band-pass; per-window band/total energy ratios read out of running sums. Windows carrying
|
|
100
|
+
* less than 2% of the loudest candidate's loudness score 0 rather than rank.
|
|
101
|
+
*/
|
|
102
|
+
export declare const modulationScores: (samples: Float32Array, options: {
|
|
103
|
+
readonly hop: number;
|
|
104
|
+
readonly window: number;
|
|
105
|
+
readonly step: number;
|
|
106
|
+
}) => readonly ModulationScore[];
|
|
107
|
+
/**
|
|
108
|
+
* Start offsets of the most speech-like 30 s windows, ascending (Frontend.swift:137-150).
|
|
109
|
+
*
|
|
110
|
+
* Candidates are every window/3 samples; short audio (<= one window) is listened to from 0.
|
|
111
|
+
*/
|
|
112
|
+
export declare const windowOffsets: (samples: Float32Array, count: number) => readonly number[];
|
|
113
|
+
//# sourceMappingURL=ear-frontend.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"ear-frontend.d.ts","sourceRoot":"","sources":["../src/ear-frontend.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;GAkBG;AAEH,+BAA+B;AAC/B,eAAO,MAAM,eAAe,QAAS,CAAA;AACrC,8BAA8B;AAC9B,eAAO,MAAM,SAAS,MAAM,CAAA;AAC5B,mCAAmC;AACnC,eAAO,MAAM,OAAO,MAAM,CAAA;AAC1B,iBAAiB;AACjB,eAAO,MAAM,QAAQ,KAAK,CAAA;AAC1B,iCAAiC;AACjC,eAAO,MAAM,UAAU,OAAO,CAAA;AAC9B,mCAAmC;AACnC,eAAO,MAAM,QAAQ,QAAoB,CAAA;AACzC,2CAA2C;AAC3C,eAAO,MAAM,kBAAkB,QAAuB,CAAA;AACtD,0DAA0D;AAC1D,eAAO,MAAM,aAAa,QAAQ,CAAA;AAClC,0EAA0E;AAC1E,eAAO,MAAM,iBAAiB,IAAI,CAAA;AAClC,kBAAkB;AAClB,eAAO,MAAM,cAAc,IAAI,CAAA;AAC/B,qBAAqB;AACrB,eAAO,MAAM,iBAAiB,IAAI,CAAA;AAClC,6DAA6D;AAC7D,eAAO,MAAM,mBAAmB,IAAI,CAAA;AAEpC,wDAAwD;AACxD,eAAO,MAAM,YAAY,GAAI,OAAO,MAAM,KAAG,YAU5C,CAAA;AAED,2CAA2C;AAC3C,eAAO,MAAM,QAAQ,+BAA0B,CAAA;AAE/C;;;GAGG;AACH,eAAO,MAAM,UAAU,GAAI,SAAS,YAAY,EAAE,KAAK,MAAM,KAAG,YAW/D,CAAA;AAED,uDAAuD;AACvD,MAAM,WAAW,aAAa;IAC5B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;IACrB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;IACrB,QAAQ,CAAC,MAAM,EAAE,YAAY,CAAA;CAC9B;AAED;;;;;;;GAOG;AACH,eAAO,MAAM,eAAe,GAAI,OAAO,UAAU,KAAG,aAenD,CAAA;AAcD,mEAAmE;AACnE,MAAM,WAAW,aAAa;IAC5B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;IACvB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;IACrB,QAAQ,CAAC,KAAK,EAAE,YAAY,CAAA;CAC7B;AAED;;;GAGG;AACH,eAAO,MAAM,SAAS,GAAI,SAAS,YAAY,KAAG,aAyBjD,CAAA;AAED;;;;;;;;;GASG;AACH,eAAO,MAAM,cAAc,GAAI,SAAS,YAAY,EAAE,SAAS,aAAa,KAAG,YA6B9E,CAAA;AAED,8DAA8D;AAC9D,MAAM,WAAW,eAAe;IAC9B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;IACvB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;CACvB;AAED;;;;;;GAMG;AACH,eAAO,MAAM,gBAAgB,GAC3B,SAAS,YAAY,EACrB,SAAS;IAAE,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;CAAE,KAChF,SAAS,eAAe,EA+C1B,CAAA;AAED;;;;GAIG;AACH,eAAO,MAAM,aAAa,GAAI,SAAS,YAAY,EAAE,OAAO,MAAM,KAAG,SAAS,MAAM,EAUnF,CAAA"}
|
|
@@ -0,0 +1,258 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The ear log-mel front end and window selection, ported from the primary source.
|
|
3
|
+
*
|
|
4
|
+
* PROVENANCE (Desert-Ant-Labs/desert-ant-core @ baf4e6e3b4655c8e54aa8d5a25215b5f35f899a7):
|
|
5
|
+
* - Sources/Ear/Frontend.swift:25-40 geometry (and windowSamples = frames * hop)
|
|
6
|
+
* - Sources/Ear/Frontend.swift:48-63 Hann periodic + center STFT + filterbank size checks
|
|
7
|
+
* - Sources/Ear/Frontend.swift:65-106 features: truncate/pad to one window, STFT, mel energies,
|
|
8
|
+
* log10(clamp), floor relative to the peak, affine
|
|
9
|
+
* - Sources/Ear/Frontend.swift:137-221 windowOffsets + modulationScores
|
|
10
|
+
* - Sources/Ear/Frontend.swift:223-242 mel filterbank parse (uint32 LE mels/bins then f32 LE)
|
|
11
|
+
* - Sources/AudioDSP/STFT.swift:68-106 (bins = nFFT/2 + 1), :109-158 (REFLECT pad, frames)
|
|
12
|
+
* - Sources/AudioDSP/Windowing.swift:21-27 Hann periodic, :30-41 reflect padding
|
|
13
|
+
* - .assets/ear/ear_meta.json: 16 kHz, n_fft 400, hop 160, mels 80, frames 3000, Hann periodic,
|
|
14
|
+
* log10, clamp 1e-10, floor 8 decades, affine (x + 4) / 4.
|
|
15
|
+
*
|
|
16
|
+
* nFFT is not a power of two, so the source's portable path is an exact real-DFT matmul
|
|
17
|
+
* (STFT.swift:142-158). This module does the same transform, in double precision, then stores
|
|
18
|
+
* the features as float32 -- the tensor the model consumes.
|
|
19
|
+
*/
|
|
20
|
+
/** The model's sample rate. */
|
|
21
|
+
export const EAR_SAMPLE_RATE = 16_000;
|
|
22
|
+
/** Analysis window length. */
|
|
23
|
+
export const EAR_N_FFT = 400;
|
|
24
|
+
/** Hop between analysis frames. */
|
|
25
|
+
export const EAR_HOP = 160;
|
|
26
|
+
/** Mel bands. */
|
|
27
|
+
export const EAR_MELS = 80;
|
|
28
|
+
/** Frames the model consumes. */
|
|
29
|
+
export const EAR_FRAMES = 3000;
|
|
30
|
+
/** Spectrum bins: nFFT / 2 + 1. */
|
|
31
|
+
export const EAR_BINS = EAR_N_FFT / 2 + 1;
|
|
32
|
+
/** Samples in one 30 s analysis window. */
|
|
33
|
+
export const EAR_WINDOW_SAMPLES = EAR_FRAMES * EAR_HOP;
|
|
34
|
+
/** Floor applied to mel energies before the logarithm. */
|
|
35
|
+
export const EAR_CLAMP_MIN = 1e-10;
|
|
36
|
+
/** How far below the loudest bin the features are floored, in decades. */
|
|
37
|
+
export const EAR_FLOOR_DECADES = 8;
|
|
38
|
+
/** Affine add. */
|
|
39
|
+
export const EAR_AFFINE_ADD = 4;
|
|
40
|
+
/** Affine divide. */
|
|
41
|
+
export const EAR_AFFINE_DIVIDE = 4;
|
|
42
|
+
/** Windows the SDK listens to by default (Ear.swift:113). */
|
|
43
|
+
export const EAR_DEFAULT_WINDOWS = 3;
|
|
44
|
+
/** The periodic Hann window (Windowing.swift:21-27). */
|
|
45
|
+
export const hannPeriodic = (count) => {
|
|
46
|
+
const window = new Float32Array(Math.max(count, 0));
|
|
47
|
+
if (count <= 1) {
|
|
48
|
+
window.fill(1);
|
|
49
|
+
return window;
|
|
50
|
+
}
|
|
51
|
+
for (let index = 0; index < count; index += 1) {
|
|
52
|
+
window[index] = 0.5 - 0.5 * Math.cos((2 * Math.PI * index) / count);
|
|
53
|
+
}
|
|
54
|
+
return window;
|
|
55
|
+
};
|
|
56
|
+
/** The 400-sample periodic Hann window. */
|
|
57
|
+
export const EAR_HANN = hannPeriodic(EAR_N_FFT);
|
|
58
|
+
/**
|
|
59
|
+
* Reflect-pad by pad samples on each side, without repeating the edge sample, matching
|
|
60
|
+
* torch.stft(center=True) (Windowing.swift:34-41).
|
|
61
|
+
*/
|
|
62
|
+
export const reflectPad = (samples, pad) => {
|
|
63
|
+
if (pad <= 0 || samples.length === 0)
|
|
64
|
+
return Float32Array.from(samples);
|
|
65
|
+
const out = new Float32Array(samples.length + 2 * pad);
|
|
66
|
+
for (let index = 0; index < pad; index += 1) {
|
|
67
|
+
out[index] = samples[Math.min(pad - index, samples.length - 1)] ?? 0;
|
|
68
|
+
}
|
|
69
|
+
out.set(samples, pad);
|
|
70
|
+
for (let index = 0; index < pad; index += 1) {
|
|
71
|
+
out[pad + samples.length + index] = samples[Math.max(samples.length - 2 - index, 0)] ?? 0;
|
|
72
|
+
}
|
|
73
|
+
return out;
|
|
74
|
+
};
|
|
75
|
+
/**
|
|
76
|
+
* Parse the filterbank asset: uint32 LE mels, uint32 LE bins, then mels*bins float32 LE,
|
|
77
|
+
* mel-major (Frontend.swift:223-242).
|
|
78
|
+
*
|
|
79
|
+
* @param bytes - the whole asset
|
|
80
|
+
* @returns the parsed bank
|
|
81
|
+
* @throws Error when the document is truncated or its header disagrees with its size
|
|
82
|
+
*/
|
|
83
|
+
export const parseMelFilters = (bytes) => {
|
|
84
|
+
if (bytes.length < 8)
|
|
85
|
+
throw new Error('the mel filterbank is truncated');
|
|
86
|
+
const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
|
|
87
|
+
const mels = view.getUint32(0, true);
|
|
88
|
+
const bins = view.getUint32(4, true);
|
|
89
|
+
if (mels === 0 || bins === 0)
|
|
90
|
+
throw new Error('the mel filterbank declares zero mels or bins');
|
|
91
|
+
const count = mels * bins;
|
|
92
|
+
if (bytes.length !== 8 + count * 4) {
|
|
93
|
+
throw new Error('the mel filterbank is truncated: ' + String(bytes.length) + ' bytes for ' + String(count) + ' values');
|
|
94
|
+
}
|
|
95
|
+
const values = new Float32Array(count);
|
|
96
|
+
for (let index = 0; index < count; index += 1) {
|
|
97
|
+
values[index] = view.getFloat32(8 + index * 4, true);
|
|
98
|
+
}
|
|
99
|
+
return { mels, bins, values };
|
|
100
|
+
};
|
|
101
|
+
// The portable path's real-DFT bases, built once (STFT.swift:79-89). Float64 here; the
|
|
102
|
+
// source uses Float32 tables, so feature values agree to float32 rounding, not bit-for-bit.
|
|
103
|
+
const FWD_COS = new Float64Array(EAR_N_FFT * EAR_BINS);
|
|
104
|
+
const FWD_SIN = new Float64Array(EAR_N_FFT * EAR_BINS);
|
|
105
|
+
for (let t = 0; t < EAR_N_FFT; t += 1) {
|
|
106
|
+
for (let k = 0; k < EAR_BINS; k += 1) {
|
|
107
|
+
const angle = (2 * Math.PI * k * t) / EAR_N_FFT;
|
|
108
|
+
FWD_COS[t * EAR_BINS + k] = Math.cos(angle);
|
|
109
|
+
FWD_SIN[t * EAR_BINS + k] = -Math.sin(angle);
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
/**
|
|
113
|
+
* Squared-magnitude STFT of a signal: reflect pad nFFT/2, window each frame, exact real DFT
|
|
114
|
+
* (STFT.swift:109-158). Frames = 1 + (padded - nFFT) / hop.
|
|
115
|
+
*/
|
|
116
|
+
export const stftPower = (samples) => {
|
|
117
|
+
const padded = reflectPad(samples, EAR_N_FFT / 2);
|
|
118
|
+
if (padded.length < EAR_N_FFT)
|
|
119
|
+
return { frames: 0, bins: EAR_BINS, power: new Float64Array(0) };
|
|
120
|
+
const frames = 1 + Math.trunc((padded.length - EAR_N_FFT) / EAR_HOP);
|
|
121
|
+
const power = new Float64Array(frames * EAR_BINS);
|
|
122
|
+
const windowed = new Float64Array(EAR_N_FFT);
|
|
123
|
+
for (let frame = 0; frame < frames; frame += 1) {
|
|
124
|
+
const offset = frame * EAR_HOP;
|
|
125
|
+
for (let t = 0; t < EAR_N_FFT; t += 1) {
|
|
126
|
+
windowed[t] = (padded[offset + t] ?? 0) * (EAR_HANN[t] ?? 0);
|
|
127
|
+
}
|
|
128
|
+
const row = frame * EAR_BINS;
|
|
129
|
+
for (let bin = 0; bin < EAR_BINS; bin += 1) {
|
|
130
|
+
let re = 0;
|
|
131
|
+
let im = 0;
|
|
132
|
+
for (let t = 0; t < EAR_N_FFT; t += 1) {
|
|
133
|
+
const value = windowed[t] ?? 0;
|
|
134
|
+
const at = t * EAR_BINS + bin;
|
|
135
|
+
re += value * (FWD_COS[at] ?? 0);
|
|
136
|
+
im += value * (FWD_SIN[at] ?? 0);
|
|
137
|
+
}
|
|
138
|
+
power[row + bin] = re * re + im * im;
|
|
139
|
+
}
|
|
140
|
+
}
|
|
141
|
+
return { frames, bins: EAR_BINS, power };
|
|
142
|
+
};
|
|
143
|
+
/**
|
|
144
|
+
* The model's log-mel features: [mels * frames], mel-major (Frontend.swift:65-106).
|
|
145
|
+
*
|
|
146
|
+
* Audio is truncated or zero-padded to exactly one 30 s window first; only the first
|
|
147
|
+
* 'frames' STFT frames are used (the transform computes 3001 for 480000 samples).
|
|
148
|
+
*
|
|
149
|
+
* @param samples - any-length mono samples at 16 kHz
|
|
150
|
+
* @param filters - the parsed mel bank
|
|
151
|
+
* @returns float32 features, ready to reshape to [1, mels, frames]
|
|
152
|
+
*/
|
|
153
|
+
export const logMelFeatures = (samples, filters) => {
|
|
154
|
+
const signal = new Float32Array(EAR_WINDOW_SAMPLES);
|
|
155
|
+
const take = Math.min(samples.length, EAR_WINDOW_SAMPLES);
|
|
156
|
+
signal.set(samples.subarray(0, take));
|
|
157
|
+
const spectrum = stftPower(signal);
|
|
158
|
+
const frames = Math.min(EAR_FRAMES, spectrum.frames);
|
|
159
|
+
const log = new Float64Array(filters.mels * EAR_FRAMES);
|
|
160
|
+
let peak = -Number.MAX_VALUE;
|
|
161
|
+
for (let mel = 0; mel < filters.mels; mel += 1) {
|
|
162
|
+
const filterBase = mel * filters.bins;
|
|
163
|
+
for (let frame = 0; frame < frames; frame += 1) {
|
|
164
|
+
const row = frame * EAR_BINS;
|
|
165
|
+
let energy = 0;
|
|
166
|
+
for (let bin = 0; bin < filters.bins; bin += 1) {
|
|
167
|
+
const weight = filters.values[filterBase + bin] ?? 0;
|
|
168
|
+
if (weight === 0)
|
|
169
|
+
continue;
|
|
170
|
+
energy += weight * (spectrum.power[row + bin] ?? 0);
|
|
171
|
+
}
|
|
172
|
+
const value = Math.log10(Math.max(energy, EAR_CLAMP_MIN));
|
|
173
|
+
log[mel * EAR_FRAMES + frame] = value;
|
|
174
|
+
if (value > peak)
|
|
175
|
+
peak = value;
|
|
176
|
+
}
|
|
177
|
+
}
|
|
178
|
+
const floor = peak - EAR_FLOOR_DECADES;
|
|
179
|
+
const features = new Float32Array(log.length);
|
|
180
|
+
for (let index = 0; index < log.length; index += 1) {
|
|
181
|
+
features[index] = (Math.max(log[index] ?? 0, floor) + EAR_AFFINE_ADD) / EAR_AFFINE_DIVIDE;
|
|
182
|
+
}
|
|
183
|
+
return features;
|
|
184
|
+
};
|
|
185
|
+
/**
|
|
186
|
+
* Per-window syllable-modulation scores (Frontend.swift:163-221).
|
|
187
|
+
*
|
|
188
|
+
* RMS envelope at 'hop', then the difference of one-pole 8 Hz and 2 Hz low-passes as a
|
|
189
|
+
* band-pass; per-window band/total energy ratios read out of running sums. Windows carrying
|
|
190
|
+
* less than 2% of the loudest candidate's loudness score 0 rather than rank.
|
|
191
|
+
*/
|
|
192
|
+
export const modulationScores = (samples, options) => {
|
|
193
|
+
const frames = Math.trunc(samples.length / options.hop);
|
|
194
|
+
if (frames <= 8)
|
|
195
|
+
return [{ offset: 0, score: 0 }];
|
|
196
|
+
const envelope = new Float64Array(frames);
|
|
197
|
+
const quarter = options.hop / 4;
|
|
198
|
+
for (let frame = 0; frame < frames; frame += 1) {
|
|
199
|
+
let sum = 0;
|
|
200
|
+
const base = frame * options.hop;
|
|
201
|
+
for (let index = base; index < base + options.hop; index += 4) {
|
|
202
|
+
const value = samples[index] ?? 0;
|
|
203
|
+
sum += value * value;
|
|
204
|
+
}
|
|
205
|
+
envelope[frame] = Math.sqrt(sum / quarter);
|
|
206
|
+
}
|
|
207
|
+
const rate = EAR_SAMPLE_RATE / options.hop;
|
|
208
|
+
const coefficient = (hz) => 1 - Math.exp((-2 * Math.PI * hz) / rate);
|
|
209
|
+
const fast = coefficient(8);
|
|
210
|
+
const slow = coefficient(2);
|
|
211
|
+
let lowFast = 0;
|
|
212
|
+
let lowSlow = 0;
|
|
213
|
+
const bandEnergy = new Float64Array(frames + 1);
|
|
214
|
+
const totalEnergy = new Float64Array(frames + 1);
|
|
215
|
+
const loudness = new Float64Array(frames + 1);
|
|
216
|
+
for (let frame = 0; frame < frames; frame += 1) {
|
|
217
|
+
const value = envelope[frame] ?? 0;
|
|
218
|
+
lowFast += fast * (value - lowFast);
|
|
219
|
+
lowSlow += slow * (value - lowSlow);
|
|
220
|
+
const syllabic = lowFast - lowSlow;
|
|
221
|
+
bandEnergy[frame + 1] = (bandEnergy[frame] ?? 0) + syllabic * syllabic;
|
|
222
|
+
totalEnergy[frame + 1] = (totalEnergy[frame] ?? 0) + (value - lowSlow) * (value - lowSlow);
|
|
223
|
+
loudness[frame + 1] = (loudness[frame] ?? 0) + value * value;
|
|
224
|
+
}
|
|
225
|
+
const windowFrames = Math.max(Math.trunc(options.window / options.hop), 1);
|
|
226
|
+
const raw = [];
|
|
227
|
+
for (let offset = 0; offset <= Math.max(samples.length - options.window, 0); offset += options.step) {
|
|
228
|
+
const from = Math.min(Math.trunc(offset / options.hop), frames);
|
|
229
|
+
const to = Math.min(from + windowFrames, frames);
|
|
230
|
+
if (to <= from)
|
|
231
|
+
continue;
|
|
232
|
+
const band = (bandEnergy[to] ?? 0) - (bandEnergy[from] ?? 0);
|
|
233
|
+
const total = (totalEnergy[to] ?? 0) - (totalEnergy[from] ?? 0);
|
|
234
|
+
raw.push({ offset, ratio: total > 0 ? band / total : 0, loud: (loudness[to] ?? 0) - (loudness[from] ?? 0) });
|
|
235
|
+
}
|
|
236
|
+
if (raw.length === 0)
|
|
237
|
+
return [{ offset: 0, score: 0 }];
|
|
238
|
+
const peak = raw.reduce((maximum, entry) => Math.max(maximum, entry.loud), 0);
|
|
239
|
+
return raw.map((entry) => ({ offset: entry.offset, score: entry.loud >= 0.02 * peak ? entry.ratio : 0 }));
|
|
240
|
+
};
|
|
241
|
+
/**
|
|
242
|
+
* Start offsets of the most speech-like 30 s windows, ascending (Frontend.swift:137-150).
|
|
243
|
+
*
|
|
244
|
+
* Candidates are every window/3 samples; short audio (<= one window) is listened to from 0.
|
|
245
|
+
*/
|
|
246
|
+
export const windowOffsets = (samples, count) => {
|
|
247
|
+
const want = EAR_WINDOW_SAMPLES;
|
|
248
|
+
if (samples.length <= want)
|
|
249
|
+
return [0];
|
|
250
|
+
const step = Math.max(Math.trunc(want / 3), 1);
|
|
251
|
+
const scored = modulationScores(samples, { hop: EAR_HOP, window: want, step });
|
|
252
|
+
const selected = [...scored]
|
|
253
|
+
.sort((a, b) => b.score - a.score)
|
|
254
|
+
.slice(0, Math.max(count, 1))
|
|
255
|
+
.map((entry) => entry.offset);
|
|
256
|
+
return selected.sort((a, b) => a - b);
|
|
257
|
+
};
|
|
258
|
+
//# sourceMappingURL=ear-frontend.js.map
|
package/lib/ear.d.ts
ADDED
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Processor ear/mel-v1: spoken-language identification over the ear LiteRT artifact.
|
|
3
|
+
*
|
|
4
|
+
* PRIMARY SOURCE (Desert-Ant-Labs/desert-ant-core @ baf4e6e3b4655c8e54aa8d5a25215b5f35f899a7):
|
|
5
|
+
* - Sources/Ear/Model.swift:19 languages from languages.json; :72-102 identify (linear resample,
|
|
6
|
+
* window offsets, per-window softmax and probability averaging, alias folding, top-5);
|
|
7
|
+
* :104-114 tensor name/shape [1, 80, 3000]; :119-123 linear resample.
|
|
8
|
+
* - Sources/Ear/Ear.swift:29-34 isReliable; :54 reliableMargin = 0.25; :113 defaultWindows = 3;
|
|
9
|
+
* :156-165 non-empty audio and windows > 0.
|
|
10
|
+
* - Sources/Ear/Languages.swift:4-8 aliases (tl->fil, nb->no, yue->zh); :20 confusables
|
|
11
|
+
* (no, sv, da); :22-24 canonicalLanguage.
|
|
12
|
+
* - Sources/Ear/Frontend.swift + AudioDSP: the log-mel front end and window selection, ported
|
|
13
|
+
* in ear-frontend.ts.
|
|
14
|
+
* - .assets/ear/ear_meta.json: 16 kHz, 30 s windows, recommended 3, n_fft 400, hop 160,
|
|
15
|
+
* mels 80, frames 3000, clamp 1e-10, floor 8 decades, affine (x + 4) / 4, input 'mel',
|
|
16
|
+
* output 'output_0', 103 logits in languages.json order.
|
|
17
|
+
*
|
|
18
|
+
* INPUT closed: { audio_path: string 1..4096, windows?: integer 1..16 }.
|
|
19
|
+
* WAV only (RIFF/WAVE), PCM 16-bit or IEEE float32, mono or stereo, any declared rate (linear
|
|
20
|
+
* resample to 16 kHz). Short audio is zero-padded to one window; long audio is searched for the
|
|
21
|
+
* most speech-like windows (window/3 candidate stride) and only the selected ones are scored.
|
|
22
|
+
*
|
|
23
|
+
* OUTPUT exactly: { language, confidence, isReliable,
|
|
24
|
+
* candidates: [{language, probability}] (top 5, folded, descending),
|
|
25
|
+
* windows } (integer count actually listened to).
|
|
26
|
+
*
|
|
27
|
+
* HONEST LIMITS: 103 model labels = 102 canonical languages after yue -> zh folding (this label
|
|
28
|
+
* set has no tl or nb, so only yue -> zh fires; the full alias map is kept for future label
|
|
29
|
+
* sets). Scores are decision probabilities averaged over a few 30 s windows, not calibrated
|
|
30
|
+
* confidence, and this is not a speech-recognition or nationality claim.
|
|
31
|
+
*/
|
|
32
|
+
import { type RecipeProcessor } from './processor.ts';
|
|
33
|
+
/** The processor id this module owns. */
|
|
34
|
+
export declare const EAR_PROCESSOR = "ear/mel-v1";
|
|
35
|
+
/** Codes the detector emits that differ from the codes callers expect (Languages.swift:4-8). */
|
|
36
|
+
export declare const EAR_LANGUAGE_ALIASES: {
|
|
37
|
+
readonly tl: "fil";
|
|
38
|
+
readonly nb: "no";
|
|
39
|
+
readonly yue: "zh";
|
|
40
|
+
};
|
|
41
|
+
/** Languages the detector confuses confidently rather than uncertainly (Languages.swift:20). */
|
|
42
|
+
export declare const EAR_CONFUSABLE_LANGUAGES: readonly ["no", "sv", "da"];
|
|
43
|
+
/** How far ahead the top candidate must be before routing on it (Ear.swift:54). */
|
|
44
|
+
export declare const EAR_RELIABLE_MARGIN = 0.25;
|
|
45
|
+
/** The label count at the pinned revision (103 logits, languages.json order). */
|
|
46
|
+
export declare const EAR_LANGUAGE_COUNT = 103;
|
|
47
|
+
/** The largest windows value the tool accepts. */
|
|
48
|
+
export declare const EAR_MAX_WINDOWS = 16;
|
|
49
|
+
/** The ear processor. */
|
|
50
|
+
export declare const earMelProcessor: RecipeProcessor;
|
|
51
|
+
//# sourceMappingURL=ear.d.ts.map
|
package/lib/ear.d.ts.map
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"ear.d.ts","sourceRoot":"","sources":["../src/ear.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8BG;AAyBH,OAAO,EAAyC,KAAK,eAAe,EAAE,MAAM,gBAAgB,CAAA;AAI5F,yCAAyC;AACzC,eAAO,MAAM,aAAa,eAAe,CAAA;AAEzC,gGAAgG;AAChG,eAAO,MAAM,oBAAoB;;;;CAA8C,CAAA;AAE/E,gGAAgG;AAChG,eAAO,MAAM,wBAAwB,6BAA8B,CAAA;AAEnE,mFAAmF;AACnF,eAAO,MAAM,mBAAmB,OAAO,CAAA;AAEvC,iFAAiF;AACjF,eAAO,MAAM,kBAAkB,MAAM,CAAA;AAErC,kDAAkD;AAClD,eAAO,MAAM,eAAe,KAAK,CAAA;AAsNjC,yBAAyB;AACzB,eAAO,MAAM,eAAe,EAAE,eAsC7B,CAAA"}
|