@try-works/tool-mode-runtime-onnx 0.0.0-stage → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +64 -2
- package/lib/clear-dsp.d.ts +235 -0
- package/lib/clear-dsp.d.ts.map +1 -0
- package/lib/clear-dsp.js +487 -0
- package/lib/clear-processor.d.ts +30 -0
- package/lib/clear-processor.d.ts.map +1 -0
- package/lib/clear-processor.js +172 -0
- package/lib/errors.d.ts +14 -0
- package/lib/errors.d.ts.map +1 -0
- package/lib/errors.js +29 -0
- package/lib/fft.d.ts +86 -0
- package/lib/fft.d.ts.map +1 -0
- package/lib/fft.js +231 -0
- package/lib/frames.d.ts +99 -0
- package/lib/frames.d.ts.map +1 -0
- package/lib/frames.js +284 -0
- package/lib/index.d.ts +27 -0
- package/lib/index.d.ts.map +1 -0
- package/lib/index.js +27 -0
- package/lib/processor.d.ts +74 -0
- package/lib/processor.d.ts.map +1 -0
- package/lib/processor.js +82 -0
- package/lib/provider-factory.d.ts +47 -0
- package/lib/provider-factory.d.ts.map +1 -0
- package/lib/provider-factory.js +276 -0
- package/lib/runtime.d.ts +122 -0
- package/lib/runtime.d.ts.map +1 -0
- package/lib/runtime.js +175 -0
- package/lib/uhm-processor.d.ts +23 -0
- package/lib/uhm-processor.d.ts.map +1 -0
- package/lib/uhm-processor.js +126 -0
- package/lib/wav.d.ts +97 -0
- package/lib/wav.d.ts.map +1 -0
- package/lib/wav.js +196 -0
- package/package.json +48 -4
|
@@ -0,0 +1,126 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Processor `uhm/waveform-v1`: filler-word detection (Phase 3, the first model).
|
|
3
|
+
*
|
|
4
|
+
* The pipeline is `frames.ts` + `wav.ts`; this module is only the processor seam: the
|
|
5
|
+
* labels sidecar, the one-input/one-output graph binding, and the window loop.
|
|
6
|
+
*
|
|
7
|
+
* @module uhm-processor
|
|
8
|
+
*/
|
|
9
|
+
import { readFile } from 'node:fs/promises';
|
|
10
|
+
import path from 'node:path';
|
|
11
|
+
import { Effect } from 'effect';
|
|
12
|
+
import { InferenceFailed, RuntimeUnavailable, describeCause } from "./errors.js";
|
|
13
|
+
import { UHM_LABELS_FILE, parseUhmLabels, uhmResultFromWindows, } from "./frames.js";
|
|
14
|
+
import { UHM_SAMPLE_RATE, UHM_WINDOW_SAMPLES, windowsOf } from "./wav.js";
|
|
15
|
+
/** The recipe processor id this module implements. */
|
|
16
|
+
export const UHM_WAVEFORM_PROCESSOR = 'uhm/waveform-v1';
|
|
17
|
+
const PROVIDER = 'onnx';
|
|
18
|
+
/** Read and validate the pinned labels document beside the model artifact. */
|
|
19
|
+
async function readLabels(directory) {
|
|
20
|
+
const labelsPath = path.join(directory, UHM_LABELS_FILE);
|
|
21
|
+
let text;
|
|
22
|
+
try {
|
|
23
|
+
text = await readFile(labelsPath, 'utf8');
|
|
24
|
+
}
|
|
25
|
+
catch (cause) {
|
|
26
|
+
throw new RuntimeUnavailable({
|
|
27
|
+
provider: PROVIDER,
|
|
28
|
+
reason: 'the uhm labels file "' + UHM_LABELS_FILE + '" cannot be read from the prepared asset directory: ' + describeCause(cause),
|
|
29
|
+
});
|
|
30
|
+
}
|
|
31
|
+
let raw;
|
|
32
|
+
try {
|
|
33
|
+
raw = JSON.parse(text);
|
|
34
|
+
}
|
|
35
|
+
catch (cause) {
|
|
36
|
+
throw new RuntimeUnavailable({
|
|
37
|
+
provider: PROVIDER,
|
|
38
|
+
reason: 'the uhm labels file "' + UHM_LABELS_FILE + '" is not valid JSON: ' + describeCause(cause),
|
|
39
|
+
});
|
|
40
|
+
}
|
|
41
|
+
const parsed = parseUhmLabels(raw);
|
|
42
|
+
if (!parsed.ok) {
|
|
43
|
+
throw new RuntimeUnavailable({
|
|
44
|
+
provider: PROVIDER,
|
|
45
|
+
reason: 'the uhm labels file "' + UHM_LABELS_FILE + '" cannot be used: ' + parsed.reason,
|
|
46
|
+
});
|
|
47
|
+
}
|
|
48
|
+
return parsed.labels;
|
|
49
|
+
}
|
|
50
|
+
/** Map the uhm result onto the tool output contract. */
|
|
51
|
+
function payloadOf(result) {
|
|
52
|
+
const fillers = result.fillers.map((span) => ({
|
|
53
|
+
startMs: span.startMs,
|
|
54
|
+
endMs: span.endMs,
|
|
55
|
+
type: span.type,
|
|
56
|
+
confidence: span.confidence,
|
|
57
|
+
}));
|
|
58
|
+
const coverage = {
|
|
59
|
+
complete: result.coverage.complete,
|
|
60
|
+
processedSeconds: result.coverage.processedSeconds,
|
|
61
|
+
totalSeconds: result.coverage.totalSeconds,
|
|
62
|
+
};
|
|
63
|
+
if (result.coverage.reason !== undefined) {
|
|
64
|
+
return { fillers, coverage: { ...coverage, reason: result.coverage.reason } };
|
|
65
|
+
}
|
|
66
|
+
return { fillers, coverage };
|
|
67
|
+
}
|
|
68
|
+
/**
|
|
69
|
+
* The uhm processor definition.
|
|
70
|
+
*
|
|
71
|
+
* @returns the definition the provider factory dispatches to
|
|
72
|
+
*/
|
|
73
|
+
export function uhmProcessor() {
|
|
74
|
+
return {
|
|
75
|
+
id: UHM_WAVEFORM_PROCESSOR,
|
|
76
|
+
sampleRate: UHM_SAMPLE_RATE,
|
|
77
|
+
loadSidecar: readLabels,
|
|
78
|
+
bindGraph: (names) => {
|
|
79
|
+
const inputName = names.inputs[0];
|
|
80
|
+
const outputName = names.outputs[0];
|
|
81
|
+
if (names.inputs.length !== 1 || inputName === undefined || outputName === undefined) {
|
|
82
|
+
return {
|
|
83
|
+
ok: false,
|
|
84
|
+
reason: 'the uhm graph must expose exactly one input and at least one output; it exposes '
|
|
85
|
+
+ String(names.inputs.length)
|
|
86
|
+
+ ' input(s) ('
|
|
87
|
+
+ (names.inputs.join(', ') || 'none')
|
|
88
|
+
+ ') and '
|
|
89
|
+
+ String(names.outputs.length)
|
|
90
|
+
+ ' output(s)',
|
|
91
|
+
};
|
|
92
|
+
}
|
|
93
|
+
return { ok: true, value: { inputName, outputName } };
|
|
94
|
+
},
|
|
95
|
+
invoke: (request) => Effect.gen(function* () {
|
|
96
|
+
const windows = windowsOf(request.audio.samples);
|
|
97
|
+
const runs = [];
|
|
98
|
+
for (const window of windows) {
|
|
99
|
+
const outputs = yield* request.engine.run({
|
|
100
|
+
[request.binding.inputName]: { data: window.samples, dims: [1, UHM_WINDOW_SAMPLES] },
|
|
101
|
+
});
|
|
102
|
+
const output = outputs[request.binding.outputName];
|
|
103
|
+
if (output === undefined) {
|
|
104
|
+
return yield* Effect.fail(new InferenceFailed({
|
|
105
|
+
provider: PROVIDER,
|
|
106
|
+
phase: 'post',
|
|
107
|
+
message: 'the graph produced no output named ' + JSON.stringify(request.binding.outputName),
|
|
108
|
+
}));
|
|
109
|
+
}
|
|
110
|
+
if (!(output.data instanceof Float32Array)) {
|
|
111
|
+
return yield* Effect.fail(new InferenceFailed({
|
|
112
|
+
provider: PROVIDER,
|
|
113
|
+
phase: 'post',
|
|
114
|
+
message: 'the graph output ' + JSON.stringify(request.binding.outputName) + ' is not a float32 tensor',
|
|
115
|
+
}));
|
|
116
|
+
}
|
|
117
|
+
runs.push({ window, data: output.data, dims: output.dims });
|
|
118
|
+
}
|
|
119
|
+
return yield* Effect.try({
|
|
120
|
+
try: () => payloadOf(uhmResultFromWindows({ runs, totalSamples: request.audio.samples.length, labels: request.sidecar })),
|
|
121
|
+
catch: (cause) => new InferenceFailed({ provider: PROVIDER, phase: 'post', message: describeCause(cause) }),
|
|
122
|
+
});
|
|
123
|
+
}),
|
|
124
|
+
};
|
|
125
|
+
}
|
|
126
|
+
//# sourceMappingURL=uhm-processor.js.map
|
package/lib/wav.d.ts
ADDED
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* WAV decoding, resampling and windowing (PHASE3 processor uhm/waveform-v1, steps 1-2).
|
|
3
|
+
*
|
|
4
|
+
* Two facts drive the whole module:
|
|
5
|
+
*
|
|
6
|
+
* - **The model consumes 16 kHz mono float samples.** A caller's file may be any rate,
|
|
7
|
+
* one or two channels, PCM 16-bit or float32, so decode and resample are separate steps
|
|
8
|
+
* with separate tests rather than one opaque pipeline.
|
|
9
|
+
* - **The window geometry is the MODEL's, not the caller's.** 30 s windows with a 1 s
|
|
10
|
+
* overlap and a zero-padded final window are pinned constants (480000 samples, hop
|
|
11
|
+
* 464000). The overlap exists so a filler word spanning a window boundary is seen
|
|
12
|
+
* whole by one of the two windows; the frame timeline in `frames.ts` merges them.
|
|
13
|
+
*
|
|
14
|
+
* @module wav
|
|
15
|
+
*/
|
|
16
|
+
/** The model's sample rate. */
|
|
17
|
+
export declare const UHM_SAMPLE_RATE = 16000;
|
|
18
|
+
/** One model window, in samples (30 s at 16 kHz). */
|
|
19
|
+
export declare const UHM_WINDOW_SAMPLES = 480000;
|
|
20
|
+
/** The overlap between consecutive windows, in samples (1 s). */
|
|
21
|
+
export declare const UHM_OVERLAP_SAMPLES = 16000;
|
|
22
|
+
/** The hop between consecutive windows: window minus overlap. */
|
|
23
|
+
export declare const UHM_HOP_SAMPLES: number;
|
|
24
|
+
/** One model output frame, in samples (20 ms at 16 kHz). */
|
|
25
|
+
export declare const UHM_FRAME_SAMPLES = 320;
|
|
26
|
+
/** One model output frame, in milliseconds. */
|
|
27
|
+
export declare const UHM_FRAME_MS = 20;
|
|
28
|
+
/** A decoded WAV: mono float samples at the SOURCE rate. */
|
|
29
|
+
export interface WavAudio {
|
|
30
|
+
/** The rate the file declares. */
|
|
31
|
+
readonly sampleRate: number;
|
|
32
|
+
/** The channel count the file declares. */
|
|
33
|
+
readonly channels: number;
|
|
34
|
+
/** Which sample encoding was decoded. */
|
|
35
|
+
readonly encoding: 'pcm16' | 'float32';
|
|
36
|
+
/** Mono samples at the source rate; stereo is averaged. */
|
|
37
|
+
readonly samples: Float32Array;
|
|
38
|
+
}
|
|
39
|
+
/** What {@link decodeWav} returns. */
|
|
40
|
+
export type WavDecodeResult = {
|
|
41
|
+
readonly ok: true;
|
|
42
|
+
readonly audio: WavAudio;
|
|
43
|
+
} | {
|
|
44
|
+
readonly ok: false;
|
|
45
|
+
readonly reason: string;
|
|
46
|
+
};
|
|
47
|
+
/** One window of samples, always exactly `windowSamples` long. */
|
|
48
|
+
export interface AudioWindow {
|
|
49
|
+
/** Window index, in order. */
|
|
50
|
+
readonly index: number;
|
|
51
|
+
/** Absolute sample offset of the window start. */
|
|
52
|
+
readonly startSample: number;
|
|
53
|
+
/** The window samples, zero padded past the end of the audio. */
|
|
54
|
+
readonly samples: Float32Array;
|
|
55
|
+
}
|
|
56
|
+
/** The window geometry to apply. */
|
|
57
|
+
export interface WindowGeometry {
|
|
58
|
+
readonly windowSamples: number;
|
|
59
|
+
readonly hopSamples: number;
|
|
60
|
+
}
|
|
61
|
+
/**
|
|
62
|
+
* Decode a WAV document (RIFF/WAVE) to mono float samples.
|
|
63
|
+
*
|
|
64
|
+
* Accepted: `WAVE_FORMAT_PCM` with 16 bits, and `WAVE_FORMAT_IEEE_FLOAT` with 32 bits,
|
|
65
|
+
* one or two channels. Everything else is refused BY NAME rather than guessed at, because
|
|
66
|
+
* a mis-decoded waveform is a wrong transcript with no other symptom.
|
|
67
|
+
*
|
|
68
|
+
* @param bytes - the file bytes
|
|
69
|
+
* @returns the decoded audio, or the reason it cannot be decoded
|
|
70
|
+
*/
|
|
71
|
+
export declare function decodeWav(bytes: Uint8Array): WavDecodeResult;
|
|
72
|
+
/**
|
|
73
|
+
* Resample mono samples with linear interpolation.
|
|
74
|
+
*
|
|
75
|
+
* Linear, not windowed-sinc: PHASE3 specifies linear, and the model was qualified on that
|
|
76
|
+
* path. A file already at the target rate is copied so a caller never aliases the file
|
|
77
|
+
* buffer.
|
|
78
|
+
*
|
|
79
|
+
* @param samples - mono source samples
|
|
80
|
+
* @param fromRate - the source sample rate
|
|
81
|
+
* @param toRate - the target sample rate
|
|
82
|
+
* @returns the resampled samples
|
|
83
|
+
*/
|
|
84
|
+
export declare function resampleLinear(samples: Float32Array, fromRate: number, toRate: number): Float32Array;
|
|
85
|
+
/**
|
|
86
|
+
* Slice mono samples into zero-padded model windows.
|
|
87
|
+
*
|
|
88
|
+
* A window starts at every hop while the audio lasts, so EXACTLY 30 s of audio still
|
|
89
|
+
* produces two windows (the second repeats the final second). That is the pinned overlap,
|
|
90
|
+
* not an off-by-one: a filler that starts in the last second must appear whole somewhere.
|
|
91
|
+
*
|
|
92
|
+
* @param samples - mono 16 kHz samples
|
|
93
|
+
* @param geometry - window and hop sizes; defaults to the uhm geometry
|
|
94
|
+
* @returns the windows, in order
|
|
95
|
+
*/
|
|
96
|
+
export declare function windowsOf(samples: Float32Array, geometry?: WindowGeometry): readonly AudioWindow[];
|
|
97
|
+
//# sourceMappingURL=wav.d.ts.map
|
package/lib/wav.d.ts.map
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"wav.d.ts","sourceRoot":"","sources":["../src/wav.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAEH,+BAA+B;AAC/B,eAAO,MAAM,eAAe,QAAS,CAAA;AAErC,qDAAqD;AACrD,eAAO,MAAM,kBAAkB,SAAU,CAAA;AAEzC,iEAAiE;AACjE,eAAO,MAAM,mBAAmB,QAAS,CAAA;AAEzC,iEAAiE;AACjE,eAAO,MAAM,eAAe,QAA2C,CAAA;AAEvE,4DAA4D;AAC5D,eAAO,MAAM,iBAAiB,MAAM,CAAA;AAEpC,+CAA+C;AAC/C,eAAO,MAAM,YAAY,KAAK,CAAA;AAE9B,4DAA4D;AAC5D,MAAM,WAAW,QAAQ;IACvB,kCAAkC;IAClC,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAA;IAC3B,2CAA2C;IAC3C,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAA;IACzB,yCAAyC;IACzC,QAAQ,CAAC,QAAQ,EAAE,OAAO,GAAG,SAAS,CAAA;IACtC,2DAA2D;IAC3D,QAAQ,CAAC,OAAO,EAAE,YAAY,CAAA;CAC/B;AAED,sCAAsC;AACtC,MAAM,MAAM,eAAe,GACvB;IAAE,QAAQ,CAAC,EAAE,EAAE,IAAI,CAAC;IAAC,QAAQ,CAAC,KAAK,EAAE,QAAQ,CAAA;CAAE,GAC/C;IAAE,QAAQ,CAAC,EAAE,EAAE,KAAK,CAAC;IAAC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;CAAE,CAAA;AAEnD,kEAAkE;AAClE,MAAM,WAAW,WAAW;IAC1B,8BAA8B;IAC9B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;IACtB,kDAAkD;IAClD,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAA;IAC5B,iEAAiE;IACjE,QAAQ,CAAC,OAAO,EAAE,YAAY,CAAA;CAC/B;AAED,oCAAoC;AACpC,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAA;IAC9B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAA;CAC5B;AAiBD;;;;;;;;;GASG;AACH,wBAAgB,SAAS,CAAC,KAAK,EAAE,UAAU,GAAG,eAAe,CAoF5D;AAED;;;;;;;;;;;GAWG;AACH,wBAAgB,cAAc,CAAC,OAAO,EAAE,YAAY,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,YAAY,CAoBpG;AAED;;;;;;;;;;GAUG;AACH,wBAAgB,SAAS,CACvB,OAAO,EAAE,YAAY,EACrB,QAAQ,GAAE,cAAmF,GAC5F,SAAS,WAAW,EAAE,CAsBxB"}
|
package/lib/wav.js
ADDED
|
@@ -0,0 +1,196 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* WAV decoding, resampling and windowing (PHASE3 processor uhm/waveform-v1, steps 1-2).
|
|
3
|
+
*
|
|
4
|
+
* Two facts drive the whole module:
|
|
5
|
+
*
|
|
6
|
+
* - **The model consumes 16 kHz mono float samples.** A caller's file may be any rate,
|
|
7
|
+
* one or two channels, PCM 16-bit or float32, so decode and resample are separate steps
|
|
8
|
+
* with separate tests rather than one opaque pipeline.
|
|
9
|
+
* - **The window geometry is the MODEL's, not the caller's.** 30 s windows with a 1 s
|
|
10
|
+
* overlap and a zero-padded final window are pinned constants (480000 samples, hop
|
|
11
|
+
* 464000). The overlap exists so a filler word spanning a window boundary is seen
|
|
12
|
+
* whole by one of the two windows; the frame timeline in `frames.ts` merges them.
|
|
13
|
+
*
|
|
14
|
+
* @module wav
|
|
15
|
+
*/
|
|
16
|
+
/** The model's sample rate. */
|
|
17
|
+
export const UHM_SAMPLE_RATE = 16_000;
|
|
18
|
+
/** One model window, in samples (30 s at 16 kHz). */
|
|
19
|
+
export const UHM_WINDOW_SAMPLES = 480_000;
|
|
20
|
+
/** The overlap between consecutive windows, in samples (1 s). */
|
|
21
|
+
export const UHM_OVERLAP_SAMPLES = 16_000;
|
|
22
|
+
/** The hop between consecutive windows: window minus overlap. */
|
|
23
|
+
export const UHM_HOP_SAMPLES = UHM_WINDOW_SAMPLES - UHM_OVERLAP_SAMPLES;
|
|
24
|
+
/** One model output frame, in samples (20 ms at 16 kHz). */
|
|
25
|
+
export const UHM_FRAME_SAMPLES = 320;
|
|
26
|
+
/** One model output frame, in milliseconds. */
|
|
27
|
+
export const UHM_FRAME_MS = 20;
|
|
28
|
+
/** Read five ASCII bytes as text. */
|
|
29
|
+
function ascii(bytes, offset) {
|
|
30
|
+
return String.fromCharCode(bytes[offset] ?? 0, bytes[offset + 1] ?? 0, bytes[offset + 2] ?? 0, bytes[offset + 3] ?? 0);
|
|
31
|
+
}
|
|
32
|
+
/** Refuse a document with a named reason. */
|
|
33
|
+
function refuse(reason) {
|
|
34
|
+
return { ok: false, reason };
|
|
35
|
+
}
|
|
36
|
+
/**
|
|
37
|
+
* Decode a WAV document (RIFF/WAVE) to mono float samples.
|
|
38
|
+
*
|
|
39
|
+
* Accepted: `WAVE_FORMAT_PCM` with 16 bits, and `WAVE_FORMAT_IEEE_FLOAT` with 32 bits,
|
|
40
|
+
* one or two channels. Everything else is refused BY NAME rather than guessed at, because
|
|
41
|
+
* a mis-decoded waveform is a wrong transcript with no other symptom.
|
|
42
|
+
*
|
|
43
|
+
* @param bytes - the file bytes
|
|
44
|
+
* @returns the decoded audio, or the reason it cannot be decoded
|
|
45
|
+
*/
|
|
46
|
+
export function decodeWav(bytes) {
|
|
47
|
+
if (bytes.length < 12)
|
|
48
|
+
return refuse('the document is truncated: it is shorter than a RIFF header');
|
|
49
|
+
if (ascii(bytes, 0) !== 'RIFF') {
|
|
50
|
+
return refuse(`the document is not a RIFF file: it starts with ${JSON.stringify(ascii(bytes, 0))}`);
|
|
51
|
+
}
|
|
52
|
+
if (ascii(bytes, 8) !== 'WAVE') {
|
|
53
|
+
return refuse(`the document is not a WAVE document: its form type is ${JSON.stringify(ascii(bytes, 8))}`);
|
|
54
|
+
}
|
|
55
|
+
const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
|
|
56
|
+
let format;
|
|
57
|
+
let data;
|
|
58
|
+
let offset = 12;
|
|
59
|
+
while (offset + 8 <= bytes.length) {
|
|
60
|
+
const id = ascii(bytes, offset);
|
|
61
|
+
const size = view.getUint32(offset + 4, true);
|
|
62
|
+
const body = offset + 8;
|
|
63
|
+
if (body + size > bytes.length) {
|
|
64
|
+
return refuse(`the ${JSON.stringify(id)} chunk is truncated: it declares ${String(size)} bytes`);
|
|
65
|
+
}
|
|
66
|
+
if (id === 'fmt ') {
|
|
67
|
+
if (size < 16)
|
|
68
|
+
return refuse('the fmt chunk is too short to describe a sample format');
|
|
69
|
+
format = {
|
|
70
|
+
audioFormat: view.getUint16(body, true),
|
|
71
|
+
channels: view.getUint16(body + 2, true),
|
|
72
|
+
sampleRate: view.getUint32(body + 4, true),
|
|
73
|
+
bitsPerSample: view.getUint16(body + 14, true),
|
|
74
|
+
};
|
|
75
|
+
}
|
|
76
|
+
else if (id === 'data' && data === undefined) {
|
|
77
|
+
data = bytes.subarray(body, body + size);
|
|
78
|
+
}
|
|
79
|
+
// Chunks are word aligned.
|
|
80
|
+
offset = body + size + (size % 2);
|
|
81
|
+
}
|
|
82
|
+
if (format === undefined)
|
|
83
|
+
return refuse('the document has no fmt chunk, so its samples cannot be interpreted');
|
|
84
|
+
if (data === undefined)
|
|
85
|
+
return refuse('the document has no data chunk');
|
|
86
|
+
const encoding = format.audioFormat === 1
|
|
87
|
+
? 'pcm16'
|
|
88
|
+
: format.audioFormat === 3
|
|
89
|
+
? 'float32'
|
|
90
|
+
: undefined;
|
|
91
|
+
if (encoding === undefined) {
|
|
92
|
+
return refuse(`the audio format ${String(format.audioFormat)} is not supported; export PCM 16-bit or IEEE float32`);
|
|
93
|
+
}
|
|
94
|
+
const requiredBits = encoding === 'pcm16' ? 16 : 32;
|
|
95
|
+
const supportedLabel = encoding === 'pcm16' ? 'PCM 16-bit' : 'IEEE float32';
|
|
96
|
+
if (format.bitsPerSample !== requiredBits) {
|
|
97
|
+
return refuse(`${String(format.bitsPerSample)} bits per sample is not supported in this encoding; export ${supportedLabel}`);
|
|
98
|
+
}
|
|
99
|
+
if (format.channels !== 1 && format.channels !== 2) {
|
|
100
|
+
return refuse(`${String(format.channels)} audio channels is not supported; mono or stereo only`);
|
|
101
|
+
}
|
|
102
|
+
if (!Number.isInteger(format.sampleRate) || format.sampleRate <= 0) {
|
|
103
|
+
return refuse(`the sample rate ${String(format.sampleRate)} is not a positive integer`);
|
|
104
|
+
}
|
|
105
|
+
const bytesPerSample = requiredBits / 8;
|
|
106
|
+
const bytesPerFrame = bytesPerSample * format.channels;
|
|
107
|
+
if (data.length % bytesPerFrame !== 0) {
|
|
108
|
+
return refuse(`the data chunk is ${String(data.length)} bytes, which is not a whole number of ${String(format.channels)}-channel frames`);
|
|
109
|
+
}
|
|
110
|
+
const frames = data.length / bytesPerFrame;
|
|
111
|
+
const samples = new Float32Array(frames);
|
|
112
|
+
const dataOffset = data.byteOffset;
|
|
113
|
+
for (let frame = 0; frame < frames; frame += 1) {
|
|
114
|
+
let sum = 0;
|
|
115
|
+
for (let channel = 0; channel < format.channels; channel += 1) {
|
|
116
|
+
const at = dataOffset + frame * bytesPerFrame + channel * bytesPerSample;
|
|
117
|
+
const value = encoding === 'pcm16' ? view.getInt16(at, true) / 32_768 : view.getFloat32(at, true);
|
|
118
|
+
if (!Number.isFinite(value))
|
|
119
|
+
return refuse(`sample ${String(frame)} of channel ${String(channel)} is not finite`);
|
|
120
|
+
sum += value;
|
|
121
|
+
}
|
|
122
|
+
samples[frame] = sum / format.channels;
|
|
123
|
+
}
|
|
124
|
+
return { ok: true, audio: { sampleRate: format.sampleRate, channels: format.channels, encoding, samples } };
|
|
125
|
+
}
|
|
126
|
+
/**
|
|
127
|
+
* Resample mono samples with linear interpolation.
|
|
128
|
+
*
|
|
129
|
+
* Linear, not windowed-sinc: PHASE3 specifies linear, and the model was qualified on that
|
|
130
|
+
* path. A file already at the target rate is copied so a caller never aliases the file
|
|
131
|
+
* buffer.
|
|
132
|
+
*
|
|
133
|
+
* @param samples - mono source samples
|
|
134
|
+
* @param fromRate - the source sample rate
|
|
135
|
+
* @param toRate - the target sample rate
|
|
136
|
+
* @returns the resampled samples
|
|
137
|
+
*/
|
|
138
|
+
export function resampleLinear(samples, fromRate, toRate) {
|
|
139
|
+
if (!Number.isFinite(fromRate) || fromRate <= 0)
|
|
140
|
+
throw new Error('the source sample rate must be a positive number');
|
|
141
|
+
if (!Number.isFinite(toRate) || toRate <= 0)
|
|
142
|
+
throw new Error('the target sample rate must be a positive number');
|
|
143
|
+
if (samples.length === 0)
|
|
144
|
+
return new Float32Array(0);
|
|
145
|
+
if (fromRate === toRate)
|
|
146
|
+
return Float32Array.from(samples);
|
|
147
|
+
const length = Math.max(1, Math.round((samples.length * toRate) / fromRate));
|
|
148
|
+
const output = new Float32Array(length);
|
|
149
|
+
const ratio = fromRate / toRate;
|
|
150
|
+
const last = samples.length - 1;
|
|
151
|
+
for (let index = 0; index < length; index += 1) {
|
|
152
|
+
const position = index * ratio;
|
|
153
|
+
const lower = Math.min(Math.floor(position), last);
|
|
154
|
+
const upper = Math.min(lower + 1, last);
|
|
155
|
+
const fraction = position - lower;
|
|
156
|
+
const start = samples[lower] ?? 0;
|
|
157
|
+
const end = samples[upper] ?? start;
|
|
158
|
+
output[index] = start + (end - start) * fraction;
|
|
159
|
+
}
|
|
160
|
+
return output;
|
|
161
|
+
}
|
|
162
|
+
/**
|
|
163
|
+
* Slice mono samples into zero-padded model windows.
|
|
164
|
+
*
|
|
165
|
+
* A window starts at every hop while the audio lasts, so EXACTLY 30 s of audio still
|
|
166
|
+
* produces two windows (the second repeats the final second). That is the pinned overlap,
|
|
167
|
+
* not an off-by-one: a filler that starts in the last second must appear whole somewhere.
|
|
168
|
+
*
|
|
169
|
+
* @param samples - mono 16 kHz samples
|
|
170
|
+
* @param geometry - window and hop sizes; defaults to the uhm geometry
|
|
171
|
+
* @returns the windows, in order
|
|
172
|
+
*/
|
|
173
|
+
export function windowsOf(samples, geometry = { windowSamples: UHM_WINDOW_SAMPLES, hopSamples: UHM_HOP_SAMPLES }) {
|
|
174
|
+
const { windowSamples, hopSamples } = geometry;
|
|
175
|
+
if (!Number.isInteger(windowSamples) || windowSamples <= 0) {
|
|
176
|
+
throw new Error('the window sample count must be a positive integer');
|
|
177
|
+
}
|
|
178
|
+
if (!Number.isInteger(hopSamples) || hopSamples <= 0) {
|
|
179
|
+
throw new Error('the hop sample count must be a positive integer');
|
|
180
|
+
}
|
|
181
|
+
if (hopSamples > windowSamples) {
|
|
182
|
+
throw new Error('the hop must not exceed the window, which would skip audio between windows');
|
|
183
|
+
}
|
|
184
|
+
if (samples.length === 0)
|
|
185
|
+
return [];
|
|
186
|
+
const windows = [];
|
|
187
|
+
let index = 0;
|
|
188
|
+
for (let start = 0; start < samples.length; start += hopSamples) {
|
|
189
|
+
const window = new Float32Array(windowSamples);
|
|
190
|
+
window.set(samples.subarray(start, Math.min(start + windowSamples, samples.length)));
|
|
191
|
+
windows.push({ index, startSample: start, samples: window });
|
|
192
|
+
index += 1;
|
|
193
|
+
}
|
|
194
|
+
return windows;
|
|
195
|
+
}
|
|
196
|
+
//# sourceMappingURL=wav.js.map
|
package/package.json
CHANGED
|
@@ -1,6 +1,50 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@try-works/tool-mode-runtime-onnx",
|
|
3
|
-
"version": "0.
|
|
4
|
-
"
|
|
5
|
-
"
|
|
6
|
-
|
|
3
|
+
"version": "0.1.0",
|
|
4
|
+
"description": "ONNX Runtime engine for Tool Mode: an Effect OnnxSession with execution-provider negotiation and the uhm/waveform-v1 filler processor",
|
|
5
|
+
"license": "Apache-2.0",
|
|
6
|
+
"publishConfig": {
|
|
7
|
+
"access": "public",
|
|
8
|
+
"registry": "https://registry.npmjs.org/"
|
|
9
|
+
},
|
|
10
|
+
"repository": {
|
|
11
|
+
"type": "git",
|
|
12
|
+
"url": "git+https://github.com/try-works/tool-mode.git",
|
|
13
|
+
"directory": "packages/tool-mode-runtime-onnx"
|
|
14
|
+
},
|
|
15
|
+
"type": "module",
|
|
16
|
+
"main": "lib/index.js",
|
|
17
|
+
"types": "lib/index.d.ts",
|
|
18
|
+
"exports": {
|
|
19
|
+
".": {
|
|
20
|
+
"types": "./lib/index.d.ts",
|
|
21
|
+
"default": "./lib/index.js"
|
|
22
|
+
},
|
|
23
|
+
"./package.json": "./package.json"
|
|
24
|
+
},
|
|
25
|
+
"files": [
|
|
26
|
+
"lib/**/*.js",
|
|
27
|
+
"lib/**/*.d.ts",
|
|
28
|
+
"lib/**/*.d.ts.map",
|
|
29
|
+
"README.md"
|
|
30
|
+
],
|
|
31
|
+
"dependencies": {
|
|
32
|
+
"@try-works/tool-mode-extension-sdk": "0.1.0",
|
|
33
|
+
"@try-works/tool-mode-runtime-core": "0.1.0",
|
|
34
|
+
"@try-works/tool-mode-runtime-recipes": "0.1.0",
|
|
35
|
+
"onnxruntime-node": "1.30.0"
|
|
36
|
+
},
|
|
37
|
+
"peerDependencies": {
|
|
38
|
+
"effect": "4.1.2"
|
|
39
|
+
},
|
|
40
|
+
"scripts": {
|
|
41
|
+
"build": "tsc -p tsconfig.build.json",
|
|
42
|
+
"typecheck": "tsc --noEmit -p tsconfig.json",
|
|
43
|
+
"test": "tsx --test tests/*.test.ts"
|
|
44
|
+
},
|
|
45
|
+
"devDependencies": {
|
|
46
|
+
"@types/node": "^24.0.0",
|
|
47
|
+
"tsx": "^4.20.0",
|
|
48
|
+
"typescript": "^5.9.0"
|
|
49
|
+
}
|
|
50
|
+
}
|