@onjmin/dtm 2.1.12 → 2.1.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +26 -0
- package/dist/index.d.mts +363 -110
- package/dist/index.d.ts +363 -110
- package/dist/index.js +2622 -486
- package/dist/index.mjs +2610 -486
- package/dist/voice-worker.js +547 -5
- package/package.json +4 -3
package/dist/voice-worker.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
(() => {
|
|
3
|
-
// node_modules/.pnpm/@onjmin+koe@1.0.
|
|
3
|
+
// node_modules/.pnpm/@onjmin+koe@1.0.10/node_modules/@onjmin/koe/dist/index.js
|
|
4
4
|
var u8 = Uint8Array;
|
|
5
5
|
var u16 = Uint16Array;
|
|
6
6
|
var i32 = Int32Array;
|
|
@@ -388,6 +388,130 @@
|
|
|
388
388
|
* @returns Float32 PCM, or null when `pcm` is shorter than
|
|
389
389
|
* {@link MIN_WORLDLINE_SAMPLES} (too short for stable F0 analysis).
|
|
390
390
|
*/
|
|
391
|
+
renderPhrase(params) {
|
|
392
|
+
const {
|
|
393
|
+
units,
|
|
394
|
+
pitch,
|
|
395
|
+
gender = 0.5,
|
|
396
|
+
tension = 0.5,
|
|
397
|
+
breathiness = 0.5,
|
|
398
|
+
voicing = 1,
|
|
399
|
+
tempo = 120
|
|
400
|
+
} = params;
|
|
401
|
+
if (units.length === 0) return null;
|
|
402
|
+
const WL = this.wasm;
|
|
403
|
+
const FS2 = WORLDLINE_SAMPLE_RATE;
|
|
404
|
+
let totalMs = 0;
|
|
405
|
+
for (const u of units) {
|
|
406
|
+
const endMs = u.posMs + u.lengthMs;
|
|
407
|
+
if (endMs > totalMs) totalMs = endMs;
|
|
408
|
+
}
|
|
409
|
+
const ps = WL._PhraseSynthNew();
|
|
410
|
+
if (!ps) return null;
|
|
411
|
+
const pointersToFree = [];
|
|
412
|
+
for (const u of units) {
|
|
413
|
+
if (!u.pcm || u.pcm.length < MIN_WORLDLINE_SAMPLES) continue;
|
|
414
|
+
const reqPtr = WL._malloc(SYNTH_REQ_SIZE);
|
|
415
|
+
if (!reqPtr) continue;
|
|
416
|
+
pointersToFree.push(reqPtr);
|
|
417
|
+
const samplePtr = WL._malloc(u.pcm.length * 8);
|
|
418
|
+
if (!samplePtr) continue;
|
|
419
|
+
pointersToFree.push(samplePtr);
|
|
420
|
+
WL.HEAPF64.set(u.pcm, samplePtr >> 3);
|
|
421
|
+
const sv = (off, val, type) => WL.setValue(reqPtr + off, val, type);
|
|
422
|
+
sv(0, FS2, "i32");
|
|
423
|
+
sv(4, u.pcm.length, "i32");
|
|
424
|
+
sv(8, samplePtr, "*");
|
|
425
|
+
sv(12, 0, "i32");
|
|
426
|
+
sv(16, 0, "*");
|
|
427
|
+
sv(20, u.tone ?? 69, "i32");
|
|
428
|
+
sv(24, 100, "double");
|
|
429
|
+
sv(32, 0, "double");
|
|
430
|
+
sv(40, u.requiredLengthMs ?? u.lengthMs, "double");
|
|
431
|
+
sv(48, u.consonantMs, "double");
|
|
432
|
+
const cutMs = u.cutMs ?? WL_FRAME_MS * 2;
|
|
433
|
+
sv(56, cutMs, "double");
|
|
434
|
+
sv(64, u.volume ?? 100, "double");
|
|
435
|
+
sv(72, 0, "double");
|
|
436
|
+
sv(80, tempo, "double");
|
|
437
|
+
sv(88, 0, "i32");
|
|
438
|
+
sv(92, 0, "*");
|
|
439
|
+
sv(96, 0, "i32");
|
|
440
|
+
sv(100, 0, "i32");
|
|
441
|
+
sv(104, 100, "i32");
|
|
442
|
+
sv(108, 0, "i32");
|
|
443
|
+
sv(112, 0, "i32");
|
|
444
|
+
sv(116, 100, "i32");
|
|
445
|
+
WL._PhraseSynthAddRequest(
|
|
446
|
+
ps,
|
|
447
|
+
reqPtr,
|
|
448
|
+
u.posMs,
|
|
449
|
+
u.skipMs,
|
|
450
|
+
u.lengthMs,
|
|
451
|
+
u.fadeInMs,
|
|
452
|
+
u.fadeOutMs,
|
|
453
|
+
0
|
|
454
|
+
);
|
|
455
|
+
}
|
|
456
|
+
totalMs += WL_FRAME_MS * 2;
|
|
457
|
+
const nFrames = Math.ceil(totalMs / WL_FRAME_MS) + 4;
|
|
458
|
+
const f0Arr = new Float64Array(nFrames);
|
|
459
|
+
const gArr = new Float64Array(nFrames);
|
|
460
|
+
const tArr = new Float64Array(nFrames);
|
|
461
|
+
const bArr = new Float64Array(nFrames);
|
|
462
|
+
const vArr = new Float64Array(nFrames);
|
|
463
|
+
for (let i2 = 0; i2 < nFrames; i2++) {
|
|
464
|
+
const tMs = i2 * WL_FRAME_MS;
|
|
465
|
+
f0Arr[i2] = sampleCurve(tMs, pitch, totalMs);
|
|
466
|
+
gArr[i2] = sampleCurve(tMs, gender, totalMs);
|
|
467
|
+
tArr[i2] = sampleCurve(tMs, tension, totalMs);
|
|
468
|
+
bArr[i2] = sampleCurve(tMs, breathiness, totalMs);
|
|
469
|
+
vArr[i2] = sampleCurve(tMs, voicing, totalMs);
|
|
470
|
+
}
|
|
471
|
+
const f0Ptr = WL._malloc(nFrames * 8);
|
|
472
|
+
const gPtr = WL._malloc(nFrames * 8);
|
|
473
|
+
const tPtr = WL._malloc(nFrames * 8);
|
|
474
|
+
const bPtr = WL._malloc(nFrames * 8);
|
|
475
|
+
const vPtr = WL._malloc(nFrames * 8);
|
|
476
|
+
if (f0Ptr && gPtr && tPtr && bPtr && vPtr) {
|
|
477
|
+
WL.HEAPF64.set(f0Arr, f0Ptr >> 3);
|
|
478
|
+
WL.HEAPF64.set(gArr, gPtr >> 3);
|
|
479
|
+
WL.HEAPF64.set(tArr, tPtr >> 3);
|
|
480
|
+
WL.HEAPF64.set(bArr, bPtr >> 3);
|
|
481
|
+
WL.HEAPF64.set(vArr, vPtr >> 3);
|
|
482
|
+
WL._PhraseSynthSetCurves(
|
|
483
|
+
ps,
|
|
484
|
+
f0Ptr,
|
|
485
|
+
gPtr,
|
|
486
|
+
tPtr,
|
|
487
|
+
bPtr,
|
|
488
|
+
vPtr,
|
|
489
|
+
nFrames,
|
|
490
|
+
WL_FRAME_MS
|
|
491
|
+
);
|
|
492
|
+
}
|
|
493
|
+
if (f0Ptr) WL._free(f0Ptr);
|
|
494
|
+
if (gPtr) WL._free(gPtr);
|
|
495
|
+
if (tPtr) WL._free(tPtr);
|
|
496
|
+
if (bPtr) WL._free(bPtr);
|
|
497
|
+
if (vPtr) WL._free(vPtr);
|
|
498
|
+
const yPtrPtr = WL._malloc(4);
|
|
499
|
+
let audio = null;
|
|
500
|
+
if (yPtrPtr) {
|
|
501
|
+
const outLen = WL._PhraseSynthSynth(ps, yPtrPtr, 0);
|
|
502
|
+
const yPtr = WL.getValue(yPtrPtr, "*");
|
|
503
|
+
if (outLen > 0 && yPtr) {
|
|
504
|
+
audio = new Float32Array(WL.HEAPF32.buffer, yPtr, outLen).slice();
|
|
505
|
+
WL._free(yPtr);
|
|
506
|
+
}
|
|
507
|
+
WL._free(yPtrPtr);
|
|
508
|
+
}
|
|
509
|
+
for (const ptr of pointersToFree) {
|
|
510
|
+
WL._free(ptr);
|
|
511
|
+
}
|
|
512
|
+
WL._PhraseSynthDelete(ps);
|
|
513
|
+
return audio;
|
|
514
|
+
}
|
|
391
515
|
renderNote(params) {
|
|
392
516
|
const {
|
|
393
517
|
pcm,
|
|
@@ -403,8 +527,12 @@
|
|
|
403
527
|
} = params;
|
|
404
528
|
if (!pcm || pcm.length < MIN_WORLDLINE_SAMPLES) return null;
|
|
405
529
|
const WL = this.wasm;
|
|
406
|
-
const
|
|
407
|
-
const basePitch = sampleCurve(
|
|
530
|
+
const FS2 = WORLDLINE_SAMPLE_RATE;
|
|
531
|
+
const basePitch = sampleCurve(
|
|
532
|
+
preMs + durationMs / 2,
|
|
533
|
+
pitch,
|
|
534
|
+
preMs + durationMs
|
|
535
|
+
);
|
|
408
536
|
const midiNote = Math.round(69 + 12 * Math.log2(basePitch / 440));
|
|
409
537
|
const posMs = 0;
|
|
410
538
|
const reqLen = preMs + durationMs;
|
|
@@ -424,7 +552,7 @@
|
|
|
424
552
|
}
|
|
425
553
|
WL.HEAPF64.set(pcm, samplePtr >> 3);
|
|
426
554
|
const sv = (off, val, type) => WL.setValue(reqPtr + off, val, type);
|
|
427
|
-
sv(0,
|
|
555
|
+
sv(0, FS2, "i32");
|
|
428
556
|
sv(4, pcm.length, "i32");
|
|
429
557
|
sv(8, samplePtr, "*");
|
|
430
558
|
sv(12, 0, "i32");
|
|
@@ -512,6 +640,370 @@
|
|
|
512
640
|
return audio;
|
|
513
641
|
}
|
|
514
642
|
};
|
|
643
|
+
var RATE = 16e3;
|
|
644
|
+
var HOP_MS = 2;
|
|
645
|
+
var HOP = RATE * HOP_MS / 1e3;
|
|
646
|
+
var FFT_SIZE = 512;
|
|
647
|
+
var BINS = FFT_SIZE / 2;
|
|
648
|
+
var PITCH_DECIM = 2;
|
|
649
|
+
var PITCH_RATE = RATE / PITCH_DECIM;
|
|
650
|
+
var WINDOW_CENTRE_MS = FFT_SIZE / 2 / RATE * 1e3;
|
|
651
|
+
function sparse_features(token) {
|
|
652
|
+
if (token.pause || token.accent_phrase_position === void 0 || token.accent_phrase_length === void 0 || token.accent_nucleus === void 0) {
|
|
653
|
+
return {};
|
|
654
|
+
}
|
|
655
|
+
const phrase_length = Math.max(1, token.accent_phrase_length);
|
|
656
|
+
const phrase_position = token.accent_phrase_position;
|
|
657
|
+
const nucleus = token.accent_nucleus;
|
|
658
|
+
const result = {
|
|
659
|
+
"accent_position": phrase_position / phrase_length,
|
|
660
|
+
"accent_from_end": (phrase_length - phrase_position) / phrase_length,
|
|
661
|
+
"accent_nucleus_position": nucleus / phrase_length,
|
|
662
|
+
"accent_high": token.accent_high ? 1 : 0,
|
|
663
|
+
"accent_phrase_start": token.accent_phrase_start ? 1 : 0,
|
|
664
|
+
"accent_phrase_end": token.accent_phrase_end ? 1 : 0,
|
|
665
|
+
"word_start": token.word_start ? 1 : 0,
|
|
666
|
+
"word_end": token.word_end ? 1 : 0
|
|
667
|
+
};
|
|
668
|
+
result[`pos=${token.pos || "*"}`] = 1;
|
|
669
|
+
result[`pos_group1=${token.pos_group1 || "*"}`] = 1;
|
|
670
|
+
if (nucleus === 0) {
|
|
671
|
+
result["accent_type=heiban"] = 1;
|
|
672
|
+
} else if (phrase_position < nucleus) {
|
|
673
|
+
result["accent_type=before"] = 1;
|
|
674
|
+
} else if (phrase_position === nucleus) {
|
|
675
|
+
result["accent_type=nucleus"] = 1;
|
|
676
|
+
} else {
|
|
677
|
+
result["accent_type=after"] = 1;
|
|
678
|
+
}
|
|
679
|
+
return result;
|
|
680
|
+
}
|
|
681
|
+
function readingFromFeatures(features) {
|
|
682
|
+
return features.map((frame) => frame.pause ? "\u3001" : frame.mora).join("");
|
|
683
|
+
}
|
|
684
|
+
var FS = WORLDLINE_SAMPLE_RATE;
|
|
685
|
+
var msToSamples2 = (ms) => Math.round(ms / 1e3 * FS);
|
|
686
|
+
function f0At(timeline, tMs) {
|
|
687
|
+
const curve = timeline.f0_curve;
|
|
688
|
+
if (curve.length === 0) return timeline.reference_hz || 220;
|
|
689
|
+
const position = Math.max(0, tMs) / timeline.frame_ms;
|
|
690
|
+
const left = Math.floor(position);
|
|
691
|
+
if (left >= curve.length - 1) return curve[curve.length - 1];
|
|
692
|
+
const progress = position - left;
|
|
693
|
+
return curve[left] * (1 - progress) + curve[left + 1] * progress;
|
|
694
|
+
}
|
|
695
|
+
function applyMoraGains(plan, gains) {
|
|
696
|
+
const morae = plan.morae ?? [];
|
|
697
|
+
const timings = plan.mora_timings ?? [];
|
|
698
|
+
const byIndex = morae.length === gains.length;
|
|
699
|
+
const moraAt = (unit) => {
|
|
700
|
+
if (byIndex) return unit.position;
|
|
701
|
+
let found = -1;
|
|
702
|
+
for (let i2 = 0; i2 < timings.length && i2 < gains.length; i2++) {
|
|
703
|
+
if (timings[i2].StartMS <= unit.note_start_ms + 1e-6) found = i2;
|
|
704
|
+
else break;
|
|
705
|
+
}
|
|
706
|
+
return found;
|
|
707
|
+
};
|
|
708
|
+
for (const unit of plan.timeline.units) {
|
|
709
|
+
const index = moraAt(unit);
|
|
710
|
+
const gain = gains[index];
|
|
711
|
+
if (index < 0 || gain === void 0 || !Number.isFinite(gain)) continue;
|
|
712
|
+
unit.volume *= gain;
|
|
713
|
+
}
|
|
714
|
+
}
|
|
715
|
+
function planChunks(units, firstChunkUnits, chunkUnits) {
|
|
716
|
+
const ranges = [];
|
|
717
|
+
if (units.length === 0) return ranges;
|
|
718
|
+
let start = 0;
|
|
719
|
+
let coveredEndMs = units[0].position_ms + units[0].length_ms;
|
|
720
|
+
let headSeam = false;
|
|
721
|
+
for (let i2 = 1; i2 <= units.length; i2++) {
|
|
722
|
+
const limit = ranges.length === 0 ? firstChunkUnits : chunkUnits;
|
|
723
|
+
const atEnd = i2 === units.length;
|
|
724
|
+
const clean = !atEnd && units[i2].position_ms >= coveredEndMs - 1e-6;
|
|
725
|
+
const full = !atEnd && i2 - start >= limit;
|
|
726
|
+
if (atEnd || clean || full) {
|
|
727
|
+
const tailSeam = !atEnd && !clean;
|
|
728
|
+
ranges.push({ start, end: i2, headSeam, tailSeam });
|
|
729
|
+
start = i2;
|
|
730
|
+
headSeam = tailSeam;
|
|
731
|
+
if (!atEnd) coveredEndMs = units[i2].position_ms + units[i2].length_ms;
|
|
732
|
+
continue;
|
|
733
|
+
}
|
|
734
|
+
coveredEndMs = Math.max(
|
|
735
|
+
coveredEndMs,
|
|
736
|
+
units[i2].position_ms + units[i2].length_ms
|
|
737
|
+
);
|
|
738
|
+
}
|
|
739
|
+
return ranges;
|
|
740
|
+
}
|
|
741
|
+
function seamTimeMs(next, crossfadeMs) {
|
|
742
|
+
const earliest = next.position_ms + next.fade_in_ms + crossfadeMs / 2;
|
|
743
|
+
const latest = next.position_ms + next.length_ms - crossfadeMs / 2;
|
|
744
|
+
return Math.min(
|
|
745
|
+
earliest,
|
|
746
|
+
Math.max(next.position_ms + crossfadeMs / 2, latest)
|
|
747
|
+
);
|
|
748
|
+
}
|
|
749
|
+
function applyFade(pcm, fromSample, samples, fadeIn) {
|
|
750
|
+
const n = Math.max(1, Math.min(samples, pcm.length - fromSample));
|
|
751
|
+
for (let k = 0; k < n; k++) {
|
|
752
|
+
const t = (k + 0.5) / n;
|
|
753
|
+
const gain = fadeIn ? Math.sin(Math.PI / 2 * t) : Math.cos(Math.PI / 2 * t);
|
|
754
|
+
pcm[fromSample + k] *= gain;
|
|
755
|
+
}
|
|
756
|
+
}
|
|
757
|
+
var UtauTTSAdapter = class _UtauTTSAdapter {
|
|
758
|
+
constructor(worldline2) {
|
|
759
|
+
this.worldline = worldline2;
|
|
760
|
+
}
|
|
761
|
+
worldline;
|
|
762
|
+
static modelId = null;
|
|
763
|
+
bankAliases = /* @__PURE__ */ new WeakMap();
|
|
764
|
+
currentBank = null;
|
|
765
|
+
pcmCache = /* @__PURE__ */ new Map();
|
|
766
|
+
/**
|
|
767
|
+
* Load the UtauTTS Go wasm. `wasm_exec.js` must already be on the page.
|
|
768
|
+
* Pass `fetch` (e.g. koe's `fetchAsset`) to stream from the Cache API.
|
|
769
|
+
*/
|
|
770
|
+
static async initializeWasm(wasmUrl = "utautts.wasm", options = {}) {
|
|
771
|
+
if (typeof utautts_plan === "function") return;
|
|
772
|
+
if (typeof Go === "undefined") {
|
|
773
|
+
throw new Error(
|
|
774
|
+
"wasm_exec.js must be loaded before calling initializeWasm."
|
|
775
|
+
);
|
|
776
|
+
}
|
|
777
|
+
const go = new Go();
|
|
778
|
+
const responsePromise = (options.fetch ?? fetch)(wasmUrl);
|
|
779
|
+
let instance;
|
|
780
|
+
try {
|
|
781
|
+
instance = (await WebAssembly.instantiateStreaming(responsePromise, go.importObject)).instance;
|
|
782
|
+
} catch {
|
|
783
|
+
const bytes = await (await (options.fetch ?? fetch)(wasmUrl)).arrayBuffer();
|
|
784
|
+
instance = (await WebAssembly.instantiate(bytes, go.importObject)).instance;
|
|
785
|
+
}
|
|
786
|
+
void go.run(instance);
|
|
787
|
+
for (let attempt = 0; attempt < 100 && typeof utautts_plan !== "function"; attempt++) {
|
|
788
|
+
await new Promise((resolve) => setTimeout(resolve, 10));
|
|
789
|
+
}
|
|
790
|
+
if (typeof utautts_plan !== "function") {
|
|
791
|
+
throw new Error("utautts_plan failed to initialize in global scope.");
|
|
792
|
+
}
|
|
793
|
+
}
|
|
794
|
+
static get ready() {
|
|
795
|
+
return typeof utautts_plan === "function";
|
|
796
|
+
}
|
|
797
|
+
/** Parse and cache a prosody model (e.g. `frame-intonation-v8.json`) inside the wasm. */
|
|
798
|
+
static setModel(modelJSON) {
|
|
799
|
+
_UtauTTSAdapter.assertReady();
|
|
800
|
+
const result = utautts_set_model(modelJSON);
|
|
801
|
+
if (!result.success)
|
|
802
|
+
throw new Error(`UtauTTS model error: ${result.error}`);
|
|
803
|
+
_UtauTTSAdapter.modelId = result.id ?? null;
|
|
804
|
+
return _UtauTTSAdapter.modelId;
|
|
805
|
+
}
|
|
806
|
+
static get currentModelId() {
|
|
807
|
+
return _UtauTTSAdapter.modelId;
|
|
808
|
+
}
|
|
809
|
+
static assertReady() {
|
|
810
|
+
if (typeof utautts_plan !== "function") {
|
|
811
|
+
throw new Error(
|
|
812
|
+
"UtauTTS wasm is not initialized; call UtauTTSAdapter.initializeWasm() first."
|
|
813
|
+
);
|
|
814
|
+
}
|
|
815
|
+
}
|
|
816
|
+
/**
|
|
817
|
+
* Register a koe voice bank with the planner: the manifest becomes a virtual
|
|
818
|
+
* oto.ini (koe PCM is pre-trimmed, so offset is 0) plus each sample's
|
|
819
|
+
* recorded pitch. Called automatically by {@link plan}; cheap when unchanged.
|
|
820
|
+
*/
|
|
821
|
+
setBank(bank2) {
|
|
822
|
+
_UtauTTSAdapter.assertReady();
|
|
823
|
+
if (this.currentBank === bank2 && this.bankAliases.has(bank2)) return;
|
|
824
|
+
const entries = {};
|
|
825
|
+
const pitch = {};
|
|
826
|
+
for (const [alias, phoneme] of Object.entries(bank2.manifest.phonemes)) {
|
|
827
|
+
entries[alias] = [
|
|
828
|
+
{
|
|
829
|
+
Filename: `koe:${alias}`,
|
|
830
|
+
Alias: alias,
|
|
831
|
+
Offset: 0,
|
|
832
|
+
Fixed: phoneme.consonant / 48,
|
|
833
|
+
Blank: 0,
|
|
834
|
+
Preutterance: phoneme.pre / 48,
|
|
835
|
+
Overlap: phoneme.overlap / 48,
|
|
836
|
+
SourceGroup: "koe"
|
|
837
|
+
}
|
|
838
|
+
];
|
|
839
|
+
if (phoneme.pitch > 0) pitch[alias] = phoneme.pitch;
|
|
840
|
+
}
|
|
841
|
+
const result = utautts_set_bank(
|
|
842
|
+
JSON.stringify({
|
|
843
|
+
name: "koe",
|
|
844
|
+
oto_entries: entries,
|
|
845
|
+
source_pitch_hz: pitch
|
|
846
|
+
})
|
|
847
|
+
);
|
|
848
|
+
if (!result.success) throw new Error(`UtauTTS bank error: ${result.error}`);
|
|
849
|
+
this.bankAliases.set(bank2, result.aliases ?? 0);
|
|
850
|
+
this.currentBank = bank2;
|
|
851
|
+
this.pcmCache.clear();
|
|
852
|
+
}
|
|
853
|
+
/**
|
|
854
|
+
* Plan an utterance: unit selection, timing, pitch contour and worldline
|
|
855
|
+
* placement. `features` are the mora-level frames from `openjtalkAnalyze`
|
|
856
|
+
* (pauses included); the kana reading is derived from them.
|
|
857
|
+
*/
|
|
858
|
+
plan(bank2, text, features, options = {}) {
|
|
859
|
+
this.setBank(bank2);
|
|
860
|
+
const request = {
|
|
861
|
+
text,
|
|
862
|
+
reading: readingFromFeatures(features),
|
|
863
|
+
frames: features.map((frame) => sparse_features(frame)),
|
|
864
|
+
tone: options.tone ?? "C4",
|
|
865
|
+
mora_duration_ms: options.moraDurationMs ?? 0,
|
|
866
|
+
pause_duration_ms: options.pauseDurationMs ?? 0,
|
|
867
|
+
release_ms: options.releaseMs ?? 20,
|
|
868
|
+
leading_preutterance_ms: options.leadingPreutteranceMs ?? 0,
|
|
869
|
+
apply_pitch: options.applyPitch ?? true,
|
|
870
|
+
intonation_strength: options.intonationStrength ?? 1,
|
|
871
|
+
speech_timing: options.speechTiming ?? false,
|
|
872
|
+
word_boundary_envelope: options.wordBoundaryEnvelope ?? false,
|
|
873
|
+
mora_durations_ms: options.prosody?.moraDurationsMs,
|
|
874
|
+
pitch_curve: options.prosody?.pitchCurve
|
|
875
|
+
};
|
|
876
|
+
const response = utautts_plan(JSON.stringify(request));
|
|
877
|
+
if (!response.success || !response.plan) {
|
|
878
|
+
throw new Error(`UtauTTS error: ${response.error ?? "no plan"}`);
|
|
879
|
+
}
|
|
880
|
+
const plan = JSON.parse(response.plan);
|
|
881
|
+
const gains = options.prosody?.moraGains;
|
|
882
|
+
if (gains) applyMoraGains(plan, gains);
|
|
883
|
+
return plan;
|
|
884
|
+
}
|
|
885
|
+
getPcm(bank2, alias) {
|
|
886
|
+
let cached = this.pcmCache.get(alias);
|
|
887
|
+
if (!cached) {
|
|
888
|
+
cached = bank2.getPcm(alias);
|
|
889
|
+
this.pcmCache.set(alias, cached);
|
|
890
|
+
}
|
|
891
|
+
return cached;
|
|
892
|
+
}
|
|
893
|
+
/**
|
|
894
|
+
* Render a plan chunk by chunk. Each chunk is independent audio positioned
|
|
895
|
+
* at `startMs`; schedule them as they arrive (see the demo) or sum them.
|
|
896
|
+
*
|
|
897
|
+
* Chunk breaks fall on pauses when possible. Inside a phrase a break renders
|
|
898
|
+
* one neighbouring unit of context on each side so the unit crossfade stays
|
|
899
|
+
* WORLD's spectral one, then the two renders are joined with a short
|
|
900
|
+
* equal-power crossfade in the following vowel.
|
|
901
|
+
*/
|
|
902
|
+
async *renderChunks(bank2, plan, options = {}) {
|
|
903
|
+
const {
|
|
904
|
+
firstChunkUnits = 3,
|
|
905
|
+
chunkUnits = 6,
|
|
906
|
+
seamCrossfadeMs = 20,
|
|
907
|
+
signal
|
|
908
|
+
} = options;
|
|
909
|
+
const timeline = plan.timeline;
|
|
910
|
+
const units = timeline.units.filter((unit) => unit.length_ms > 0);
|
|
911
|
+
const ranges = planChunks(
|
|
912
|
+
units,
|
|
913
|
+
Math.max(1, firstChunkUnits),
|
|
914
|
+
Math.max(1, chunkUnits)
|
|
915
|
+
);
|
|
916
|
+
const crossfadeSamples = Math.max(2, msToSamples2(seamCrossfadeMs));
|
|
917
|
+
for (let index = 0; index < ranges.length; index++) {
|
|
918
|
+
if (signal?.aborted) return;
|
|
919
|
+
const range = ranges[index];
|
|
920
|
+
const renderStart = range.headSeam ? range.start - 1 : range.start;
|
|
921
|
+
const renderEnd = range.tailSeam ? range.end + 1 : range.end;
|
|
922
|
+
const rendered = units.slice(renderStart, renderEnd);
|
|
923
|
+
const baseMs = Math.min(...rendered.map((unit) => unit.position_ms));
|
|
924
|
+
const phraseUnits = [];
|
|
925
|
+
for (const unit of rendered) {
|
|
926
|
+
const pcm2 = await this.getPcm(bank2, unit.alias);
|
|
927
|
+
if (!pcm2 || pcm2.length < MIN_WORLDLINE_SAMPLES) {
|
|
928
|
+
console.warn(
|
|
929
|
+
`[utautts] no usable PCM for alias "${unit.alias}"; skipped`
|
|
930
|
+
);
|
|
931
|
+
continue;
|
|
932
|
+
}
|
|
933
|
+
phraseUnits.push({
|
|
934
|
+
pcm: pcm2,
|
|
935
|
+
posMs: unit.position_ms - baseMs,
|
|
936
|
+
skipMs: unit.skip_ms,
|
|
937
|
+
lengthMs: unit.length_ms,
|
|
938
|
+
fadeInMs: unit.fade_in_ms,
|
|
939
|
+
fadeOutMs: unit.fade_out_ms,
|
|
940
|
+
consonantMs: unit.consonant_ms,
|
|
941
|
+
requiredLengthMs: unit.required_length_ms,
|
|
942
|
+
volume: unit.volume,
|
|
943
|
+
tone: unit.tone
|
|
944
|
+
});
|
|
945
|
+
}
|
|
946
|
+
if (signal?.aborted) return;
|
|
947
|
+
if (phraseUnits.length === 0) continue;
|
|
948
|
+
const audio = this.worldline.renderPhrase({
|
|
949
|
+
units: phraseUnits,
|
|
950
|
+
pitch: (tMs) => f0At(timeline, baseMs + tMs),
|
|
951
|
+
gender: options.gender,
|
|
952
|
+
tension: options.tension,
|
|
953
|
+
breathiness: options.breathiness,
|
|
954
|
+
voicing: options.voicing
|
|
955
|
+
});
|
|
956
|
+
if (!audio || audio.length === 0) continue;
|
|
957
|
+
let fromSample = 0;
|
|
958
|
+
let toSample = audio.length;
|
|
959
|
+
let startMs = baseMs;
|
|
960
|
+
if (range.headSeam) {
|
|
961
|
+
const seamMs = seamTimeMs(units[range.start], seamCrossfadeMs);
|
|
962
|
+
fromSample = Math.max(
|
|
963
|
+
0,
|
|
964
|
+
msToSamples2(seamMs - baseMs) - crossfadeSamples / 2
|
|
965
|
+
);
|
|
966
|
+
startMs = baseMs + fromSample / FS * 1e3;
|
|
967
|
+
}
|
|
968
|
+
if (range.tailSeam) {
|
|
969
|
+
const seamMs = seamTimeMs(units[range.end], seamCrossfadeMs);
|
|
970
|
+
toSample = Math.min(
|
|
971
|
+
audio.length,
|
|
972
|
+
msToSamples2(seamMs - baseMs) + crossfadeSamples / 2
|
|
973
|
+
);
|
|
974
|
+
}
|
|
975
|
+
if (toSample <= fromSample) continue;
|
|
976
|
+
const pcm = audio.slice(fromSample, toSample);
|
|
977
|
+
if (range.headSeam) applyFade(pcm, 0, crossfadeSamples, true);
|
|
978
|
+
if (range.tailSeam)
|
|
979
|
+
applyFade(
|
|
980
|
+
pcm,
|
|
981
|
+
Math.max(0, pcm.length - crossfadeSamples),
|
|
982
|
+
crossfadeSamples,
|
|
983
|
+
false
|
|
984
|
+
);
|
|
985
|
+
yield { pcm, startMs, index, units: units.slice(range.start, range.end) };
|
|
986
|
+
await new Promise((resolve) => setTimeout(resolve, 0));
|
|
987
|
+
}
|
|
988
|
+
}
|
|
989
|
+
/**
|
|
990
|
+
* Plan + render an utterance into one buffer (Float32, 48 kHz) covering the
|
|
991
|
+
* whole timeline. Use {@link plan} + {@link renderChunks} for streaming.
|
|
992
|
+
*/
|
|
993
|
+
async synthesizeText(bank2, text, features, options = {}) {
|
|
994
|
+
const plan = this.plan(bank2, text, features, options);
|
|
995
|
+
const total = msToSamples2(plan.timeline.duration_ms) + msToSamples2(200);
|
|
996
|
+
const out = new Float32Array(total);
|
|
997
|
+
let any = false;
|
|
998
|
+
for await (const chunk of this.renderChunks(bank2, plan, options)) {
|
|
999
|
+
any = true;
|
|
1000
|
+
const offset = msToSamples2(chunk.startMs);
|
|
1001
|
+
const n = Math.min(chunk.pcm.length, out.length - offset);
|
|
1002
|
+
for (let k = 0; k < n; k++) out[offset + k] += chunk.pcm[k];
|
|
1003
|
+
}
|
|
1004
|
+
return any ? out : null;
|
|
1005
|
+
}
|
|
1006
|
+
};
|
|
515
1007
|
|
|
516
1008
|
// src/vibrato.ts
|
|
517
1009
|
var VIBRATO_RATE_HZ = 5.5;
|
|
@@ -589,6 +1081,8 @@
|
|
|
589
1081
|
var wself = globalThis;
|
|
590
1082
|
var bank = null;
|
|
591
1083
|
var worldline = null;
|
|
1084
|
+
var speechAdapter = null;
|
|
1085
|
+
var speechAborts = /* @__PURE__ */ new Map();
|
|
592
1086
|
var pcmCache = /* @__PURE__ */ new Map();
|
|
593
1087
|
var getPcm = (alias) => {
|
|
594
1088
|
let p = pcmCache.get(alias);
|
|
@@ -701,7 +1195,8 @@
|
|
|
701
1195
|
);
|
|
702
1196
|
wself.postMessage({
|
|
703
1197
|
type: "ready",
|
|
704
|
-
aliases: Object.keys(bank.manifest.phonemes)
|
|
1198
|
+
aliases: Object.keys(bank.manifest.phonemes),
|
|
1199
|
+
phonemes: bank.manifest.phonemes
|
|
705
1200
|
});
|
|
706
1201
|
} catch (err) {
|
|
707
1202
|
wself.postMessage({
|
|
@@ -751,6 +1246,53 @@
|
|
|
751
1246
|
} catch {
|
|
752
1247
|
wself.postMessage({ type: "rendered", id, pcm: null });
|
|
753
1248
|
}
|
|
1249
|
+
return;
|
|
1250
|
+
}
|
|
1251
|
+
if (msg.type === "speak-abort") {
|
|
1252
|
+
speechAborts.get(msg.id)?.abort();
|
|
1253
|
+
return;
|
|
1254
|
+
}
|
|
1255
|
+
if (msg.type === "speak") {
|
|
1256
|
+
const { id, plan, gender, breathiness, tension } = msg;
|
|
1257
|
+
if (!bank || !worldline) {
|
|
1258
|
+
wself.postMessage({
|
|
1259
|
+
type: "speech-end",
|
|
1260
|
+
id,
|
|
1261
|
+
error: "speech needs worldline (not available in lightweight mode)"
|
|
1262
|
+
});
|
|
1263
|
+
return;
|
|
1264
|
+
}
|
|
1265
|
+
const abort = new AbortController();
|
|
1266
|
+
speechAborts.set(id, abort);
|
|
1267
|
+
try {
|
|
1268
|
+
speechAdapter ??= new UtauTTSAdapter(worldline);
|
|
1269
|
+
for await (const chunk of speechAdapter.renderChunks(bank, plan, {
|
|
1270
|
+
signal: abort.signal,
|
|
1271
|
+
gender,
|
|
1272
|
+
breathiness,
|
|
1273
|
+
tension
|
|
1274
|
+
})) {
|
|
1275
|
+
wself.postMessage(
|
|
1276
|
+
{
|
|
1277
|
+
type: "speech-chunk",
|
|
1278
|
+
id,
|
|
1279
|
+
pcm: chunk.pcm,
|
|
1280
|
+
startMs: chunk.startMs,
|
|
1281
|
+
index: chunk.index
|
|
1282
|
+
},
|
|
1283
|
+
[chunk.pcm.buffer]
|
|
1284
|
+
);
|
|
1285
|
+
}
|
|
1286
|
+
wself.postMessage({ type: "speech-end", id });
|
|
1287
|
+
} catch (err) {
|
|
1288
|
+
wself.postMessage({
|
|
1289
|
+
type: "speech-end",
|
|
1290
|
+
id,
|
|
1291
|
+
error: String(err?.message ?? err)
|
|
1292
|
+
});
|
|
1293
|
+
} finally {
|
|
1294
|
+
speechAborts.delete(id);
|
|
1295
|
+
}
|
|
754
1296
|
}
|
|
755
1297
|
};
|
|
756
1298
|
})();
|
package/package.json
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
"author": "onjmin",
|
|
3
3
|
"license": "MIT",
|
|
4
4
|
"name": "@onjmin/dtm",
|
|
5
|
-
"version": "2.1.
|
|
5
|
+
"version": "2.1.14",
|
|
6
6
|
"description": "MMLを中間言語に用いた、モバイルファーストなDAW / ピアノロール打ち込みコンポーネント",
|
|
7
7
|
"homepage": "https://onjmin.github.io/dtm",
|
|
8
8
|
"repository": {
|
|
@@ -56,14 +56,15 @@
|
|
|
56
56
|
},
|
|
57
57
|
"dependencies": {
|
|
58
58
|
"@onjmin/chord-parser": "1.1.1",
|
|
59
|
-
"@onjmin/koe": "1.0.
|
|
59
|
+
"@onjmin/koe": "1.0.10",
|
|
60
60
|
"midi-json-parser": "8.1.74"
|
|
61
61
|
},
|
|
62
62
|
"scripts": {
|
|
63
63
|
"dev": "rimraf dist && tsup && biome format src --write && tsx demo/server.ts",
|
|
64
64
|
"build": "rimraf dist && tsup && biome format src --write",
|
|
65
65
|
"check": "biome check src --write",
|
|
66
|
-
"
|
|
66
|
+
"version": "tsx scripts/sync-version.ts && git add src/version.ts docs/dataset-provenance.md",
|
|
67
|
+
"test": "tsx scripts/check-version.ts && tsx scripts/check-compose.ts && tsx scripts/check-registers.ts && tsx scripts/check-tracks.ts && tsx scripts/check-lyrics-fade.ts && tsx scripts/check-speech-lyrics.ts && tsx scripts/check-pitch-glide.ts && tsx scripts/check-ust.ts && tsx scripts/check-musicxml.ts && tsx scripts/check-backing-audio.ts && tsx scripts/check-mml-meta.ts && tsx scripts/check-macro-state.ts",
|
|
67
68
|
"patch": "pnpm version patch",
|
|
68
69
|
"minor": "pnpm version minor",
|
|
69
70
|
"major": "pnpm version major",
|