@anvpx/motion-audio 0.0.0-stage → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/dist/align.d.ts +40 -0
- package/dist/align.js +189 -0
- package/dist/fallback.d.ts +22 -0
- package/dist/fallback.js +29 -0
- package/dist/index.d.ts +42 -0
- package/dist/index.js +23 -0
- package/dist/kokoro.d.ts +52 -0
- package/dist/kokoro.js +156 -0
- package/dist/music.d.ts +34 -0
- package/dist/music.js +44 -0
- package/dist/silence.d.ts +31 -0
- package/dist/silence.js +84 -0
- package/dist/timing.d.ts +27 -0
- package/dist/timing.js +43 -0
- package/dist/triposr.d.ts +59 -0
- package/dist/triposr.js +241 -0
- package/package.json +30 -3
- package/README.md +0 -3
package/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 anvpx
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
package/dist/align.d.ts
ADDED
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* wav2vec2 forced align-only on KNOWN script tokens (EN, diarization OFF).
|
|
3
|
+
*
|
|
4
|
+
* TS port of the spike's `align-stage2.py` contract: the caller supplies the
|
|
5
|
+
* exact script tokens, the venv python returns per-word `[start, end]`
|
|
6
|
+
* boundaries, and this module validates them. No STT, no VAD chunking, no
|
|
7
|
+
* pyannote — unknown audio content can never invent words here because the
|
|
8
|
+
* transcript is an input, not an output. Malformed model output fails closed
|
|
9
|
+
* as `unknown`; it is never interpolated into fake timings.
|
|
10
|
+
*/
|
|
11
|
+
import type { Runner } from "./index.ts";
|
|
12
|
+
/** Pinned align-only model (spike Stage 2 receipt). */
|
|
13
|
+
export declare const ALIGN_MODEL_ID = "facebook/wav2vec2-base-960h";
|
|
14
|
+
export interface Word {
|
|
15
|
+
w: string;
|
|
16
|
+
start: number;
|
|
17
|
+
end: number;
|
|
18
|
+
}
|
|
19
|
+
export interface AlignRequest {
|
|
20
|
+
audioPath: string;
|
|
21
|
+
/** Exact known script tokens (surface forms, in order). */
|
|
22
|
+
tokens: string[];
|
|
23
|
+
/** Explicit python binary; skips the venv probe when set. */
|
|
24
|
+
python?: string;
|
|
25
|
+
}
|
|
26
|
+
export type AlignResult = {
|
|
27
|
+
ok: true;
|
|
28
|
+
words: Word[];
|
|
29
|
+
} | {
|
|
30
|
+
ok: false;
|
|
31
|
+
status: "missing" | "unknown";
|
|
32
|
+
hint: string;
|
|
33
|
+
};
|
|
34
|
+
/**
|
|
35
|
+
* Inline align driver: CTC trellis over the KNOWN token sequence only.
|
|
36
|
+
* Prints `{"words": [{"w", "start", "end"}]}`. Diarization is always OFF.
|
|
37
|
+
*/
|
|
38
|
+
export declare const ALIGN_PY: string;
|
|
39
|
+
/** Align known tokens to audio. Fails closed on any malformed output. */
|
|
40
|
+
export declare function alignWords(run: Runner, req: AlignRequest): Promise<AlignResult>;
|
package/dist/align.js
ADDED
|
@@ -0,0 +1,189 @@
|
|
|
1
|
+
import { resolveVenvPython } from "./kokoro.js";
|
|
2
|
+
/** Pinned align-only model (spike Stage 2 receipt). */
|
|
3
|
+
export const ALIGN_MODEL_ID = "facebook/wav2vec2-base-960h";
|
|
4
|
+
/** wav2vec2-base-960h is English-only; non-ASCII narration cannot align honestly. */
|
|
5
|
+
const isAsciiToken = (token) => {
|
|
6
|
+
for (let i = 0; i < token.length; i += 1) {
|
|
7
|
+
if (token.charCodeAt(i) > 127)
|
|
8
|
+
return false;
|
|
9
|
+
}
|
|
10
|
+
return true;
|
|
11
|
+
};
|
|
12
|
+
/**
|
|
13
|
+
* Inline align driver: CTC trellis over the KNOWN token sequence only.
|
|
14
|
+
* Prints `{"words": [{"w", "start", "end"}]}`. Diarization is always OFF.
|
|
15
|
+
*/
|
|
16
|
+
export const ALIGN_PY = [
|
|
17
|
+
"import json, sys",
|
|
18
|
+
"import re",
|
|
19
|
+
"import numpy as np",
|
|
20
|
+
"import soundfile as sf",
|
|
21
|
+
"import torch",
|
|
22
|
+
"from transformers import Wav2Vec2ForCTC, Wav2Vec2Processor",
|
|
23
|
+
'MODEL_ID = "facebook/wav2vec2-base-960h"',
|
|
24
|
+
"SR = 24000",
|
|
25
|
+
"FRAME_STRIDE_S = 320 / 16000",
|
|
26
|
+
"def clean_word(w):",
|
|
27
|
+
' return re.sub(r"[^A-Z\']", "", w.upper())',
|
|
28
|
+
"def aug_tokens(token_ids, blank):",
|
|
29
|
+
" aug = [blank]",
|
|
30
|
+
" for t in token_ids:",
|
|
31
|
+
" aug += [t, blank]",
|
|
32
|
+
" return aug",
|
|
33
|
+
"def trellis_forward(log_probs, aug, blank):",
|
|
34
|
+
" T = log_probs.shape[0]",
|
|
35
|
+
" S = len(aug)",
|
|
36
|
+
" NEG = -1e9",
|
|
37
|
+
" tr = np.full((T + 1, S + 1), NEG, dtype=np.float64)",
|
|
38
|
+
" tr[0, 0] = 0.0",
|
|
39
|
+
" emit = log_probs[:, np.asarray(aug, dtype=np.int64)]",
|
|
40
|
+
" for t in range(1, T + 1):",
|
|
41
|
+
" tr[t, 0] = tr[t - 1, 0] + log_probs[t - 1, blank]",
|
|
42
|
+
" stay = tr[t - 1, 1:] + emit[t - 1]",
|
|
43
|
+
" chg = tr[t - 1, :-1] + emit[t - 1]",
|
|
44
|
+
" tr[t, 1:] = np.maximum(stay, chg)",
|
|
45
|
+
" return tr",
|
|
46
|
+
"def backtrack_word_spans(tr, log_probs, aug, blank, word_ranges):",
|
|
47
|
+
" T = log_probs.shape[0]",
|
|
48
|
+
" S = len(aug)",
|
|
49
|
+
" path = [0] * T",
|
|
50
|
+
" t, s = T, S",
|
|
51
|
+
" while t > 0:",
|
|
52
|
+
" if s > 0:",
|
|
53
|
+
" emit = log_probs[t - 1, aug[s - 1]]",
|
|
54
|
+
" if tr[t - 1, s - 1] + emit > tr[t - 1, s] + emit:",
|
|
55
|
+
" s -= 1",
|
|
56
|
+
" path[t - 1] = s",
|
|
57
|
+
" t -= 1",
|
|
58
|
+
" spans = []",
|
|
59
|
+
" prev_end = 0",
|
|
60
|
+
" for (a0, a1) in word_ranges:",
|
|
61
|
+
" frames = [f for f in range(T) if path[f] > 0 and a0 <= path[f] - 1 < a1 and aug[path[f] - 1] != blank]",
|
|
62
|
+
" if frames:",
|
|
63
|
+
" span = (frames[0], frames[-1] + 1)",
|
|
64
|
+
" else:",
|
|
65
|
+
" span = (prev_end, prev_end)",
|
|
66
|
+
" spans.append(span)",
|
|
67
|
+
" prev_end = span[1]",
|
|
68
|
+
" return spans",
|
|
69
|
+
"def resample16(audio24, sr):",
|
|
70
|
+
" n_out = int(round(len(audio24) * 16000 / sr))",
|
|
71
|
+
" return np.interp(np.linspace(0, len(audio24), n_out, endpoint=False), np.arange(len(audio24)), audio24).astype(np.float32)",
|
|
72
|
+
"audio = sys.argv[sys.argv.index('--audio')+1]",
|
|
73
|
+
"tokens = json.loads(sys.argv[sys.argv.index('--tokens')+1])",
|
|
74
|
+
"audio24, sr = sf.read(audio, dtype='float32')",
|
|
75
|
+
"if audio24.ndim > 1:",
|
|
76
|
+
" audio24 = audio24.mean(axis=1).astype(np.float32)",
|
|
77
|
+
"audio16 = resample16(audio24, sr)",
|
|
78
|
+
"surfaces = list(tokens)",
|
|
79
|
+
"cleaned = [clean_word(w) for w in surfaces]",
|
|
80
|
+
"for surf, c in zip(surfaces, cleaned):",
|
|
81
|
+
" if not c:",
|
|
82
|
+
" sys.stderr.write(f'token {surf!r} has no alignable chars (clean_word empty); refusing to fake timing\\n')",
|
|
83
|
+
" sys.exit(1)",
|
|
84
|
+
"processor = Wav2Vec2Processor.from_pretrained(MODEL_ID)",
|
|
85
|
+
"model = Wav2Vec2ForCTC.from_pretrained(MODEL_ID).eval()",
|
|
86
|
+
"with torch.no_grad():",
|
|
87
|
+
" inputs = processor(audio16, sampling_rate=16000, return_tensors='pt', padding=True)",
|
|
88
|
+
" logits = model(inputs.input_values).logits[0].cpu().float().numpy()",
|
|
89
|
+
"log_probs = torch.from_numpy(logits).log_softmax(dim=-1).numpy()",
|
|
90
|
+
"vocab = processor.tokenizer.get_vocab()",
|
|
91
|
+
"blank = processor.tokenizer.pad_token_id",
|
|
92
|
+
"text = '|'.join(cleaned)",
|
|
93
|
+
"token_ids = [vocab[ch] for ch in text]",
|
|
94
|
+
"aug = aug_tokens(token_ids, blank)",
|
|
95
|
+
"tr = trellis_forward(log_probs, aug, blank)",
|
|
96
|
+
"if not np.isfinite(tr[-1, -1]):",
|
|
97
|
+
" sys.stderr.write('trellis failed: transcript incompatible with audio\\n')",
|
|
98
|
+
" sys.exit(1)",
|
|
99
|
+
"ranges = []",
|
|
100
|
+
"pos = 1",
|
|
101
|
+
"for i, w in enumerate(cleaned):",
|
|
102
|
+
" a0 = pos",
|
|
103
|
+
" ranges.append((a0, a0 + 2 * len(w) - 1))",
|
|
104
|
+
" pos += 2 * len(w)",
|
|
105
|
+
" if i < len(cleaned) - 1:",
|
|
106
|
+
" pos += 2",
|
|
107
|
+
"spans = backtrack_word_spans(tr, log_probs, aug, blank, ranges)",
|
|
108
|
+
"dur = len(audio24) / float(sr)",
|
|
109
|
+
"words = []",
|
|
110
|
+
"for surf, (f0, f1) in zip(surfaces, spans):",
|
|
111
|
+
" start = round(min(max(f0 * FRAME_STRIDE_S, 0.0), dur), 3)",
|
|
112
|
+
" end = round(min(max(f1 * FRAME_STRIDE_S, 0.0), dur), 3)",
|
|
113
|
+
" if end < start:",
|
|
114
|
+
" end = start",
|
|
115
|
+
" words.append({'w': surf, 'start': start, 'end': end})",
|
|
116
|
+
"print(json.dumps({'words': words}))",
|
|
117
|
+
].join("\n");
|
|
118
|
+
const isWord = (value) => {
|
|
119
|
+
if (typeof value !== "object" || value === null)
|
|
120
|
+
return false;
|
|
121
|
+
const w = value.w;
|
|
122
|
+
const start = value.start;
|
|
123
|
+
const end = value.end;
|
|
124
|
+
return (typeof w === "string" &&
|
|
125
|
+
w.length > 0 &&
|
|
126
|
+
typeof start === "number" &&
|
|
127
|
+
Number.isFinite(start) &&
|
|
128
|
+
typeof end === "number" &&
|
|
129
|
+
Number.isFinite(end) &&
|
|
130
|
+
start >= 0 &&
|
|
131
|
+
end >= start);
|
|
132
|
+
};
|
|
133
|
+
/** Validate aligner stdout; count must match the known tokens exactly. */
|
|
134
|
+
const parseAlignPayload = (stdout, tokenCount) => {
|
|
135
|
+
let payload;
|
|
136
|
+
try {
|
|
137
|
+
payload = JSON.parse(stdout);
|
|
138
|
+
}
|
|
139
|
+
catch {
|
|
140
|
+
return { ok: false, status: "unknown", hint: "aligner gave unparsable output" };
|
|
141
|
+
}
|
|
142
|
+
if (!Array.isArray(payload.words) || !payload.words.every(isWord)) {
|
|
143
|
+
return { ok: false, status: "unknown", hint: "aligner gave malformed word boundaries" };
|
|
144
|
+
}
|
|
145
|
+
if (payload.words.length !== tokenCount) {
|
|
146
|
+
return {
|
|
147
|
+
ok: false,
|
|
148
|
+
status: "unknown",
|
|
149
|
+
hint: `aligner returned ${payload.words.length} words for ${tokenCount} tokens`,
|
|
150
|
+
};
|
|
151
|
+
}
|
|
152
|
+
return { ok: true, words: payload.words };
|
|
153
|
+
};
|
|
154
|
+
/** Align known tokens to audio. Fails closed on any malformed output. */
|
|
155
|
+
export async function alignWords(run, req) {
|
|
156
|
+
if (req.tokens.length === 0) {
|
|
157
|
+
return { ok: false, status: "unknown", hint: "align needs at least one known token" };
|
|
158
|
+
}
|
|
159
|
+
const bad = req.tokens.find((t) => !isAsciiToken(t));
|
|
160
|
+
if (bad !== undefined) {
|
|
161
|
+
return {
|
|
162
|
+
ok: false,
|
|
163
|
+
status: "unknown",
|
|
164
|
+
hint: `EN-only wav2vec2 aligner (${ALIGN_MODEL_ID}) cannot align non-ASCII token "${bad.slice(0, 24)}"; refusing Japanese/non-English narration. Run \`mk doctor\` for the voice setup steps.`,
|
|
165
|
+
};
|
|
166
|
+
}
|
|
167
|
+
let python = req.python;
|
|
168
|
+
if (python === undefined) {
|
|
169
|
+
const probe = await resolveVenvPython(run);
|
|
170
|
+
if (!probe.ok)
|
|
171
|
+
return probe;
|
|
172
|
+
python = probe.python;
|
|
173
|
+
}
|
|
174
|
+
const res = await run(python, [
|
|
175
|
+
"-c",
|
|
176
|
+
ALIGN_PY,
|
|
177
|
+
"--audio",
|
|
178
|
+
req.audioPath,
|
|
179
|
+
"--tokens",
|
|
180
|
+
JSON.stringify(req.tokens),
|
|
181
|
+
]);
|
|
182
|
+
if (res.code === 127) {
|
|
183
|
+
return { ok: false, status: "missing", hint: `${python} is not runnable` };
|
|
184
|
+
}
|
|
185
|
+
if (res.code !== 0) {
|
|
186
|
+
return { ok: false, status: "unknown", hint: res.stderr.slice(-500) || "align failed" };
|
|
187
|
+
}
|
|
188
|
+
return parseAlignPayload(res.stdout, req.tokens.length);
|
|
189
|
+
}
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* No-venv fallback: proportional split labeled `source: "authored"`.
|
|
3
|
+
*
|
|
4
|
+
* Voice is opt-in — when the venv is missing the existing render path
|
|
5
|
+
* (autoDuration) must stay untouched. This fallback NEVER claims
|
|
6
|
+
* `source: "voice"`: it distributes a known total proportionally to word
|
|
7
|
+
* counts and labels the result `authored`, so downstream consumers can tell
|
|
8
|
+
* measured voice timing apart from an estimate. Fail-closed: non-positive
|
|
9
|
+
* totals throw instead of emitting fake timings.
|
|
10
|
+
*/
|
|
11
|
+
import type { TimingManifest } from "@anvpx/motion-schema";
|
|
12
|
+
export interface FallbackSceneInput {
|
|
13
|
+
id: string;
|
|
14
|
+
text: string;
|
|
15
|
+
}
|
|
16
|
+
/** Split a total into per-weight shares; the last share absorbs rounding drift. */
|
|
17
|
+
export declare function splitDurations(weights: readonly number[], total: number): number[];
|
|
18
|
+
/**
|
|
19
|
+
* Proportional `authored` manifest for the no-venv path. Word counts are the
|
|
20
|
+
* split weights; exact float cumulative starts (mirrors buildVoiceManifest).
|
|
21
|
+
*/
|
|
22
|
+
export declare function fallbackManifest(scenes: readonly FallbackSceneInput[], totalDurationS: number): TimingManifest;
|
package/dist/fallback.js
ADDED
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
const round3 = (value) => Math.round(value * 1000) / 1000;
|
|
2
|
+
const wordsOf = (text) => text.split(/\s+/).filter((w) => w.length > 0).length;
|
|
3
|
+
/** Split a total into per-weight shares; the last share absorbs rounding drift. */
|
|
4
|
+
export function splitDurations(weights, total) {
|
|
5
|
+
if (!(total > 0) || !Number.isFinite(total)) {
|
|
6
|
+
throw new Error("fallback split needs a positive finite total");
|
|
7
|
+
}
|
|
8
|
+
const sum = weights.reduce((acc, w) => acc + w, 0);
|
|
9
|
+
if (!(sum > 0))
|
|
10
|
+
throw new Error("fallback split needs positive weights");
|
|
11
|
+
const parts = weights.map((w) => round3((w / sum) * total));
|
|
12
|
+
const drift = round3(total - parts.reduce((acc, p) => acc + p, 0));
|
|
13
|
+
parts[parts.length - 1] = round3(parts[parts.length - 1] + drift);
|
|
14
|
+
return parts;
|
|
15
|
+
}
|
|
16
|
+
/**
|
|
17
|
+
* Proportional `authored` manifest for the no-venv path. Word counts are the
|
|
18
|
+
* split weights; exact float cumulative starts (mirrors buildVoiceManifest).
|
|
19
|
+
*/
|
|
20
|
+
export function fallbackManifest(scenes, totalDurationS) {
|
|
21
|
+
const durations = splitDurations(scenes.map((s) => Math.max(wordsOf(s.text), 1)), totalDurationS);
|
|
22
|
+
let cursor = 0;
|
|
23
|
+
const rows = scenes.map((scene, index) => {
|
|
24
|
+
const row = { id: scene.id, start: cursor, duration: durations[index] };
|
|
25
|
+
cursor = row.start + row.duration;
|
|
26
|
+
return row;
|
|
27
|
+
});
|
|
28
|
+
return { source: "authored", scenes: rows };
|
|
29
|
+
}
|
package/dist/index.d.ts
ADDED
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @anvpx/motion-audio — voice port (opt-in).
|
|
3
|
+
*
|
|
4
|
+
* Standalone adapter driving a venv Python for Kokoro TTS plus wav2vec2
|
|
5
|
+
* align-only, consumed by build/CLI via structural {@link Runner}. Imports
|
|
6
|
+
* schema types only (constitution 2.1). Zero npm deps beyond
|
|
7
|
+
* `@anvpx/motion-schema`. No whisperx/pyannote; diarization is always OFF.
|
|
8
|
+
*/
|
|
9
|
+
export type CommandResult = {
|
|
10
|
+
code: number;
|
|
11
|
+
stdout: string;
|
|
12
|
+
stderr: string;
|
|
13
|
+
};
|
|
14
|
+
export type RunOptions = {
|
|
15
|
+
cwd?: string;
|
|
16
|
+
env?: Readonly<Record<string, string>>;
|
|
17
|
+
timeoutMs?: number;
|
|
18
|
+
};
|
|
19
|
+
/** Runs an external program. Never throws: missing programs yield 127. */
|
|
20
|
+
export type Runner = (cmd: string, args: readonly string[], options?: RunOptions) => Promise<CommandResult>;
|
|
21
|
+
/** Voice stack provenance (spike Stage 2 PASS receipts). */
|
|
22
|
+
export declare const VOICE_PROVENANCE: {
|
|
23
|
+
readonly kokoro: "0.9.4";
|
|
24
|
+
readonly fasterWhisper: "1.2.1";
|
|
25
|
+
readonly alignModel: "facebook/wav2vec2-base-960h";
|
|
26
|
+
readonly alignModelRev: "22aad52d";
|
|
27
|
+
readonly diarization: "OFF";
|
|
28
|
+
};
|
|
29
|
+
export type { AlignRequest, AlignResult, Word } from "./align.ts";
|
|
30
|
+
export { ALIGN_MODEL_ID, alignWords } from "./align.ts";
|
|
31
|
+
export type { FallbackSceneInput } from "./fallback.ts";
|
|
32
|
+
export { fallbackManifest, splitDurations } from "./fallback.ts";
|
|
33
|
+
export type { SynthRequest, SynthResult, VenvProbe } from "./kokoro.ts";
|
|
34
|
+
export { isJapaneseVoice, KOKORO_VERSION_PIN, resolveVenvPython, synthesize } from "./kokoro.ts";
|
|
35
|
+
export type { MusicBedInput, MusicBedResult, ValidMusicBed } from "./music.ts";
|
|
36
|
+
export { DEFAULT_MUSIC_VOLUME, MUSIC_LICENCE, validateMusicBed } from "./music.ts";
|
|
37
|
+
export type { SilenceRequest, SilenceResult } from "./silence.ts";
|
|
38
|
+
export { detectSilence, SILENCE_MIN_D, SILENCE_NOISE_N, wordsInSilence, } from "./silence.ts";
|
|
39
|
+
export type { VoiceManifestResult, VoiceSceneInput } from "./timing.ts";
|
|
40
|
+
export { buildVoiceManifest, sceneDurationFromAudio } from "./timing.ts";
|
|
41
|
+
export type { Generate3dDeps, Generate3dRequest, Generate3dResult } from "./triposr.ts";
|
|
42
|
+
export { generate3d, TRIPOSR_MISSING_HINT, TRIPOSR_MODEL_ID, TRIPOSR_MODEL_REV, TRIPOSR_PROVENANCE, TRIPOSR_VENDOR_REV, TRIPOSR_WEIGHT_BYTES, } from "./triposr.ts";
|
package/dist/index.js
ADDED
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @anvpx/motion-audio — voice port (opt-in).
|
|
3
|
+
*
|
|
4
|
+
* Standalone adapter driving a venv Python for Kokoro TTS plus wav2vec2
|
|
5
|
+
* align-only, consumed by build/CLI via structural {@link Runner}. Imports
|
|
6
|
+
* schema types only (constitution 2.1). Zero npm deps beyond
|
|
7
|
+
* `@anvpx/motion-schema`. No whisperx/pyannote; diarization is always OFF.
|
|
8
|
+
*/
|
|
9
|
+
/** Voice stack provenance (spike Stage 2 PASS receipts). */
|
|
10
|
+
export const VOICE_PROVENANCE = {
|
|
11
|
+
kokoro: "0.9.4",
|
|
12
|
+
fasterWhisper: "1.2.1",
|
|
13
|
+
alignModel: "facebook/wav2vec2-base-960h",
|
|
14
|
+
alignModelRev: "22aad52d",
|
|
15
|
+
diarization: "OFF",
|
|
16
|
+
};
|
|
17
|
+
export { ALIGN_MODEL_ID, alignWords } from "./align.js";
|
|
18
|
+
export { fallbackManifest, splitDurations } from "./fallback.js";
|
|
19
|
+
export { isJapaneseVoice, KOKORO_VERSION_PIN, resolveVenvPython, synthesize } from "./kokoro.js";
|
|
20
|
+
export { DEFAULT_MUSIC_VOLUME, MUSIC_LICENCE, validateMusicBed } from "./music.js";
|
|
21
|
+
export { detectSilence, SILENCE_MIN_D, SILENCE_NOISE_N, wordsInSilence, } from "./silence.js";
|
|
22
|
+
export { buildVoiceManifest, sceneDurationFromAudio } from "./timing.js";
|
|
23
|
+
export { generate3d, TRIPOSR_MISSING_HINT, TRIPOSR_MODEL_ID, TRIPOSR_MODEL_REV, TRIPOSR_PROVENANCE, TRIPOSR_VENDOR_REV, TRIPOSR_WEIGHT_BYTES, } from "./triposr.js";
|
package/dist/kokoro.d.ts
ADDED
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Kokoro TTS via the voice venv python (opt-in).
|
|
3
|
+
*
|
|
4
|
+
* Shells out with execFile semantics (no shell) through the injected
|
|
5
|
+
* {@link Runner}. Missing venv fails closed as `missing` — the caller keeps
|
|
6
|
+
* the existing render path untouched.
|
|
7
|
+
*/
|
|
8
|
+
import type { Runner } from "./index.ts";
|
|
9
|
+
/** Pinned TTS version (spike Stage 1/2 receipt). */
|
|
10
|
+
export declare const KOKORO_VERSION_PIN = "0.9.4";
|
|
11
|
+
/** Venv python candidates, windows-first (mirrors `mk doctor` probes). */
|
|
12
|
+
export declare const VENV_PYS: readonly [".venv-voice/Scripts/python", ".venv-voice/bin/python"];
|
|
13
|
+
/** Max narration chars per synth call (bounds TTS cost/latency; fail closed above). */
|
|
14
|
+
export declare const MAX_SYNTH_TEXT_CHARS = 6000;
|
|
15
|
+
/** Voice ids are adapter slugs (no paths, no shell); mirrors storyboard SLUG. */
|
|
16
|
+
export declare const VOICE_ID_RE: RegExp;
|
|
17
|
+
/** `j*` voices select the Kokoro Japanese pipeline; they are unaligned-only. */
|
|
18
|
+
export declare const isJapaneseVoice: (voice: string) => boolean;
|
|
19
|
+
export type VenvProbe = {
|
|
20
|
+
ok: true;
|
|
21
|
+
python: string;
|
|
22
|
+
} | {
|
|
23
|
+
ok: false;
|
|
24
|
+
status: "missing" | "unknown";
|
|
25
|
+
hint: string;
|
|
26
|
+
};
|
|
27
|
+
export interface SynthRequest {
|
|
28
|
+
text: string;
|
|
29
|
+
voice: string;
|
|
30
|
+
outPath: string;
|
|
31
|
+
/** Explicit python binary; skips the venv probe when set. */
|
|
32
|
+
python?: string;
|
|
33
|
+
}
|
|
34
|
+
export type SynthResult = {
|
|
35
|
+
ok: true;
|
|
36
|
+
outPath: string;
|
|
37
|
+
durationS: number;
|
|
38
|
+
} | {
|
|
39
|
+
ok: false;
|
|
40
|
+
status: "missing" | "unknown";
|
|
41
|
+
hint: string;
|
|
42
|
+
};
|
|
43
|
+
/**
|
|
44
|
+
* Inline synth driver: Kokoro pipeline writes the wav, prints one JSON line.
|
|
45
|
+
* Voice prefix selects the pipeline language (`j*` voices are Japanese,
|
|
46
|
+
* matching the spike's `jf_alpha`/`jm_kumo` clips); diarization N/A (TTS).
|
|
47
|
+
*/
|
|
48
|
+
export declare const SYNTH_PY: string;
|
|
49
|
+
/** Probe the voice venv pythons, windows-first. Never throws. */
|
|
50
|
+
export declare function resolveVenvPython(run: Runner): Promise<VenvProbe>;
|
|
51
|
+
/** Synthesize text with Kokoro. Fails closed when the venv is absent. */
|
|
52
|
+
export declare function synthesize(run: Runner, req: SynthRequest): Promise<SynthResult>;
|
package/dist/kokoro.js
ADDED
|
@@ -0,0 +1,156 @@
|
|
|
1
|
+
/** Pinned TTS version (spike Stage 1/2 receipt). */
|
|
2
|
+
export const KOKORO_VERSION_PIN = "0.9.4";
|
|
3
|
+
/** Control chars (except tab/LF/CR) have no honest TTS rendering; fail closed. */
|
|
4
|
+
const hasControlChars = (text) => {
|
|
5
|
+
for (let i = 0; i < text.length; i += 1) {
|
|
6
|
+
const code = text.charCodeAt(i);
|
|
7
|
+
if (code === 9 || code === 10 || code === 13)
|
|
8
|
+
continue;
|
|
9
|
+
if (code < 32 || code === 127)
|
|
10
|
+
return true;
|
|
11
|
+
}
|
|
12
|
+
return false;
|
|
13
|
+
};
|
|
14
|
+
/** Venv python candidates, windows-first (mirrors `mk doctor` probes). */
|
|
15
|
+
export const VENV_PYS = [".venv-voice/Scripts/python", ".venv-voice/bin/python"];
|
|
16
|
+
/** Max narration chars per synth call (bounds TTS cost/latency; fail closed above). */
|
|
17
|
+
export const MAX_SYNTH_TEXT_CHARS = 6000;
|
|
18
|
+
/** Voice ids are adapter slugs (no paths, no shell); mirrors storyboard SLUG. */
|
|
19
|
+
export const VOICE_ID_RE = /^[A-Za-z0-9][A-Za-z0-9_-]*$/;
|
|
20
|
+
/** `j*` voices select the Kokoro Japanese pipeline; they are unaligned-only. */
|
|
21
|
+
export const isJapaneseVoice = (voice) => voice.startsWith("j");
|
|
22
|
+
/**
|
|
23
|
+
* Inline synth driver: Kokoro pipeline writes the wav, prints one JSON line.
|
|
24
|
+
* Voice prefix selects the pipeline language (`j*` voices are Japanese,
|
|
25
|
+
* matching the spike's `jf_alpha`/`jm_kumo` clips); diarization N/A (TTS).
|
|
26
|
+
*/
|
|
27
|
+
export const SYNTH_PY = [
|
|
28
|
+
"import json, sys",
|
|
29
|
+
"import numpy as np",
|
|
30
|
+
"import soundfile as sf",
|
|
31
|
+
"from kokoro import KPipeline",
|
|
32
|
+
"voice = sys.argv[sys.argv.index('--voice')+1]",
|
|
33
|
+
"out = sys.argv[sys.argv.index('--out')+1]",
|
|
34
|
+
"text = sys.argv[sys.argv.index('--text')+1]",
|
|
35
|
+
"pipe = KPipeline(lang_code=('j' if voice.startswith('j') else 'a'), repo_id='hexgrad/Kokoro-82M')",
|
|
36
|
+
"chunks = []",
|
|
37
|
+
"for _, _, audio in pipe(text, voice=voice): chunks.append(audio.numpy() if hasattr(audio, 'numpy') else np.asarray(audio))",
|
|
38
|
+
"wav = np.concatenate(chunks).astype(np.float32)",
|
|
39
|
+
"sf.write(out, wav, 24000)",
|
|
40
|
+
"print(json.dumps({'out': out, 'duration_s': round(float(len(wav))/24000, 3)}))",
|
|
41
|
+
].join("\n");
|
|
42
|
+
const isFiniteNumber = (value) => typeof value === "number" && Number.isFinite(value);
|
|
43
|
+
/** Probe the voice venv pythons, windows-first. Never throws. */
|
|
44
|
+
export async function resolveVenvPython(run) {
|
|
45
|
+
let sawVenv = false;
|
|
46
|
+
for (const cmd of VENV_PYS) {
|
|
47
|
+
const res = await run(cmd, ["--version"]);
|
|
48
|
+
if (res.code !== 0)
|
|
49
|
+
continue;
|
|
50
|
+
sawVenv = true;
|
|
51
|
+
if (/3\.12\.\d+/.test(`${res.stdout}${res.stderr}`)) {
|
|
52
|
+
return { ok: true, python: cmd };
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
if (sawVenv) {
|
|
56
|
+
return {
|
|
57
|
+
ok: false,
|
|
58
|
+
status: "unknown",
|
|
59
|
+
hint: "venv python is not 3.12; recreate with py -3.12 -m venv .venv-voice",
|
|
60
|
+
};
|
|
61
|
+
}
|
|
62
|
+
return {
|
|
63
|
+
ok: false,
|
|
64
|
+
status: "missing",
|
|
65
|
+
hint: "create the voice venv first: py -3.12 -m venv .venv-voice (voice phase only)",
|
|
66
|
+
};
|
|
67
|
+
}
|
|
68
|
+
const synthArgs = (req) => [
|
|
69
|
+
"-c",
|
|
70
|
+
SYNTH_PY,
|
|
71
|
+
"--voice",
|
|
72
|
+
req.voice,
|
|
73
|
+
"--out",
|
|
74
|
+
req.outPath,
|
|
75
|
+
"--text",
|
|
76
|
+
req.text,
|
|
77
|
+
];
|
|
78
|
+
/** Validate synth stdout JSON; tolerates leading stdout noise (e.g. Kokoro
|
|
79
|
+
* repo_id warning) by taking the last line that parses as the expected shape.
|
|
80
|
+
* Never invents a duration. */
|
|
81
|
+
const parseSynthPayload = (stdout, expectedOut) => {
|
|
82
|
+
const lines = stdout
|
|
83
|
+
.split(/\r?\n/)
|
|
84
|
+
.map((line) => line.trim())
|
|
85
|
+
.filter((line) => line.length > 0);
|
|
86
|
+
let sawMalformedJson = false;
|
|
87
|
+
for (let i = lines.length - 1; i >= 0; i -= 1) {
|
|
88
|
+
const raw = lines[i];
|
|
89
|
+
if (raw === undefined)
|
|
90
|
+
continue;
|
|
91
|
+
let payload;
|
|
92
|
+
try {
|
|
93
|
+
payload = JSON.parse(raw);
|
|
94
|
+
}
|
|
95
|
+
catch {
|
|
96
|
+
continue;
|
|
97
|
+
}
|
|
98
|
+
if (typeof payload.out !== "string" || !isFiniteNumber(payload.duration_s)) {
|
|
99
|
+
sawMalformedJson = true;
|
|
100
|
+
continue;
|
|
101
|
+
}
|
|
102
|
+
if (payload.out !== expectedOut) {
|
|
103
|
+
sawMalformedJson = true;
|
|
104
|
+
continue;
|
|
105
|
+
}
|
|
106
|
+
if (!(payload.duration_s > 0)) {
|
|
107
|
+
return { ok: false, status: "unknown", hint: "kokoro synth reported no audio" };
|
|
108
|
+
}
|
|
109
|
+
return { ok: true, outPath: payload.out, durationS: payload.duration_s };
|
|
110
|
+
}
|
|
111
|
+
if (sawMalformedJson) {
|
|
112
|
+
return { ok: false, status: "unknown", hint: "kokoro synth gave malformed JSON" };
|
|
113
|
+
}
|
|
114
|
+
return { ok: false, status: "unknown", hint: "kokoro synth gave unparsable output" };
|
|
115
|
+
};
|
|
116
|
+
/** Explicit binary wins; otherwise probe the voice venv (windows-first). */
|
|
117
|
+
const pickPython = async (run, explicit) => {
|
|
118
|
+
if (explicit !== undefined)
|
|
119
|
+
return { python: explicit };
|
|
120
|
+
const probe = await resolveVenvPython(run);
|
|
121
|
+
return probe.ok ? { python: probe.python } : probe;
|
|
122
|
+
};
|
|
123
|
+
/** Synthesize text with Kokoro. Fails closed when the venv is absent. */
|
|
124
|
+
export async function synthesize(run, req) {
|
|
125
|
+
if (req.text.trim().length === 0) {
|
|
126
|
+
return { ok: false, status: "unknown", hint: "refusing to synthesize empty text" };
|
|
127
|
+
}
|
|
128
|
+
if (!VOICE_ID_RE.test(req.voice)) {
|
|
129
|
+
return {
|
|
130
|
+
ok: false,
|
|
131
|
+
status: "unknown",
|
|
132
|
+
hint: `refusing voice id "${req.voice.slice(0, 32)}": must match ${VOICE_ID_RE.source}`,
|
|
133
|
+
};
|
|
134
|
+
}
|
|
135
|
+
if (req.text.length > MAX_SYNTH_TEXT_CHARS) {
|
|
136
|
+
return {
|
|
137
|
+
ok: false,
|
|
138
|
+
status: "unknown",
|
|
139
|
+
hint: `refusing to synthesize ${req.text.length} chars (cap ${MAX_SYNTH_TEXT_CHARS})`,
|
|
140
|
+
};
|
|
141
|
+
}
|
|
142
|
+
if (hasControlChars(req.text)) {
|
|
143
|
+
return { ok: false, status: "unknown", hint: "refusing text with control characters" };
|
|
144
|
+
}
|
|
145
|
+
const picked = await pickPython(run, req.python);
|
|
146
|
+
if (!("python" in picked))
|
|
147
|
+
return picked;
|
|
148
|
+
const res = await run(picked.python, synthArgs(req));
|
|
149
|
+
if (res.code === 127) {
|
|
150
|
+
return { ok: false, status: "missing", hint: `${picked.python} is not runnable` };
|
|
151
|
+
}
|
|
152
|
+
if (res.code !== 0) {
|
|
153
|
+
return { ok: false, status: "unknown", hint: res.stderr.slice(-500) || "kokoro synth failed" };
|
|
154
|
+
}
|
|
155
|
+
return parseSynthPayload(res.stdout, req.outPath);
|
|
156
|
+
}
|
package/dist/music.d.ts
ADDED
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* User-supplied CC0 music bed (opt-in, explicit flag only).
|
|
3
|
+
*
|
|
4
|
+
* The engine never bundles copyrighted music: a bed is mixed only when the
|
|
5
|
+
* caller passes an explicit `{ wav }` bed pointing at a user CC0 file (or
|
|
6
|
+
* the repo's procedural `packages/assets/music/bed.wav`). Absent means
|
|
7
|
+
* silent/narration-only — never a synthetic fallback. Missing files fail
|
|
8
|
+
* closed via the mix layer's existence check.
|
|
9
|
+
*
|
|
10
|
+
* Guard rules live in `@anvpx/motion-schema` (`MusicBedSchema`), the single
|
|
11
|
+
* source shared with the build mix layer; this module maps schema failures
|
|
12
|
+
* to the CLI-facing hints below.
|
|
13
|
+
*/
|
|
14
|
+
/** Ducked bed level, re-exported from the schema single source. */
|
|
15
|
+
export declare const DEFAULT_MUSIC_VOLUME: number;
|
|
16
|
+
/** Only CC0 beds are supported until MK-013 (gated audio). */
|
|
17
|
+
export declare const MUSIC_LICENCE = "CC0-1.0";
|
|
18
|
+
export interface MusicBedInput {
|
|
19
|
+
readonly wav: string;
|
|
20
|
+
readonly volume?: number;
|
|
21
|
+
}
|
|
22
|
+
export interface ValidMusicBed {
|
|
23
|
+
readonly wav: string;
|
|
24
|
+
readonly volume: number;
|
|
25
|
+
}
|
|
26
|
+
export type MusicBedResult = {
|
|
27
|
+
ok: true;
|
|
28
|
+
bed: ValidMusicBed;
|
|
29
|
+
} | {
|
|
30
|
+
ok: false;
|
|
31
|
+
hint: string;
|
|
32
|
+
};
|
|
33
|
+
/** Validate an explicit user bed; never invents a path or level. */
|
|
34
|
+
export declare function validateMusicBed(value: unknown): MusicBedResult;
|
package/dist/music.js
ADDED
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
import { parseMusicBed as parseSchemaMusicBed, DEFAULT_MUSIC_VOLUME as SCHEMA_DEFAULT_MUSIC_VOLUME, } from "@anvpx/motion-schema";
|
|
2
|
+
/**
|
|
3
|
+
* User-supplied CC0 music bed (opt-in, explicit flag only).
|
|
4
|
+
*
|
|
5
|
+
* The engine never bundles copyrighted music: a bed is mixed only when the
|
|
6
|
+
* caller passes an explicit `{ wav }` bed pointing at a user CC0 file (or
|
|
7
|
+
* the repo's procedural `packages/assets/music/bed.wav`). Absent means
|
|
8
|
+
* silent/narration-only — never a synthetic fallback. Missing files fail
|
|
9
|
+
* closed via the mix layer's existence check.
|
|
10
|
+
*
|
|
11
|
+
* Guard rules live in `@anvpx/motion-schema` (`MusicBedSchema`), the single
|
|
12
|
+
* source shared with the build mix layer; this module maps schema failures
|
|
13
|
+
* to the CLI-facing hints below.
|
|
14
|
+
*/
|
|
15
|
+
/** Ducked bed level, re-exported from the schema single source. */
|
|
16
|
+
export const DEFAULT_MUSIC_VOLUME = SCHEMA_DEFAULT_MUSIC_VOLUME;
|
|
17
|
+
/** Only CC0 beds are supported until MK-013 (gated audio). */
|
|
18
|
+
export const MUSIC_LICENCE = "CC0-1.0";
|
|
19
|
+
const isFiniteVolume = (value) => typeof value === "number" && Number.isFinite(value) && value > 0 && value <= 1;
|
|
20
|
+
/** Validate an explicit user bed; never invents a path or level. */
|
|
21
|
+
export function validateMusicBed(value) {
|
|
22
|
+
const parsed = parseSchemaMusicBed(value);
|
|
23
|
+
if (parsed.ok)
|
|
24
|
+
return { ok: true, bed: parsed.bed };
|
|
25
|
+
if (typeof value !== "object" || value === null) {
|
|
26
|
+
return { ok: false, hint: "music bed needs an explicit { wav } (user CC0 .wav)" };
|
|
27
|
+
}
|
|
28
|
+
const record = value;
|
|
29
|
+
const wav = record.wav;
|
|
30
|
+
if (typeof wav !== "string" || wav.length === 0) {
|
|
31
|
+
return { ok: false, hint: "music bed needs an explicit { wav } (user CC0 .wav)" };
|
|
32
|
+
}
|
|
33
|
+
if (!wav.endsWith(".wav") || wav.includes("..")) {
|
|
34
|
+
return {
|
|
35
|
+
ok: false,
|
|
36
|
+
hint: `refusing music bed "${wav.slice(0, 48)}": must be a local .wav without traversal`,
|
|
37
|
+
};
|
|
38
|
+
}
|
|
39
|
+
const raw = record.volume;
|
|
40
|
+
if (raw !== undefined && !isFiniteVolume(raw)) {
|
|
41
|
+
return { ok: false, hint: "music bed volume must be within (0, 1]" };
|
|
42
|
+
}
|
|
43
|
+
return { ok: false, hint: parsed.hint };
|
|
44
|
+
}
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* ffprobe silencedetect gate over voice audio (non-speech guard).
|
|
3
|
+
*
|
|
4
|
+
* Mirrors the spike's `silence_intervals`: ffprobe takes LINEAR amplitude
|
|
5
|
+
* (not dB), so `0.003` is the Kokoro noise floor (~-50 dBFS). A word fully
|
|
6
|
+
* inside a detected silence interval is flagged — surfacing it keeps the
|
|
7
|
+
* gate honest instead of silently passing a vacuous check.
|
|
8
|
+
*/
|
|
9
|
+
import type { Word } from "./align.ts";
|
|
10
|
+
import type { Runner } from "./index.ts";
|
|
11
|
+
/** Linear amplitude threshold (NOT dB) — spike receipt. */
|
|
12
|
+
export declare const SILENCE_NOISE_N = 0.003;
|
|
13
|
+
/** Minimum silence duration, seconds — spike receipt. */
|
|
14
|
+
export declare const SILENCE_MIN_D = 0.2;
|
|
15
|
+
export interface SilenceRequest {
|
|
16
|
+
audioPath: string;
|
|
17
|
+
}
|
|
18
|
+
export type SilenceResult = {
|
|
19
|
+
ok: true;
|
|
20
|
+
intervals: Array<readonly [number, number]>;
|
|
21
|
+
} | {
|
|
22
|
+
ok: false;
|
|
23
|
+
status: "missing" | "unknown";
|
|
24
|
+
hint: string;
|
|
25
|
+
};
|
|
26
|
+
/** Escape a path for the lavfi `amovie='…'` single-quoted string (no breakout). */
|
|
27
|
+
export declare const escapeAmoviePath: (path: string) => string;
|
|
28
|
+
/** Detect non-speech intervals. Fails closed when ffprobe is absent. */
|
|
29
|
+
export declare function detectSilence(run: Runner, req: SilenceRequest): Promise<SilenceResult>;
|
|
30
|
+
/** Surface words fully enclosed in a silence interval (non-speech gate), with their timing. */
|
|
31
|
+
export declare function wordsInSilence(words: readonly Pick<Word, "w" | "start" | "end">[], intervals: readonly (readonly [number, number])[]): Array<Pick<Word, "w" | "start" | "end">>;
|
package/dist/silence.js
ADDED
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* ffprobe silencedetect gate over voice audio (non-speech guard).
|
|
3
|
+
*
|
|
4
|
+
* Mirrors the spike's `silence_intervals`: ffprobe takes LINEAR amplitude
|
|
5
|
+
* (not dB), so `0.003` is the Kokoro noise floor (~-50 dBFS). A word fully
|
|
6
|
+
* inside a detected silence interval is flagged — surfacing it keeps the
|
|
7
|
+
* gate honest instead of silently passing a vacuous check.
|
|
8
|
+
*/
|
|
9
|
+
/** Linear amplitude threshold (NOT dB) — spike receipt. */
|
|
10
|
+
export const SILENCE_NOISE_N = 0.003;
|
|
11
|
+
/** Minimum silence duration, seconds — spike receipt. */
|
|
12
|
+
export const SILENCE_MIN_D = 0.2;
|
|
13
|
+
const START_RE = /silence_start:\s*([\d.]+)/;
|
|
14
|
+
const END_RE = /silence_end:\s*([\d.]+)/;
|
|
15
|
+
/** Fold one parsed line; returns a hint when the line is incoherent. */
|
|
16
|
+
const foldLine = (line, starts, intervals) => {
|
|
17
|
+
const s = START_RE.exec(line);
|
|
18
|
+
if (s) {
|
|
19
|
+
starts.push(Number(s[1]));
|
|
20
|
+
return undefined;
|
|
21
|
+
}
|
|
22
|
+
const e = END_RE.exec(line);
|
|
23
|
+
if (e !== null && starts.length > 0) {
|
|
24
|
+
const s0 = starts.pop();
|
|
25
|
+
const s1 = Number(e[1]);
|
|
26
|
+
if (!(s1 > s0))
|
|
27
|
+
return "silencedetect gave inverted interval";
|
|
28
|
+
intervals.push([s0, s1]);
|
|
29
|
+
}
|
|
30
|
+
return undefined;
|
|
31
|
+
};
|
|
32
|
+
/** Parse silencedetect stderr; unterminated starts fail closed (no vacuous pass). */
|
|
33
|
+
const parseSilenceStderr = (stderr) => {
|
|
34
|
+
const intervals = [];
|
|
35
|
+
const starts = [];
|
|
36
|
+
for (const line of stderr.split("\n")) {
|
|
37
|
+
const hint = foldLine(line, starts, intervals);
|
|
38
|
+
if (hint !== undefined)
|
|
39
|
+
return { ok: false, status: "unknown", hint };
|
|
40
|
+
}
|
|
41
|
+
if (starts.length > 0) {
|
|
42
|
+
return { ok: false, status: "unknown", hint: "silencedetect left an interval unterminated" };
|
|
43
|
+
}
|
|
44
|
+
return { ok: true, intervals };
|
|
45
|
+
};
|
|
46
|
+
/** Escape a path for the lavfi `amovie='…'` single-quoted string (no breakout). */
|
|
47
|
+
export const escapeAmoviePath = (path) => path.replace(/\\/g, "\\\\").replace(/'/g, "\\'").replace(/:/g, "\\:").replace(/,/g, "\\,");
|
|
48
|
+
/** Detect non-speech intervals. Fails closed when ffprobe is absent. */
|
|
49
|
+
export async function detectSilence(run, req) {
|
|
50
|
+
const filter = `amovie='${escapeAmoviePath(req.audioPath)}',silencedetect=n=${SILENCE_NOISE_N}:d=${SILENCE_MIN_D}`;
|
|
51
|
+
const res = await run("ffprobe", [
|
|
52
|
+
"-hide_banner",
|
|
53
|
+
"-v",
|
|
54
|
+
"info",
|
|
55
|
+
"-f",
|
|
56
|
+
"lavfi",
|
|
57
|
+
"-i",
|
|
58
|
+
filter,
|
|
59
|
+
"-show_entries",
|
|
60
|
+
"frame=pkt_pts_time",
|
|
61
|
+
"-of",
|
|
62
|
+
"csv",
|
|
63
|
+
]);
|
|
64
|
+
if (res.code === 127) {
|
|
65
|
+
return { ok: false, status: "missing", hint: "install ffmpeg >= 6 and put ffprobe on PATH" };
|
|
66
|
+
}
|
|
67
|
+
if (res.code !== 0) {
|
|
68
|
+
return { ok: false, status: "unknown", hint: res.stderr.slice(-500) || "silencedetect failed" };
|
|
69
|
+
}
|
|
70
|
+
return parseSilenceStderr(res.stderr);
|
|
71
|
+
}
|
|
72
|
+
/** Surface words fully enclosed in a silence interval (non-speech gate), with their timing. */
|
|
73
|
+
export function wordsInSilence(words, intervals) {
|
|
74
|
+
const bad = [];
|
|
75
|
+
for (const w of words) {
|
|
76
|
+
for (const [s0, s1] of intervals) {
|
|
77
|
+
if (w.start >= s0 && w.end <= s1) {
|
|
78
|
+
bad.push(w);
|
|
79
|
+
break;
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
return bad;
|
|
84
|
+
}
|
package/dist/timing.d.ts
ADDED
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Voice TimingManifest builder: measured audio durations override autoDuration.
|
|
3
|
+
*
|
|
4
|
+
* Each row's duration is the MEASURED wav length for that scene — never
|
|
5
|
+
* inferred from word spans, so gaps and pauses cannot shrink a scene.
|
|
6
|
+
* Starts are exact float cumulative sums (no rounding): the schema's strict
|
|
7
|
+
* `start < prev.start + prev.duration` comparison false-positives on rounded
|
|
8
|
+
* starts (spike NOTE 2026-10-29). The manifest validates against
|
|
9
|
+
* `TimingManifestSchema` before it is returned.
|
|
10
|
+
*/
|
|
11
|
+
import { type TimingManifest } from "@anvpx/motion-schema";
|
|
12
|
+
export interface VoiceSceneInput {
|
|
13
|
+
id: string;
|
|
14
|
+
/** Measured wav duration for the scene, seconds. Must be positive. */
|
|
15
|
+
durationS: number;
|
|
16
|
+
}
|
|
17
|
+
export type VoiceManifestResult = {
|
|
18
|
+
ok: true;
|
|
19
|
+
manifest: TimingManifest;
|
|
20
|
+
} | {
|
|
21
|
+
ok: false;
|
|
22
|
+
problems: string[];
|
|
23
|
+
};
|
|
24
|
+
/** Voice timing for one scene is its measured audio length. */
|
|
25
|
+
export declare const sceneDurationFromAudio: (audioDurationS: number) => number;
|
|
26
|
+
/** Build a `source: "voice"` manifest from measured scene durations. */
|
|
27
|
+
export declare function buildVoiceManifest(rows: readonly VoiceSceneInput[]): VoiceManifestResult;
|
package/dist/timing.js
ADDED
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Voice TimingManifest builder: measured audio durations override autoDuration.
|
|
3
|
+
*
|
|
4
|
+
* Each row's duration is the MEASURED wav length for that scene — never
|
|
5
|
+
* inferred from word spans, so gaps and pauses cannot shrink a scene.
|
|
6
|
+
* Starts are exact float cumulative sums (no rounding): the schema's strict
|
|
7
|
+
* `start < prev.start + prev.duration` comparison false-positives on rounded
|
|
8
|
+
* starts (spike NOTE 2026-10-29). The manifest validates against
|
|
9
|
+
* `TimingManifestSchema` before it is returned.
|
|
10
|
+
*/
|
|
11
|
+
import { TimingManifestSchema } from "@anvpx/motion-schema";
|
|
12
|
+
/** Voice timing for one scene is its measured audio length. */
|
|
13
|
+
export const sceneDurationFromAudio = (audioDurationS) => audioDurationS;
|
|
14
|
+
/** Build a `source: "voice"` manifest from measured scene durations. */
|
|
15
|
+
export function buildVoiceManifest(rows) {
|
|
16
|
+
const problems = [];
|
|
17
|
+
const seen = new Set();
|
|
18
|
+
rows.forEach((row) => {
|
|
19
|
+
if (seen.has(row.id))
|
|
20
|
+
problems.push(`duplicate scene id "${row.id}"`);
|
|
21
|
+
seen.add(row.id);
|
|
22
|
+
if (!(row.durationS > 0) || !Number.isFinite(row.durationS)) {
|
|
23
|
+
problems.push(`scene "${row.id}": duration must be a positive finite number`);
|
|
24
|
+
}
|
|
25
|
+
});
|
|
26
|
+
if (problems.length > 0)
|
|
27
|
+
return { ok: false, problems };
|
|
28
|
+
// Exact float cumulative starts — never rounded (see module doc).
|
|
29
|
+
let cursor = 0;
|
|
30
|
+
const scenes = rows.map((row) => {
|
|
31
|
+
const scene = { id: row.id, start: cursor, duration: row.durationS };
|
|
32
|
+
cursor = scene.start + scene.duration;
|
|
33
|
+
return scene;
|
|
34
|
+
});
|
|
35
|
+
const parsed = TimingManifestSchema.safeParse({ source: "voice", scenes });
|
|
36
|
+
if (!parsed.success) {
|
|
37
|
+
return {
|
|
38
|
+
ok: false,
|
|
39
|
+
problems: parsed.error.issues.map((issue) => `${issue.path.join("/")}: ${issue.message}`),
|
|
40
|
+
};
|
|
41
|
+
}
|
|
42
|
+
return { ok: true, manifest: parsed.data };
|
|
43
|
+
}
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
import type { Runner } from "./index.ts";
|
|
2
|
+
/** Local GLB validator (byte-correct port of cli validateGlb; no cross-layer import). */
|
|
3
|
+
export declare const validateGlbBytes: (bytes: Uint8Array) => {
|
|
4
|
+
ok: true;
|
|
5
|
+
bytes: number;
|
|
6
|
+
} | {
|
|
7
|
+
ok: false;
|
|
8
|
+
message: string;
|
|
9
|
+
};
|
|
10
|
+
/** Pinned model id (spike `run_spike.py` + REPORT). */
|
|
11
|
+
export declare const TRIPOSR_MODEL_ID = "stabilityai/TripoSR";
|
|
12
|
+
/** Pinned weights snapshot (REPORT: HF cache snapshot hash). */
|
|
13
|
+
export declare const TRIPOSR_MODEL_REV = "5b521936b01fbe1890f6f9baed0254ab6351c04a";
|
|
14
|
+
/** Pinned TripoSR source revision vendored for the spike. */
|
|
15
|
+
export declare const TRIPOSR_VENDOR_REV = "107cefdc244c39106fa830359024f6a2f1c78871";
|
|
16
|
+
/** Weights size on disk: 1,677,247,729 bytes (~1.68GB, one-time fetch). */
|
|
17
|
+
export declare const TRIPOSR_WEIGHT_BYTES = 1677247729;
|
|
18
|
+
/** Model + weights provenance (spike REPORT receipts). */
|
|
19
|
+
export declare const TRIPOSR_PROVENANCE: {
|
|
20
|
+
readonly model: "stabilityai/TripoSR";
|
|
21
|
+
readonly revision: "5b521936b01fbe1890f6f9baed0254ab6351c04a";
|
|
22
|
+
readonly vendorRev: "107cefdc244c39106fa830359024f6a2f1c78871";
|
|
23
|
+
readonly weightBytes: 1677247729;
|
|
24
|
+
readonly weightNote: "one-time ~1.68GB fetch, cached offline after first fetch";
|
|
25
|
+
readonly license: "MIT";
|
|
26
|
+
readonly device: "cpu";
|
|
27
|
+
readonly resolution: 256;
|
|
28
|
+
};
|
|
29
|
+
/** Doctor-style hint: audio owns no doctor, so the hint carries the setup. */
|
|
30
|
+
export declare const TRIPOSR_MISSING_HINT = "triposr opt-in unavailable: install triposr in the voice venv (.venv-voice) and fetch weights stabilityai/TripoSR@5b52193 (~1.68GB one-time, cached offline after first fetch)";
|
|
31
|
+
export interface Generate3dRequest {
|
|
32
|
+
image: string;
|
|
33
|
+
outGlb: string;
|
|
34
|
+
/** Explicit python binary; skips the venv probe when set. */
|
|
35
|
+
python?: string;
|
|
36
|
+
/** Marching-cubes resolution (spike: 256). */
|
|
37
|
+
resolution?: number;
|
|
38
|
+
}
|
|
39
|
+
export type Generate3dResult = {
|
|
40
|
+
ok: true;
|
|
41
|
+
outGlb: string;
|
|
42
|
+
glbBytes: number;
|
|
43
|
+
} | {
|
|
44
|
+
ok: false;
|
|
45
|
+
status: "missing" | "unknown";
|
|
46
|
+
hint: string;
|
|
47
|
+
};
|
|
48
|
+
export interface Generate3dDeps {
|
|
49
|
+
/** Overrideable file reader (tests inject fakes; default reads disk bytes). */
|
|
50
|
+
readFile?: (path: string) => Promise<Uint8Array>;
|
|
51
|
+
}
|
|
52
|
+
/**
|
|
53
|
+
* Inline TripoSR driver: rembg + TSR inference + mesh extract + GLB export,
|
|
54
|
+
* then a byte-level GLB check via local `validateGlbBytes`. Prints one JSON line:
|
|
55
|
+
* `{"out": ..., "glb_bytes": ...}`.
|
|
56
|
+
*/
|
|
57
|
+
export declare const TRIPOSR_PY: string;
|
|
58
|
+
/** Image -> GLB via the venv python. Fails closed when opt-in deps are absent. */
|
|
59
|
+
export declare function generate3d(run: Runner, req: Generate3dRequest, deps?: Generate3dDeps): Promise<Generate3dResult>;
|
package/dist/triposr.js
ADDED
|
@@ -0,0 +1,241 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Opt-in TripoSR image-to-3D adapter (no repo GLB changes here).
|
|
3
|
+
*
|
|
4
|
+
* Shells out with execFile semantics (no shell) through the injected
|
|
5
|
+
* {@link Runner}, reusing the voice venv probe from `kokoro.ts`. Missing
|
|
6
|
+
* venv or missing TripoSR weights fail closed as `missing` with a
|
|
7
|
+
* doctor-style hint — the caller keeps the existing path untouched and no
|
|
8
|
+
* fake GLB is ever produced. The fetch is pinned to `TRIPOSR_MODEL_REV` and
|
|
9
|
+
* the downloaded checkpoint is verified pre-inference (byte size always,
|
|
10
|
+
* sha256 when `TRIPOSR_WEIGHT_SHA256` is set); a mismatch fails closed as
|
|
11
|
+
* `unknown`. Output bytes are validated with the local
|
|
12
|
+
* `validateGlbBytes` port (magic `glTF`, version 2, declared length matches).
|
|
13
|
+
*/
|
|
14
|
+
import { readFile as fsReadFile } from "node:fs/promises";
|
|
15
|
+
import { resolveVenvPython } from "./kokoro.js";
|
|
16
|
+
/** Local GLB validator (byte-correct port of cli validateGlb; no cross-layer import). */
|
|
17
|
+
export const validateGlbBytes = (bytes) => {
|
|
18
|
+
if (bytes.length < 12 ||
|
|
19
|
+
bytes[0] !== 0x67 ||
|
|
20
|
+
bytes[1] !== 0x6c ||
|
|
21
|
+
bytes[2] !== 0x54 ||
|
|
22
|
+
bytes[3] !== 0x46) {
|
|
23
|
+
return { ok: false, message: "not a GLB model (missing glTF magic)" };
|
|
24
|
+
}
|
|
25
|
+
const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
|
|
26
|
+
const version = view.getUint32(4, true);
|
|
27
|
+
if (version !== 2)
|
|
28
|
+
return { ok: false, message: `unsupported glTF version ${version}; need 2` };
|
|
29
|
+
const total = view.getUint32(8, true);
|
|
30
|
+
if (total !== bytes.length) {
|
|
31
|
+
return {
|
|
32
|
+
ok: false,
|
|
33
|
+
message: `GLB length mismatch: header says ${total} bytes, file has ${bytes.length}`,
|
|
34
|
+
};
|
|
35
|
+
}
|
|
36
|
+
return { ok: true, bytes: bytes.length };
|
|
37
|
+
};
|
|
38
|
+
/** Pinned model id (spike `run_spike.py` + REPORT). */
|
|
39
|
+
export const TRIPOSR_MODEL_ID = "stabilityai/TripoSR";
|
|
40
|
+
/** Pinned weights snapshot (REPORT: HF cache snapshot hash). */
|
|
41
|
+
export const TRIPOSR_MODEL_REV = "5b521936b01fbe1890f6f9baed0254ab6351c04a";
|
|
42
|
+
/** Pinned TripoSR source revision vendored for the spike. */
|
|
43
|
+
export const TRIPOSR_VENDOR_REV = "107cefdc244c39106fa830359024f6a2f1c78871";
|
|
44
|
+
/** Weights size on disk: 1,677,247,729 bytes (~1.68GB, one-time fetch). */
|
|
45
|
+
export const TRIPOSR_WEIGHT_BYTES = 1677247729;
|
|
46
|
+
/** Model + weights provenance (spike REPORT receipts). */
|
|
47
|
+
export const TRIPOSR_PROVENANCE = {
|
|
48
|
+
model: TRIPOSR_MODEL_ID,
|
|
49
|
+
revision: TRIPOSR_MODEL_REV,
|
|
50
|
+
vendorRev: TRIPOSR_VENDOR_REV,
|
|
51
|
+
weightBytes: TRIPOSR_WEIGHT_BYTES,
|
|
52
|
+
weightNote: "one-time ~1.68GB fetch, cached offline after first fetch",
|
|
53
|
+
license: "MIT",
|
|
54
|
+
device: "cpu",
|
|
55
|
+
resolution: 256,
|
|
56
|
+
};
|
|
57
|
+
/** Doctor-style hint: audio owns no doctor, so the hint carries the setup. */
|
|
58
|
+
export const TRIPOSR_MISSING_HINT = "triposr opt-in unavailable: install triposr in the voice venv (.venv-voice) and fetch weights stabilityai/TripoSR@5b52193 (~1.68GB one-time, cached offline after first fetch)";
|
|
59
|
+
/**
|
|
60
|
+
* Inline TripoSR driver: rembg + TSR inference + mesh extract + GLB export,
|
|
61
|
+
* then a byte-level GLB check via local `validateGlbBytes`. Prints one JSON line:
|
|
62
|
+
* `{"out": ..., "glb_bytes": ...}`.
|
|
63
|
+
*/
|
|
64
|
+
export const TRIPOSR_PY = [
|
|
65
|
+
"import json, os, struct, sys",
|
|
66
|
+
"image_path = sys.argv[sys.argv.index('--image')+1]",
|
|
67
|
+
"out_path = sys.argv[sys.argv.index('--out')+1]",
|
|
68
|
+
"resolution = int(sys.argv[sys.argv.index('--resolution')+1]) if '--resolution' in sys.argv else 256",
|
|
69
|
+
"import numpy as np",
|
|
70
|
+
"import rembg",
|
|
71
|
+
"import torch",
|
|
72
|
+
"from PIL import Image",
|
|
73
|
+
"from tsr.system import TSR",
|
|
74
|
+
"from tsr.utils import remove_background, resize_foreground",
|
|
75
|
+
"from huggingface_hub import hf_hub_download",
|
|
76
|
+
`ckpt_path = hf_hub_download(repo_id="${TRIPOSR_MODEL_ID}", filename="model.ckpt", revision="${TRIPOSR_MODEL_REV}")`,
|
|
77
|
+
"actual_bytes = os.path.getsize(ckpt_path)",
|
|
78
|
+
`if actual_bytes != ${TRIPOSR_WEIGHT_BYTES}:`,
|
|
79
|
+
` sys.stderr.write(f"weight verification failed: model.ckpt expected ${TRIPOSR_WEIGHT_BYTES} bytes, got {actual_bytes}\\n"); sys.exit(3)`,
|
|
80
|
+
`expected_sha256 = os.environ.get("TRIPOSR_WEIGHT_SHA256", "")`,
|
|
81
|
+
"if expected_sha256:",
|
|
82
|
+
" import hashlib",
|
|
83
|
+
" digest = hashlib.sha256()",
|
|
84
|
+
" with open(ckpt_path, 'rb') as weights_file:",
|
|
85
|
+
" for chunk in iter(lambda: weights_file.read(8388608), b''):",
|
|
86
|
+
" digest.update(chunk)",
|
|
87
|
+
" if digest.hexdigest() != expected_sha256:",
|
|
88
|
+
" sys.stderr.write('weight verification failed: sha256 mismatch\\n'); sys.exit(3)",
|
|
89
|
+
`model = TSR.from_pretrained("${TRIPOSR_MODEL_ID}", revision="${TRIPOSR_MODEL_REV}", config_name="config.yaml", weight_name="model.ckpt")`,
|
|
90
|
+
"model.renderer.set_chunk_size(8192)",
|
|
91
|
+
"model.to('cpu')",
|
|
92
|
+
"session = rembg.new_session()",
|
|
93
|
+
"image = remove_background(Image.open(image_path), session)",
|
|
94
|
+
"image = resize_foreground(image, 0.85)",
|
|
95
|
+
"arr = np.array(image).astype(np.float32) / 255.0",
|
|
96
|
+
"arr = arr[:, :, :3] * arr[:, :, 3:4] + (1 - arr[:, :, 3:4]) * 0.5",
|
|
97
|
+
"image = Image.fromarray((arr * 255.0).astype(np.uint8))",
|
|
98
|
+
"with torch.no_grad():",
|
|
99
|
+
" scene_codes = model([image], device='cpu')",
|
|
100
|
+
"meshes = model.extract_mesh(scene_codes, True, resolution=resolution)",
|
|
101
|
+
"meshes[0].export(out_path)",
|
|
102
|
+
"with open(out_path, 'rb') as f:",
|
|
103
|
+
" data = f.read()",
|
|
104
|
+
"if len(data) < 12 or data[0:4] != b'glTF':",
|
|
105
|
+
" sys.stderr.write('not a GLB model (missing glTF magic)\\n'); sys.exit(2)",
|
|
106
|
+
"(version, total) = struct.unpack('<II', data[4:12])",
|
|
107
|
+
"if version != 2:",
|
|
108
|
+
" sys.stderr.write(f'unsupported glTF version {version}; need 2\\n'); sys.exit(2)",
|
|
109
|
+
"if total != len(data):",
|
|
110
|
+
" sys.stderr.write(f'GLB length mismatch: header says {total} bytes, file has {len(data)}\\n'); sys.exit(2)",
|
|
111
|
+
"print(json.dumps({'out': out_path, 'glb_bytes': len(data)}))",
|
|
112
|
+
].join("\n");
|
|
113
|
+
const isFiniteNumber = (value) => typeof value === "number" && Number.isFinite(value);
|
|
114
|
+
const parseModelPayload = (stdout) => {
|
|
115
|
+
let payload;
|
|
116
|
+
try {
|
|
117
|
+
payload = JSON.parse(stdout);
|
|
118
|
+
}
|
|
119
|
+
catch {
|
|
120
|
+
return { ok: false, status: "unknown", hint: "triposr gave unparsable output" };
|
|
121
|
+
}
|
|
122
|
+
if (typeof payload.out !== "string" || !isFiniteNumber(payload.glb_bytes)) {
|
|
123
|
+
return { ok: false, status: "unknown", hint: "triposr gave malformed JSON" };
|
|
124
|
+
}
|
|
125
|
+
if (!(payload.glb_bytes > 0)) {
|
|
126
|
+
return { ok: false, status: "unknown", hint: "triposr reported no GLB bytes" };
|
|
127
|
+
}
|
|
128
|
+
return { ok: true, outGlb: payload.out, glbBytes: payload.glb_bytes };
|
|
129
|
+
};
|
|
130
|
+
const MODEL_MISSING_RE = /tsr|triposr|model\.ckpt|from_pretrained|huggingface|no module named/i;
|
|
131
|
+
/** Weight tampering/drift marker from the driver — a trust failure, never "missing deps". */
|
|
132
|
+
const WEIGHT_MISMATCH_RE = /weight verification failed/i;
|
|
133
|
+
const defaultReadFile = async (path) => {
|
|
134
|
+
const buf = await fsReadFile(path);
|
|
135
|
+
return new Uint8Array(buf.buffer, buf.byteOffset, buf.byteLength);
|
|
136
|
+
};
|
|
137
|
+
const validateRequest = (req) => {
|
|
138
|
+
if (req.image.trim().length === 0 || req.outGlb.trim().length === 0) {
|
|
139
|
+
return { ok: false, status: "unknown", hint: "refusing empty image/outGlb path" };
|
|
140
|
+
}
|
|
141
|
+
const resolution = req.resolution ?? TRIPOSR_PROVENANCE.resolution;
|
|
142
|
+
if (!Number.isInteger(resolution) || resolution < 64 || resolution > 1024) {
|
|
143
|
+
return {
|
|
144
|
+
ok: false,
|
|
145
|
+
status: "unknown",
|
|
146
|
+
hint: `refusing resolution ${String(req.resolution)} (want integer 64..1024)`,
|
|
147
|
+
};
|
|
148
|
+
}
|
|
149
|
+
return { resolution };
|
|
150
|
+
};
|
|
151
|
+
const pickPython = async (run, explicit) => {
|
|
152
|
+
if (explicit !== undefined)
|
|
153
|
+
return { python: explicit };
|
|
154
|
+
const probe = await resolveVenvPython(run);
|
|
155
|
+
if (!probe.ok) {
|
|
156
|
+
return {
|
|
157
|
+
ok: false,
|
|
158
|
+
status: probe.status,
|
|
159
|
+
hint: `${probe.hint}; ${TRIPOSR_MISSING_HINT}`,
|
|
160
|
+
};
|
|
161
|
+
}
|
|
162
|
+
return { python: probe.python };
|
|
163
|
+
};
|
|
164
|
+
const runModel = async (run, python, req, resolution) => {
|
|
165
|
+
const res = await run(python, [
|
|
166
|
+
"-c",
|
|
167
|
+
TRIPOSR_PY,
|
|
168
|
+
"--image",
|
|
169
|
+
req.image,
|
|
170
|
+
"--out",
|
|
171
|
+
req.outGlb,
|
|
172
|
+
"--resolution",
|
|
173
|
+
String(resolution),
|
|
174
|
+
]);
|
|
175
|
+
if (res.code === 127) {
|
|
176
|
+
return {
|
|
177
|
+
ok: false,
|
|
178
|
+
status: "missing",
|
|
179
|
+
hint: `${python} is not runnable; ${TRIPOSR_MISSING_HINT}`,
|
|
180
|
+
};
|
|
181
|
+
}
|
|
182
|
+
if (res.code !== 0) {
|
|
183
|
+
const tail = res.stderr.slice(-500) || "triposr failed";
|
|
184
|
+
// Weight mismatch mentions model.ckpt but means corruption/drift, not absent deps.
|
|
185
|
+
if (WEIGHT_MISMATCH_RE.test(tail)) {
|
|
186
|
+
return { ok: false, status: "unknown", hint: tail };
|
|
187
|
+
}
|
|
188
|
+
if (MODEL_MISSING_RE.test(tail)) {
|
|
189
|
+
return { ok: false, status: "missing", hint: `${tail}; ${TRIPOSR_MISSING_HINT}` };
|
|
190
|
+
}
|
|
191
|
+
return { ok: false, status: "unknown", hint: tail };
|
|
192
|
+
}
|
|
193
|
+
return { stdout: res.stdout };
|
|
194
|
+
};
|
|
195
|
+
const verifyOutput = async (readFile, outGlb, claimedBytes) => {
|
|
196
|
+
let bytes;
|
|
197
|
+
try {
|
|
198
|
+
bytes = await readFile(outGlb);
|
|
199
|
+
}
|
|
200
|
+
catch {
|
|
201
|
+
return { ok: false, status: "unknown", hint: `cannot read generated GLB "${outGlb}"` };
|
|
202
|
+
}
|
|
203
|
+
// Never trust the subprocess byte count: the file on disk is the truth.
|
|
204
|
+
if (claimedBytes !== bytes.length) {
|
|
205
|
+
return {
|
|
206
|
+
ok: false,
|
|
207
|
+
status: "unknown",
|
|
208
|
+
hint: `GLB byte-count mismatch: model said ${claimedBytes} bytes, file has ${bytes.length}`,
|
|
209
|
+
};
|
|
210
|
+
}
|
|
211
|
+
const checked = validateGlbBytes(bytes);
|
|
212
|
+
if (!checked.ok)
|
|
213
|
+
return { ok: false, status: "unknown", hint: checked.message };
|
|
214
|
+
return { glbBytes: checked.bytes };
|
|
215
|
+
};
|
|
216
|
+
/** Image -> GLB via the venv python. Fails closed when opt-in deps are absent. */
|
|
217
|
+
export async function generate3d(run, req, deps) {
|
|
218
|
+
const valid = validateRequest(req);
|
|
219
|
+
if (!("resolution" in valid))
|
|
220
|
+
return valid;
|
|
221
|
+
const picked = await pickPython(run, req.python);
|
|
222
|
+
if (!("python" in picked))
|
|
223
|
+
return picked;
|
|
224
|
+
const ran = await runModel(run, picked.python, req, valid.resolution);
|
|
225
|
+
if (!("stdout" in ran))
|
|
226
|
+
return ran;
|
|
227
|
+
const parsed = parseModelPayload(ran.stdout);
|
|
228
|
+
if (!parsed.ok)
|
|
229
|
+
return parsed;
|
|
230
|
+
if (parsed.outGlb !== req.outGlb) {
|
|
231
|
+
return {
|
|
232
|
+
ok: false,
|
|
233
|
+
status: "unknown",
|
|
234
|
+
hint: `triposr reported unexpected out path "${parsed.outGlb.slice(0, 64)}"`,
|
|
235
|
+
};
|
|
236
|
+
}
|
|
237
|
+
const verified = await verifyOutput(deps?.readFile ?? defaultReadFile, req.outGlb, parsed.glbBytes);
|
|
238
|
+
if (!("glbBytes" in verified))
|
|
239
|
+
return verified;
|
|
240
|
+
return { ok: true, outGlb: req.outGlb, glbBytes: verified.glbBytes };
|
|
241
|
+
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,33 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@anvpx/motion-audio",
|
|
3
|
-
"version": "0.
|
|
4
|
-
"
|
|
5
|
-
"
|
|
3
|
+
"version": "0.1.0",
|
|
4
|
+
"description": "Audio helpers for @anvpx/motion: voice, alignment, mixing and TripoSR pins.",
|
|
5
|
+
"license": "MIT",
|
|
6
|
+
"author": "anvpx",
|
|
7
|
+
"type": "module",
|
|
8
|
+
"engines": {
|
|
9
|
+
"node": ">=22"
|
|
10
|
+
},
|
|
11
|
+
"keywords": [
|
|
12
|
+
"motion-kit",
|
|
13
|
+
"audio",
|
|
14
|
+
"voice",
|
|
15
|
+
"tts",
|
|
16
|
+
"mixing"
|
|
17
|
+
],
|
|
18
|
+
"publishConfig": {
|
|
19
|
+
"access": "public"
|
|
20
|
+
},
|
|
21
|
+
"exports": {
|
|
22
|
+
".": "./dist/index.js",
|
|
23
|
+
"./package.json": "./package.json"
|
|
24
|
+
},
|
|
25
|
+
"files": [
|
|
26
|
+
"dist/**/*.js",
|
|
27
|
+
"dist/**/*.d.ts"
|
|
28
|
+
],
|
|
29
|
+
"dependencies": {
|
|
30
|
+
"@anvpx/motion-schema": "0.1.0"
|
|
31
|
+
},
|
|
32
|
+
"scripts": {}
|
|
6
33
|
}
|
package/README.md
DELETED