@gentbajko/slopify 3.18.2 → 3.20.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/extension/slopify-studio-chrome.zip +0 -0
- package/dist/extension/slopify-studio-firefox.zip +0 -0
- package/dist/patch-notes/3.19.0.md +9 -0
- package/dist/patch-notes/3.20.0.md +10 -0
- package/dist/patch-notes/index.json +12 -0
- package/dist/slices/admission/schema.js +3 -0
- package/dist/slices/narration/breaths-detect.js +291 -0
- package/dist/slices/narration/breaths-fixture.js +52 -0
- package/dist/slices/narration/breaths-model.js +30 -0
- package/dist/slices/narration/breaths.js +134 -0
- package/dist/slices/narration/clean-pieces.js +69 -0
- package/dist/slices/play-drafts/convert.js +4 -0
- package/dist/slices/play-drafts/schema.js +4 -0
- package/dist/slices/project-templates/from-project.js +2 -0
- package/dist/slices/rebuild/preview-details.js +2 -0
- package/dist/slices/rebuild/recipe-audio.js +3 -0
- package/dist/slices/rebuild/recipe-breaths.js +26 -0
- package/dist/slices/rebuild/recipe-voices.js +2 -0
- package/dist/slices/rebuild/runtime-local.js +17 -3
- package/dist/tutorials/Editing-a-Project.md +4 -2
- package/dist/web/assets/{index-DASWFus0.js → index-DNPct8pl.js} +143 -143
- package/dist/web/assets/{pdf-CF97Fw5v.js → pdf-tyJWWiKA.js} +1 -1
- package/dist/web/index.html +1 -1
- package/package.json +1 -1
|
Binary file
|
|
Binary file
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
# Slopify 3.19.0
|
|
2
|
+
|
|
3
|
+
Released 11 October 2026.
|
|
4
|
+
|
|
5
|
+
- **Remove breaths.** A new switch beside the pauses between sentences (Play → Video and style, Edit project → Pauses and volume) mutes the loud in-breath some voices take after almost every comma and full stop, Inworld's TTS-2 among them. Words, pauses and captions stay exactly where they were. It is off by default. See `docs/breaths.md`.
|
|
6
|
+
|
|
7
|
+
## Upgrading from 3.18.2
|
|
8
|
+
|
|
9
|
+
- Nothing becomes outdated. Turning Remove breaths on for a project joins its narration again from the pieces it already has; no speech is made again.
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
# Slopify 3.20.0
|
|
2
|
+
|
|
3
|
+
Released 11 October 2026.
|
|
4
|
+
|
|
5
|
+
- **Remove breaths is now Soften breaths, and it no longer clips words.** 3.19.0 muted every breath it found to silence. On a full-length narration some of those stretches held the edge of a word, which came out clipped ("Highs" as "Hi…s"), and the drops to silence were abrupt. Breaths are now turned down 12 dB with gentle fades, and any stretch with a word sound in it is left alone. See `docs/breaths.md`.
|
|
6
|
+
- **Soften S sounds.** A new switch under Soften breaths (Play → Video and style, Edit project → Pauses and volume) takes the sharp edge off s and sh sounds with a very light de-esser. It is off by default and works with or without Soften breaths.
|
|
7
|
+
|
|
8
|
+
## Upgrading from 3.19.0
|
|
9
|
+
|
|
10
|
+
- A project that had Remove breaths on joins its narration again once, from the pieces it already has. No speech is made again. If you used Remove breaths, remake those videos.
|
|
@@ -1,4 +1,16 @@
|
|
|
1
1
|
[
|
|
2
|
+
{
|
|
3
|
+
"id": "3.20.0",
|
|
4
|
+
"title": "Slopify 3.20.0",
|
|
5
|
+
"version": "3.20.0",
|
|
6
|
+
"date": "2026-10-11"
|
|
7
|
+
},
|
|
8
|
+
{
|
|
9
|
+
"id": "3.19.0",
|
|
10
|
+
"title": "Slopify 3.19.0",
|
|
11
|
+
"version": "3.19.0",
|
|
12
|
+
"date": "2026-10-11"
|
|
13
|
+
},
|
|
2
14
|
{
|
|
3
15
|
"id": "3.18.2",
|
|
4
16
|
"title": "Slopify 3.18.2",
|
|
@@ -185,6 +185,9 @@ export const runDraftSchema = z.object({
|
|
|
185
185
|
// Pauses between sentences and paragraphs (`narration/pauses-model.ts`); checked by admission.
|
|
186
186
|
sentencePauseSeconds: z.number().optional(),
|
|
187
187
|
paragraphPauseSeconds: z.number().optional(),
|
|
188
|
+
// Soften breaths and Soften S sounds (`narration/breaths-model.ts`); absent is off.
|
|
189
|
+
removeBreaths: z.boolean().optional(),
|
|
190
|
+
deEss: z.boolean().optional(),
|
|
188
191
|
// Absent is English (`kernel/ports/languages.ts`).
|
|
189
192
|
language: languageSchema.optional(),
|
|
190
193
|
});
|
|
@@ -0,0 +1,291 @@
|
|
|
1
|
+
import { setImmediate as nextTurn } from "node:timers/promises";
|
|
2
|
+
// The breath detector behind Soften breaths (`breaths-model.ts` says why). It reads 16-bit mono
|
|
3
|
+
// PCM at 48 kHz in 20 ms frames, 10 ms apart, and gives each frame three numbers: its level
|
|
4
|
+
// (RMS in dB), how often it crosses zero (noise crosses often, a voiced vowel seldom) and its
|
|
5
|
+
// spectral centroid (where its sound sits: an in-breath low, an "s" or "f" high).
|
|
6
|
+
// 1. A frame is breath-like when it is noisy (crosses zero in at least 6% of its samples),
|
|
7
|
+
// between a whisper and speech in level, and centred under 2.8 kHz.
|
|
8
|
+
// 2. A run of breath-like frames 140 ms or longer is a breath. It grows back over the quiet
|
|
9
|
+
// frames before it until silence, a voiced word's tail or a fricative, and forward over the
|
|
10
|
+
// quiet after it until silence or the next word's voice.
|
|
11
|
+
// 3. A stretch whose loudest 10 ms (RMS over its non-overlapping 10 ms blocks) is above -26 dB
|
|
12
|
+
// at the calibration (8.74 dB under the voice's level) holds a word sound, and is left alone. 3.19.0 muted those too
|
|
13
|
+
// and clipped word edges ("Highs" came out "Hi…s").
|
|
14
|
+
// 4. Each other breath is turned down 12 dB, with 45 ms linear fades inside its own stretch (a
|
|
15
|
+
// fade outside it once landed on the next word's first sound). Silencing it, as 3.19.0 did,
|
|
16
|
+
// dropped to nothing abruptly; a stretch of 90 ms or less is left as it is.
|
|
17
|
+
// Levels are relative to the voice's own: the median level of its voiced frames. A quieter or
|
|
18
|
+
// louder voice finds the same breaths. The thresholds are the prototype's absolute ones, which
|
|
19
|
+
// Gent approved by ear on the finished video's audio: its median voiced frame (read as
|
|
20
|
+
// `speechLevel` reads it, over the whole 2.5 h video) is -17.26 dB. The levelled pieces the
|
|
21
|
+
// join works on sit about 4 dB lower, and every threshold moves down with them.
|
|
22
|
+
export const breathRate = 48_000;
|
|
23
|
+
const frameLength = 960; // 20 ms
|
|
24
|
+
const hop = 480; // 10 ms
|
|
25
|
+
// Calibrated on the finished video's audio, which is what was approved by ear.
|
|
26
|
+
const referenceSpeech = -17.26;
|
|
27
|
+
const breathCeiling = -22;
|
|
28
|
+
const breathFloor = -55;
|
|
29
|
+
const voicedFloor = -42;
|
|
30
|
+
const silenceFloor = -58;
|
|
31
|
+
const noisyCrossings = 0.06;
|
|
32
|
+
const breathCentroid = 2800;
|
|
33
|
+
const breathFrames = 14; // 140 ms
|
|
34
|
+
// A voiced frame for the reference: under the noisy crossing rate and within 30 dB of the
|
|
35
|
+
// voice's loud frames (the 95th percentile).
|
|
36
|
+
const voicedSpan = 30;
|
|
37
|
+
// A stretch with a 10 ms block louder than this (at the calibration) is left alone.
|
|
38
|
+
const wordCeiling = -26;
|
|
39
|
+
const softGain = 10 ** (-12 / 20);
|
|
40
|
+
const fadeSamples = 2160; // 45 ms
|
|
41
|
+
// ceiling: 200 frames (2 s of audio, about 15 ms of work) between yields, so a long piece never
|
|
42
|
+
// holds the server's event loop long enough for the page to notice.
|
|
43
|
+
const framesPerTurn = 200;
|
|
44
|
+
// Where the breaths are in a piece, and which hold a word sound, read once from its PCM in order.
|
|
45
|
+
export async function scanBreaths(chunks) {
|
|
46
|
+
return breathsIn(await featuresOf(chunks));
|
|
47
|
+
}
|
|
48
|
+
// The breaths to soften in a piece.
|
|
49
|
+
export async function findBreaths(chunks) {
|
|
50
|
+
return (await scanBreaths(chunks)).stretches;
|
|
51
|
+
}
|
|
52
|
+
// The chunk starting `offset` samples into the piece with the stretches turned down; a new array.
|
|
53
|
+
export function softenChunk(chunk, offset, stretches) {
|
|
54
|
+
const out = Int16Array.from(chunk);
|
|
55
|
+
const end = offset + chunk.length;
|
|
56
|
+
for (const stretch of stretches) {
|
|
57
|
+
if (stretch.end <= offset || stretch.start >= end)
|
|
58
|
+
continue;
|
|
59
|
+
if (stretch.end - stretch.start <= 2 * fadeSamples)
|
|
60
|
+
continue;
|
|
61
|
+
const fade = fadeSamples;
|
|
62
|
+
const from = Math.max(stretch.start, offset);
|
|
63
|
+
const to = Math.min(stretch.end, end);
|
|
64
|
+
for (let at = from; at < to; at += 1) {
|
|
65
|
+
const local = at - offset;
|
|
66
|
+
out[local] = Math.round((chunk[local] ?? 0) * gainAt(stretch, fade, at));
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
return out;
|
|
70
|
+
}
|
|
71
|
+
function gainAt(stretch, fade, at) {
|
|
72
|
+
const into = at - stretch.start;
|
|
73
|
+
if (into < fade)
|
|
74
|
+
return 1 + ((softGain - 1) * into) / (fade - 1);
|
|
75
|
+
const out = at - (stretch.end - fade);
|
|
76
|
+
if (out >= 0)
|
|
77
|
+
return softGain + ((1 - softGain) * out) / (fade - 1);
|
|
78
|
+
return softGain;
|
|
79
|
+
}
|
|
80
|
+
async function featuresOf(chunks) {
|
|
81
|
+
let capacity = 4096;
|
|
82
|
+
let db = new Float32Array(capacity);
|
|
83
|
+
let crossings = new Float32Array(capacity);
|
|
84
|
+
let centroid = new Float32Array(capacity);
|
|
85
|
+
let blockDb = new Float32Array(capacity + 1);
|
|
86
|
+
let count = 0;
|
|
87
|
+
const work = fftWorkspace();
|
|
88
|
+
const frame = new Float64Array(frameLength);
|
|
89
|
+
let carry = new Int16Array(0);
|
|
90
|
+
let base = 0;
|
|
91
|
+
for await (const chunk of chunks) {
|
|
92
|
+
const pending = new Int16Array(carry.length + chunk.length);
|
|
93
|
+
pending.set(carry);
|
|
94
|
+
pending.set(chunk, carry.length);
|
|
95
|
+
// A frame is read once a sample after it has arrived, so the last frame of the piece is
|
|
96
|
+
// the last that ends before its final sample, as the prototype read it.
|
|
97
|
+
while (count * hop + frameLength < base + pending.length) {
|
|
98
|
+
if (count === capacity) {
|
|
99
|
+
capacity *= 2;
|
|
100
|
+
db = grown(db, capacity);
|
|
101
|
+
crossings = grown(crossings, capacity);
|
|
102
|
+
centroid = grown(centroid, capacity);
|
|
103
|
+
blockDb = grown(blockDb, capacity + 1);
|
|
104
|
+
}
|
|
105
|
+
const start = count * hop - base;
|
|
106
|
+
for (let at = 0; at < frameLength; at += 1)
|
|
107
|
+
frame[at] = (pending[start + at] ?? 0) / 32768;
|
|
108
|
+
db[count] = levelOf(frame);
|
|
109
|
+
crossings[count] = crossingsOf(frame);
|
|
110
|
+
centroid[count] = powerSpectrumCentroid(frame, work);
|
|
111
|
+
// A frame's halves are blocks count and count + 1; the next frame reads the second again.
|
|
112
|
+
blockDb[count] = levelOf(frame.subarray(0, hop));
|
|
113
|
+
blockDb[count + 1] = levelOf(frame.subarray(hop));
|
|
114
|
+
count += 1;
|
|
115
|
+
if (count % framesPerTurn === 0)
|
|
116
|
+
await nextTurn();
|
|
117
|
+
}
|
|
118
|
+
const keep = count * hop - base;
|
|
119
|
+
carry = pending.slice(keep);
|
|
120
|
+
base += keep;
|
|
121
|
+
}
|
|
122
|
+
return { count, db, crossings, centroid, blockDb };
|
|
123
|
+
}
|
|
124
|
+
function grown(values, capacity) {
|
|
125
|
+
const next = new Float32Array(capacity);
|
|
126
|
+
next.set(values);
|
|
127
|
+
return next;
|
|
128
|
+
}
|
|
129
|
+
function breathsIn(features) {
|
|
130
|
+
const speech = speechLevel(features);
|
|
131
|
+
if (speech === undefined)
|
|
132
|
+
return { stretches: [], skipped: [] };
|
|
133
|
+
const level = (prototype) => prototype - referenceSpeech + speech;
|
|
134
|
+
const ceiling = level(breathCeiling);
|
|
135
|
+
const floor = level(breathFloor);
|
|
136
|
+
const voicedAbove = level(voicedFloor);
|
|
137
|
+
const silentBelow = level(silenceFloor);
|
|
138
|
+
const wordAbove = level(wordCeiling);
|
|
139
|
+
const { count, db, crossings, centroid, blockDb } = features;
|
|
140
|
+
const at = (values, index) => values[index] ?? 0;
|
|
141
|
+
const voiced = (index) => at(db, index) > voicedAbove && at(crossings, index) < noisyCrossings;
|
|
142
|
+
const hit = (index) => at(crossings, index) >= noisyCrossings &&
|
|
143
|
+
at(db, index) > floor &&
|
|
144
|
+
at(db, index) < ceiling &&
|
|
145
|
+
at(centroid, index) < breathCentroid;
|
|
146
|
+
const stretches = [];
|
|
147
|
+
const skipped = [];
|
|
148
|
+
// Blocks a to b: the stretch from a * hop to (b - 1) * hop + frameLength = (b + 1) * hop.
|
|
149
|
+
const holdsWord = (a, b) => {
|
|
150
|
+
for (let block = a; block <= b; block += 1)
|
|
151
|
+
if (at(blockDb, block) > wordAbove)
|
|
152
|
+
return true;
|
|
153
|
+
return false;
|
|
154
|
+
};
|
|
155
|
+
let i = 0;
|
|
156
|
+
while (i < count) {
|
|
157
|
+
if (!hit(i)) {
|
|
158
|
+
i += 1;
|
|
159
|
+
continue;
|
|
160
|
+
}
|
|
161
|
+
let j = i;
|
|
162
|
+
while (j < count && hit(j))
|
|
163
|
+
j += 1;
|
|
164
|
+
if (j - i >= breathFrames) {
|
|
165
|
+
let a = i;
|
|
166
|
+
while (a > 0 &&
|
|
167
|
+
at(db, a - 1) > silentBelow &&
|
|
168
|
+
!voiced(a - 1) &&
|
|
169
|
+
at(centroid, a - 1) < breathCentroid)
|
|
170
|
+
a -= 1;
|
|
171
|
+
let b = j;
|
|
172
|
+
while (b < count && at(db, b) > silentBelow && !voiced(b))
|
|
173
|
+
b += 1;
|
|
174
|
+
const stretch = { start: a * hop, end: (b - 1) * hop + frameLength };
|
|
175
|
+
(holdsWord(a, b) ? skipped : stretches).push(stretch);
|
|
176
|
+
}
|
|
177
|
+
i = j;
|
|
178
|
+
}
|
|
179
|
+
return { stretches, skipped };
|
|
180
|
+
}
|
|
181
|
+
// The voice's own level: the median of its voiced frames, those under the noisy crossing rate
|
|
182
|
+
// within 30 dB of its loud frames. Undefined for a piece without a voiced frame.
|
|
183
|
+
function speechLevel(features) {
|
|
184
|
+
if (features.count === 0)
|
|
185
|
+
return undefined;
|
|
186
|
+
const levels = features.db.slice(0, features.count).sort();
|
|
187
|
+
const loud = percentile(levels, 0.95);
|
|
188
|
+
const voiced = [];
|
|
189
|
+
for (let index = 0; index < features.count; index += 1) {
|
|
190
|
+
const db = features.db[index] ?? 0;
|
|
191
|
+
if ((features.crossings[index] ?? 1) < noisyCrossings && db > loud - voicedSpan)
|
|
192
|
+
voiced.push(db);
|
|
193
|
+
}
|
|
194
|
+
if (voiced.length === 0)
|
|
195
|
+
return undefined;
|
|
196
|
+
return percentile(Float32Array.from(voiced).sort(), 0.5);
|
|
197
|
+
}
|
|
198
|
+
// numpy's default: linear between the two nearest ranks of sorted values.
|
|
199
|
+
function percentile(sorted, share) {
|
|
200
|
+
const position = share * (sorted.length - 1);
|
|
201
|
+
const below = Math.floor(position);
|
|
202
|
+
const low = sorted[below] ?? 0;
|
|
203
|
+
const high = sorted[Math.min(sorted.length - 1, below + 1)] ?? low;
|
|
204
|
+
return low + (high - low) * (position - below);
|
|
205
|
+
}
|
|
206
|
+
function levelOf(frame) {
|
|
207
|
+
let sum = 0;
|
|
208
|
+
for (const value of frame)
|
|
209
|
+
sum += value * value;
|
|
210
|
+
return 20 * Math.log10(Math.sqrt(sum / frame.length) + 1e-9);
|
|
211
|
+
}
|
|
212
|
+
// The share of neighbouring samples whose sign differs, a zero counting half.
|
|
213
|
+
function crossingsOf(frame) {
|
|
214
|
+
let sum = 0;
|
|
215
|
+
let previous = Math.sign(frame[0] ?? 0);
|
|
216
|
+
for (let at = 1; at < frame.length; at += 1) {
|
|
217
|
+
const sign = Math.sign(frame[at] ?? 0);
|
|
218
|
+
sum += Math.abs(sign - previous);
|
|
219
|
+
previous = sign;
|
|
220
|
+
}
|
|
221
|
+
return sum / 2 / (frame.length - 1);
|
|
222
|
+
}
|
|
223
|
+
function fftWorkspace() {
|
|
224
|
+
const angle = (index) => (2 * Math.PI * index) / frameLength;
|
|
225
|
+
return {
|
|
226
|
+
re: new Float64Array(frameLength),
|
|
227
|
+
im: new Float64Array(frameLength),
|
|
228
|
+
outRe: new Float64Array(frameLength),
|
|
229
|
+
outIm: new Float64Array(frameLength),
|
|
230
|
+
tempRe: new Float64Array(8),
|
|
231
|
+
tempIm: new Float64Array(8),
|
|
232
|
+
cos: Float64Array.from({ length: frameLength }, (_one, index) => Math.cos(angle(index))),
|
|
233
|
+
sin: Float64Array.from({ length: frameLength }, (_one, index) => Math.sin(angle(index))),
|
|
234
|
+
// numpy's hanning: symmetric, zero at both ends.
|
|
235
|
+
hann: Float64Array.from({ length: frameLength }, (_one, index) => 0.5 - 0.5 * Math.cos((2 * Math.PI * index) / (frameLength - 1))),
|
|
236
|
+
};
|
|
237
|
+
}
|
|
238
|
+
// Where a 20 ms frame's sound sits, in Hz: the power-weighted mean frequency of its
|
|
239
|
+
// Hann-windowed spectrum, 0 for a silent frame.
|
|
240
|
+
export function powerSpectrumCentroid(frame, work = fftWorkspace()) {
|
|
241
|
+
for (let at = 0; at < frameLength; at += 1) {
|
|
242
|
+
work.re[at] = (frame[at] ?? 0) * (work.hann[at] ?? 0);
|
|
243
|
+
work.im[at] = 0;
|
|
244
|
+
}
|
|
245
|
+
transform(work, frameLength, 0, 1, 0);
|
|
246
|
+
let weighted = 0;
|
|
247
|
+
let total = 0;
|
|
248
|
+
for (let bin = 0; bin <= frameLength / 2; bin += 1) {
|
|
249
|
+
const power = (work.outRe[bin] ?? 0) ** 2 + (work.outIm[bin] ?? 0) ** 2;
|
|
250
|
+
weighted += ((bin * breathRate) / frameLength) * power;
|
|
251
|
+
total += power;
|
|
252
|
+
}
|
|
253
|
+
return total > 0 ? weighted / total : 0;
|
|
254
|
+
}
|
|
255
|
+
// A mixed-radix decimation-in-time FFT (960 = 2^6 * 3 * 5) from `work.re/im` into
|
|
256
|
+
// `work.outRe/outIm`: the n-point transform of the samples at `from`, `from + stride`, … is
|
|
257
|
+
// written at `to`.
|
|
258
|
+
function transform(work, n, from, stride, to) {
|
|
259
|
+
if (n === 1) {
|
|
260
|
+
work.outRe[to] = work.re[from] ?? 0;
|
|
261
|
+
work.outIm[to] = work.im[from] ?? 0;
|
|
262
|
+
return;
|
|
263
|
+
}
|
|
264
|
+
const radix = n % 2 === 0 ? 2 : n % 3 === 0 ? 3 : n % 5 === 0 ? 5 : n;
|
|
265
|
+
const m = n / radix;
|
|
266
|
+
for (let r = 0; r < radix; r += 1)
|
|
267
|
+
transform(work, m, from + r * stride, stride * radix, to + r * m);
|
|
268
|
+
const step = frameLength / n;
|
|
269
|
+
for (let k = 0; k < m; k += 1) {
|
|
270
|
+
for (let q = 0; q < radix; q += 1) {
|
|
271
|
+
let sumRe = 0;
|
|
272
|
+
let sumIm = 0;
|
|
273
|
+
for (let r = 0; r < radix; r += 1) {
|
|
274
|
+
const twiddle = ((r * (k + q * m)) % n) * step;
|
|
275
|
+
const c = work.cos[twiddle] ?? 1;
|
|
276
|
+
const s = work.sin[twiddle] ?? 0;
|
|
277
|
+
const yRe = work.outRe[to + r * m + k] ?? 0;
|
|
278
|
+
const yIm = work.outIm[to + r * m + k] ?? 0;
|
|
279
|
+
// y · e^(-iθ)
|
|
280
|
+
sumRe += yRe * c + yIm * s;
|
|
281
|
+
sumIm += yIm * c - yRe * s;
|
|
282
|
+
}
|
|
283
|
+
work.tempRe[q] = sumRe;
|
|
284
|
+
work.tempIm[q] = sumIm;
|
|
285
|
+
}
|
|
286
|
+
for (let q = 0; q < radix; q += 1) {
|
|
287
|
+
work.outRe[to + q * m + k] = work.tempRe[q] ?? 0;
|
|
288
|
+
work.outIm[to + q * m + k] = work.tempIm[q] ?? 0;
|
|
289
|
+
}
|
|
290
|
+
}
|
|
291
|
+
}
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
import { breathRate } from "./breaths-detect.js";
|
|
2
|
+
// Synthetic narration at 48 kHz: a voiced tone stands for a word, band-limited noise for a
|
|
3
|
+
// breath (low band, under 2.8 kHz) or a fricative (high band, an "s" or "f").
|
|
4
|
+
export const ms = (value) => Math.round((value / 1000) * breathRate);
|
|
5
|
+
export function seeded(seed) {
|
|
6
|
+
let state = seed >>> 0;
|
|
7
|
+
return () => {
|
|
8
|
+
state = (state * 1664525 + 1013904223) >>> 0;
|
|
9
|
+
return state / 2 ** 32;
|
|
10
|
+
};
|
|
11
|
+
}
|
|
12
|
+
export function tone(length, rms) {
|
|
13
|
+
const out = new Float64Array(length);
|
|
14
|
+
const amplitude = rms * Math.SQRT2;
|
|
15
|
+
for (let at = 0; at < length; at += 1)
|
|
16
|
+
out[at] = amplitude * Math.sin((2 * Math.PI * 180 * at) / breathRate);
|
|
17
|
+
return out;
|
|
18
|
+
}
|
|
19
|
+
// Noise as many random-phase sines spread evenly over a band, scaled to the RMS asked for.
|
|
20
|
+
export function noise(length, low, high, rms, seed) {
|
|
21
|
+
const random = seeded(seed);
|
|
22
|
+
const count = 48;
|
|
23
|
+
const out = new Float64Array(length);
|
|
24
|
+
for (let one = 0; one < count; one += 1) {
|
|
25
|
+
const frequency = low + ((high - low) * (one + random())) / count;
|
|
26
|
+
const phase = 2 * Math.PI * random();
|
|
27
|
+
for (let at = 0; at < length; at += 1)
|
|
28
|
+
out[at] = (out[at] ?? 0) + Math.sin((2 * Math.PI * frequency * at) / breathRate + phase);
|
|
29
|
+
}
|
|
30
|
+
const power = out.reduce((sum, value) => sum + value * value, 0) / length;
|
|
31
|
+
const scale = rms / Math.sqrt(power);
|
|
32
|
+
return out.map((value) => value * scale);
|
|
33
|
+
}
|
|
34
|
+
export function silence(length) {
|
|
35
|
+
return new Float64Array(length);
|
|
36
|
+
}
|
|
37
|
+
export function pcm(parts, gain = 1) {
|
|
38
|
+
const length = parts.reduce((sum, part) => sum + part.length, 0);
|
|
39
|
+
const out = new Int16Array(length);
|
|
40
|
+
let at = 0;
|
|
41
|
+
for (const part of parts)
|
|
42
|
+
for (const value of part) {
|
|
43
|
+
out[at] = Math.max(-32768, Math.min(32767, Math.round(value * gain * 32767)));
|
|
44
|
+
at += 1;
|
|
45
|
+
}
|
|
46
|
+
return out;
|
|
47
|
+
}
|
|
48
|
+
// The whole clip in chunks of an odd size, so frames straddle the chunk edges.
|
|
49
|
+
export function* chunked(samples, size = 7777) {
|
|
50
|
+
for (let at = 0; at < samples.length; at += size)
|
|
51
|
+
yield samples.subarray(at, at + size);
|
|
52
|
+
}
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
// Soften breaths (the config keeps its first name, `removeBreaths`). Some voices (Inworld's
|
|
2
|
+
// TTS-2 among them) take a loud, audible in-breath after almost every comma and full stop:
|
|
3
|
+
// about a third of a second, only 12 to 15 dB under the words. Switched on, the join finds those
|
|
4
|
+
// breaths in every narration piece and turns them down 12 dB (`breaths.ts`, `breaths-detect.ts`),
|
|
5
|
+
// leaving alone any stretch with a word sound in it: nothing is cut out, so every word and its
|
|
6
|
+
// timing stay exactly where they were, and the pauses between sentences (`pauses-model.ts`) are
|
|
7
|
+
// planned from the pieces as spoken just the same. It happens in the join, so the plain narration
|
|
8
|
+
// the word timing reads and the levelled one the video plays lose the same breaths.
|
|
9
|
+
//
|
|
10
|
+
// Soften S sounds (`deEss`): each narration piece in the join goes through ffmpeg's de-esser,
|
|
11
|
+
// set very light (Gent approved these values by ear), after the breaths and before the pauses.
|
|
12
|
+
//
|
|
13
|
+
// Browser-safe: Play and Edit project read it. Off is absent from the config, which is what
|
|
14
|
+
// every project saved before them was, and the joins keep their fingerprints then.
|
|
15
|
+
export function breathsOn(config) {
|
|
16
|
+
return config.removeBreaths === true;
|
|
17
|
+
}
|
|
18
|
+
// Play's switch as a run's config holds it: only on is written.
|
|
19
|
+
export function breathsOfForm(form) {
|
|
20
|
+
return form.removeBreaths === true ? { removeBreaths: true } : {};
|
|
21
|
+
}
|
|
22
|
+
// ffmpeg's `deesser` options: intensity, most de-essing, frequency (each 0 to 1).
|
|
23
|
+
export const deEsser = { intensity: 0.35, max: 0.5, frequency: 0.5 };
|
|
24
|
+
export function deEssOn(config) {
|
|
25
|
+
return config.deEss === true;
|
|
26
|
+
}
|
|
27
|
+
// Play's switch as a run's config holds it: only on is written.
|
|
28
|
+
export function deEssOfForm(form) {
|
|
29
|
+
return form.deEss === true ? { deEss: true } : {};
|
|
30
|
+
}
|
|
@@ -0,0 +1,134 @@
|
|
|
1
|
+
import { mkdirSync, rmSync } from "node:fs";
|
|
2
|
+
import { open, stat } from "node:fs/promises";
|
|
3
|
+
import { join } from "node:path";
|
|
4
|
+
import { runFfmpeg } from "../video/ffmpeg.js";
|
|
5
|
+
import { breathRate, scanBreaths, softenChunk } from "./breaths-detect.js";
|
|
6
|
+
// Soften breaths, made in the join (`breaths-model.ts` says why there). Each narration piece is
|
|
7
|
+
// decoded by ffmpeg to 16-bit mono PCM at 48 kHz in a file beside it, read once to find its
|
|
8
|
+
// breaths (`breaths-detect.ts`) and once more to write it again as a WAV with them turned down. Both
|
|
9
|
+
// reads go in chunks, so a long piece never sits in memory whole, and the detector yields to
|
|
10
|
+
// the event loop as it goes, so the server stays responsive while it works. A piece without a
|
|
11
|
+
// breath is written again all the same: the join takes every piece in the one format.
|
|
12
|
+
// ceiling: one second of audio per read.
|
|
13
|
+
const chunkSamples = breathRate;
|
|
14
|
+
export async function softenBreaths(run, files, directory) {
|
|
15
|
+
mkdirSync(directory, { recursive: true, mode: 0o700 });
|
|
16
|
+
const written = [];
|
|
17
|
+
let softened = 0;
|
|
18
|
+
let skipped = 0;
|
|
19
|
+
for (const [index, file] of files.entries()) {
|
|
20
|
+
run.signal.throwIfAborted();
|
|
21
|
+
const name = `breaths-${String(index + 1).padStart(4, "0")}`;
|
|
22
|
+
const raw = join(directory, `${name}.pcm`);
|
|
23
|
+
const output = join(directory, `${name}.wav`);
|
|
24
|
+
try {
|
|
25
|
+
await runFfmpeg({
|
|
26
|
+
bin: run.bin,
|
|
27
|
+
args: decodeArgs(file, raw),
|
|
28
|
+
signal: run.signal,
|
|
29
|
+
log: run.log,
|
|
30
|
+
onProgress: () => { },
|
|
31
|
+
});
|
|
32
|
+
const scan = await scanBreaths(pcmChunks(raw));
|
|
33
|
+
run.signal.throwIfAborted();
|
|
34
|
+
await writeSoftened(raw, output, scan.stretches);
|
|
35
|
+
softened += scan.stretches.length;
|
|
36
|
+
skipped += scan.skipped.length;
|
|
37
|
+
}
|
|
38
|
+
catch (error) {
|
|
39
|
+
if (run.signal.aborted)
|
|
40
|
+
throw error;
|
|
41
|
+
throw new Error(`Soften breaths could not process narration piece ${String(index + 1)} (${error instanceof Error ? error.message : String(error)}), so the narration can't be joined together. Use Try again; if it fails again, turn off Soften breaths in Edit project → Pauses and volume and use Save changes, then use Download diagnostics in Settings and report it.`, { cause: error });
|
|
42
|
+
}
|
|
43
|
+
finally {
|
|
44
|
+
rmSync(raw, { force: true });
|
|
45
|
+
}
|
|
46
|
+
written.push(output);
|
|
47
|
+
}
|
|
48
|
+
return { files: written, softened, skipped };
|
|
49
|
+
}
|
|
50
|
+
export function decodeArgs(input, output) {
|
|
51
|
+
return [
|
|
52
|
+
"-hide_banner",
|
|
53
|
+
"-nostdin",
|
|
54
|
+
"-loglevel",
|
|
55
|
+
"error",
|
|
56
|
+
"-nostats",
|
|
57
|
+
"-y",
|
|
58
|
+
"-i",
|
|
59
|
+
input,
|
|
60
|
+
"-map",
|
|
61
|
+
"0:a:0",
|
|
62
|
+
"-ac",
|
|
63
|
+
"1",
|
|
64
|
+
"-ar",
|
|
65
|
+
String(breathRate),
|
|
66
|
+
"-c:a",
|
|
67
|
+
"pcm_s16le",
|
|
68
|
+
"-f",
|
|
69
|
+
"s16le",
|
|
70
|
+
output,
|
|
71
|
+
];
|
|
72
|
+
}
|
|
73
|
+
async function* pcmChunks(path) {
|
|
74
|
+
const handle = await open(path, "r");
|
|
75
|
+
try {
|
|
76
|
+
for (;;) {
|
|
77
|
+
const chunk = await readChunk(handle);
|
|
78
|
+
if (chunk === undefined)
|
|
79
|
+
return;
|
|
80
|
+
yield chunk;
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
finally {
|
|
84
|
+
await handle.close();
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
async function readChunk(handle) {
|
|
88
|
+
const buffer = Buffer.alloc(chunkSamples * 2);
|
|
89
|
+
const { bytesRead } = await handle.read(buffer, 0, buffer.length, null);
|
|
90
|
+
const samples = Math.floor(bytesRead / 2);
|
|
91
|
+
if (samples === 0)
|
|
92
|
+
return undefined;
|
|
93
|
+
// A copy into its own buffer: Int16Array needs an aligned, even-length view.
|
|
94
|
+
const out = new Int16Array(samples);
|
|
95
|
+
for (let at = 0; at < samples; at += 1)
|
|
96
|
+
out[at] = buffer.readInt16LE(at * 2);
|
|
97
|
+
return out;
|
|
98
|
+
}
|
|
99
|
+
async function writeSoftened(raw, output, stretches) {
|
|
100
|
+
const size = (await stat(raw)).size;
|
|
101
|
+
const bytes = size - (size % 2);
|
|
102
|
+
const target = await open(output, "w", 0o600);
|
|
103
|
+
try {
|
|
104
|
+
await target.write(wavHeader(bytes));
|
|
105
|
+
let offset = 0;
|
|
106
|
+
for await (const chunk of pcmChunks(raw)) {
|
|
107
|
+
const soft = softenChunk(chunk, offset, stretches);
|
|
108
|
+
// Typed arrays are little-endian on every platform Slopify ships for, as WAV is.
|
|
109
|
+
await target.write(Buffer.from(soft.buffer, soft.byteOffset, soft.byteLength));
|
|
110
|
+
offset += chunk.length;
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
finally {
|
|
114
|
+
await target.close();
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
// A canonical 44-byte header: PCM, mono, 16-bit, 48 kHz.
|
|
118
|
+
export function wavHeader(dataBytes) {
|
|
119
|
+
const header = Buffer.alloc(44);
|
|
120
|
+
header.write("RIFF", 0, "ascii");
|
|
121
|
+
header.writeUInt32LE(36 + dataBytes, 4);
|
|
122
|
+
header.write("WAVE", 8, "ascii");
|
|
123
|
+
header.write("fmt ", 12, "ascii");
|
|
124
|
+
header.writeUInt32LE(16, 16);
|
|
125
|
+
header.writeUInt16LE(1, 20);
|
|
126
|
+
header.writeUInt16LE(1, 22);
|
|
127
|
+
header.writeUInt32LE(breathRate, 24);
|
|
128
|
+
header.writeUInt32LE(breathRate * 2, 28);
|
|
129
|
+
header.writeUInt16LE(2, 32);
|
|
130
|
+
header.writeUInt16LE(16, 34);
|
|
131
|
+
header.write("data", 36, "ascii");
|
|
132
|
+
header.writeUInt32LE(dataBytes, 40);
|
|
133
|
+
return header;
|
|
134
|
+
}
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
import { mkdirSync } from "node:fs";
|
|
2
|
+
import { join } from "node:path";
|
|
3
|
+
import { runFfmpeg } from "../video/ffmpeg.js";
|
|
4
|
+
import { softenBreaths } from "./breaths.js";
|
|
5
|
+
import { deEsser } from "./breaths-model.js";
|
|
6
|
+
export async function cleanPieces(run, files, directory, cleaning) {
|
|
7
|
+
if (!cleaning.breaths && !cleaning.deEss)
|
|
8
|
+
return undefined;
|
|
9
|
+
const breaths = cleaning.breaths
|
|
10
|
+
? await softenBreaths(run, files, join(directory, "breaths"))
|
|
11
|
+
: undefined;
|
|
12
|
+
const soft = breaths?.files ?? files;
|
|
13
|
+
const cleaned = cleaning.deEss ? await deEssPieces(run, soft, join(directory, "deess")) : soft;
|
|
14
|
+
return {
|
|
15
|
+
files: cleaned,
|
|
16
|
+
...(breaths === undefined
|
|
17
|
+
? {}
|
|
18
|
+
: { breaths: { softened: breaths.softened, skipped: breaths.skipped } }),
|
|
19
|
+
};
|
|
20
|
+
}
|
|
21
|
+
export async function deEssPieces(run, files, directory) {
|
|
22
|
+
mkdirSync(directory, { recursive: true, mode: 0o700 });
|
|
23
|
+
const written = [];
|
|
24
|
+
for (const [index, file] of files.entries()) {
|
|
25
|
+
run.signal.throwIfAborted();
|
|
26
|
+
const output = join(directory, `deess-${String(index + 1).padStart(4, "0")}.wav`);
|
|
27
|
+
try {
|
|
28
|
+
await runFfmpeg({
|
|
29
|
+
bin: run.bin,
|
|
30
|
+
args: deEssArgs(file, output),
|
|
31
|
+
signal: run.signal,
|
|
32
|
+
log: run.log,
|
|
33
|
+
onProgress: () => { },
|
|
34
|
+
});
|
|
35
|
+
}
|
|
36
|
+
catch (error) {
|
|
37
|
+
if (run.signal.aborted)
|
|
38
|
+
throw error;
|
|
39
|
+
throw new Error(`Soften S sounds could not process narration piece ${String(index + 1)} (${error instanceof Error ? error.message : String(error)}), so the narration can't be joined together. Use Try again; if it fails again, turn off Soften S sounds in Edit project → Pauses and volume and use Save changes, then use Download diagnostics in Settings and report it.`, { cause: error });
|
|
40
|
+
}
|
|
41
|
+
written.push(output);
|
|
42
|
+
}
|
|
43
|
+
return written;
|
|
44
|
+
}
|
|
45
|
+
export function deEssFilter() {
|
|
46
|
+
return `deesser=i=${String(deEsser.intensity)}:m=${String(deEsser.max)}:f=${String(deEsser.frequency)}`;
|
|
47
|
+
}
|
|
48
|
+
// The piece's own rate and channels, written as 16-bit WAV like the other cleaned pieces.
|
|
49
|
+
export function deEssArgs(input, output) {
|
|
50
|
+
return [
|
|
51
|
+
"-hide_banner",
|
|
52
|
+
"-nostdin",
|
|
53
|
+
"-loglevel",
|
|
54
|
+
"error",
|
|
55
|
+
"-nostats",
|
|
56
|
+
"-y",
|
|
57
|
+
"-i",
|
|
58
|
+
input,
|
|
59
|
+
"-map",
|
|
60
|
+
"0:a:0",
|
|
61
|
+
"-af",
|
|
62
|
+
deEssFilter(),
|
|
63
|
+
"-c:a",
|
|
64
|
+
"pcm_s16le",
|
|
65
|
+
"-f",
|
|
66
|
+
"wav",
|
|
67
|
+
output,
|
|
68
|
+
];
|
|
69
|
+
}
|
|
@@ -4,6 +4,7 @@ import { runDraftSchema } from "../admission/schema.js";
|
|
|
4
4
|
import { draftDocumentThemeOf } from "../document/model.js";
|
|
5
5
|
import { defaultExpectedWords, everyMinutesMax, everyMinutesMin, expectedWordsMax, imagesPerHourMax, imagesPerHourMin, perHourFromMinutes, wordCount, } from "../images/scale.js";
|
|
6
6
|
import { defaultLoudness, loudnessOfForm, } from "../loudness/model.js";
|
|
7
|
+
import { breathsOfForm, deEssOfForm } from "../narration/breaths-model.js";
|
|
7
8
|
import { pausesOfForm } from "../narration/pauses-model.js";
|
|
8
9
|
import { reviewSettingsFromForm } from "../reviews/model.js";
|
|
9
10
|
import { stageMakesItems } from "../reviews/rules.js";
|
|
@@ -292,6 +293,9 @@ export function toAdmissionDraft(input) {
|
|
|
292
293
|
...pickLoudness(sources.audio, loudnessOfForm(form.loudness, input.loudness ?? defaultLoudness)),
|
|
293
294
|
// Pauses between sentences: the default for a new run, with a narration to pace.
|
|
294
295
|
...(sources.audio === "generate" ? pausesOfForm(form) : {}),
|
|
296
|
+
// Soften breaths and Soften S sounds: only while on, with a narration made here to clean.
|
|
297
|
+
...(sources.audio === "generate" ? breathsOfForm(form) : {}),
|
|
298
|
+
...(sources.audio === "generate" ? deEssOfForm(form) : {}),
|
|
295
299
|
edgeSilenceSeconds: measure("edgeSilenceSeconds", sources.audio !== "off", defaultEdgeSilenceSeconds, edgeSilenceSecondsProblem),
|
|
296
300
|
};
|
|
297
301
|
const parsed = runDraftSchema.safeParse(draft);
|
|
@@ -86,6 +86,10 @@ export const playDraftFormSchema = z
|
|
|
86
86
|
// paragraph minimum (`narration/pauses-model.ts`).
|
|
87
87
|
sentencePause: text.optional(),
|
|
88
88
|
paragraphPause: text.optional(),
|
|
89
|
+
// Soften breaths and Soften S sounds (`narration/breaths-model.ts`). Absent on drafts and
|
|
90
|
+
// templates saved before them: off.
|
|
91
|
+
removeBreaths: z.boolean().optional(),
|
|
92
|
+
deEss: z.boolean().optional(),
|
|
89
93
|
// Absent on drafts and templates saved before the YouTube description: off, built-in prompt.
|
|
90
94
|
youtubeDescription: z.boolean().optional(),
|
|
91
95
|
showFigures: z.boolean().optional(),
|
|
@@ -236,6 +236,8 @@ export function documentFromProject(deps, revision) {
|
|
|
236
236
|
...(config.paragraphPauseSeconds === undefined
|
|
237
237
|
? {}
|
|
238
238
|
: { paragraphPause: String(config.paragraphPauseSeconds) }),
|
|
239
|
+
...(config.removeBreaths === true ? { removeBreaths: true } : {}),
|
|
240
|
+
...(config.deEss === true ? { deEss: true } : {}),
|
|
239
241
|
imagePrompts,
|
|
240
242
|
...(config.imageScale === undefined ? {} : { imageScale: imageScaleForm(config.imageScale) }),
|
|
241
243
|
...(config.imageScenes === true ? { imageScenes: true } : {}),
|