@softure-ai/marketing-kit 0.18.0 → 0.20.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +27 -0
- package/README.md +27 -0
- package/dist/config/schema.d.ts +1 -0
- package/dist/config/schema.d.ts.map +1 -1
- package/dist/config/schema.js +1 -1
- package/dist/config/schema.js.map +1 -1
- package/dist/footage/clip-sound.d.ts +59 -4
- package/dist/footage/clip-sound.d.ts.map +1 -1
- package/dist/footage/clip-sound.js +58 -12
- package/dist/footage/clip-sound.js.map +1 -1
- package/dist/footage/declick.d.ts +35 -9
- package/dist/footage/declick.d.ts.map +1 -1
- package/dist/footage/declick.js +57 -88
- package/dist/footage/declick.js.map +1 -1
- package/dist/footage/film.d.ts +5 -2
- package/dist/footage/film.d.ts.map +1 -1
- package/dist/footage/film.js.map +1 -1
- package/dist/footage/isolate-ranges.d.ts +5 -0
- package/dist/footage/isolate-ranges.d.ts.map +1 -1
- package/dist/footage/isolate-ranges.js +4 -2
- package/dist/footage/isolate-ranges.js.map +1 -1
- package/dist/footage/mouth-clicks.d.ts +85 -0
- package/dist/footage/mouth-clicks.d.ts.map +1 -0
- package/dist/footage/mouth-clicks.js +207 -0
- package/dist/footage/mouth-clicks.js.map +1 -0
- package/dist/footage/pipeline.d.ts +11 -0
- package/dist/footage/pipeline.d.ts.map +1 -1
- package/dist/footage/pipeline.js +15 -9
- package/dist/footage/pipeline.js.map +1 -1
- package/dist/footage/sound-analysis.d.ts +25 -0
- package/dist/footage/sound-analysis.d.ts.map +1 -0
- package/dist/footage/sound-analysis.js +95 -0
- package/dist/footage/sound-analysis.js.map +1 -0
- package/dist/index.d.ts +3 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +3 -2
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
- package/schema/marketing.schema.json +3 -2
- package/src/config/schema.ts +1 -1
- package/src/footage/clip-sound.ts +95 -15
- package/src/footage/declick.ts +64 -97
- package/src/footage/film.ts +5 -2
- package/src/footage/isolate-ranges.ts +9 -3
- package/src/footage/mouth-clicks.ts +224 -0
- package/src/footage/pipeline.ts +29 -10
- package/src/footage/sound-analysis.ts +104 -0
- package/src/index.ts +9 -0
|
@@ -0,0 +1,224 @@
|
|
|
1
|
+
import { getBand, getFrameEnergies, getRunningMedian, getSquareSums, toPowerDb, type TimeSpan } from "./sound-analysis.js";
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* The mouth-click detector shared by moving a segment's cut past a click (stage A, issue #1182) and the declick
|
|
5
|
+
* stage's `repair` level (stage B, issue #1183), so a click that moves a cut and a click that is repaired are the same
|
|
6
|
+
* thing. Its input is the speech as the chain hears it: the clips' sound after the -23 LUFS pre-gain.
|
|
7
|
+
*
|
|
8
|
+
* - **An impulse** lasts 1 to 25 ms, its 2-10 kHz energy rises more than 12 dB above the band's ±50 ms running median,
|
|
9
|
+
* it is not periodic (the band's normalized autocorrelation stays under 0.3 at the pitch lags, 2.5 to 15 ms, over a
|
|
10
|
+
* 30 ms window centred on it) and it is over within its length (the 10 ms after it at least 12 dB under its peak, so
|
|
11
|
+
* the onset of a breath or a fricative is not one).
|
|
12
|
+
* - **The speech protection zone:** each word's bound on the real sound (the earlier start and the later end of the
|
|
13
|
+
* transcript's bound and the point where the 10 ms energy falls 20 dB under the word's peak), widened by 80 ms; and
|
|
14
|
+
* every stretch whose 10 ms energy is above -45 dBFS, measured with the impulses' own samples left out (a click's
|
|
15
|
+
* own energy must not protect it).
|
|
16
|
+
*/
|
|
17
|
+
export const MOUTH_CLICK_RULES = {
|
|
18
|
+
/** The shortest impulse, seconds. */
|
|
19
|
+
minSeconds: 0.001,
|
|
20
|
+
/** The longest impulse, seconds. */
|
|
21
|
+
maxSeconds: 0.025,
|
|
22
|
+
/** The band the impulse is measured in. */
|
|
23
|
+
bandLowHz: 2000,
|
|
24
|
+
bandHighHz: 10_000,
|
|
25
|
+
/** How far an impulse's band energy rises above the band's running median, dB. */
|
|
26
|
+
jumpDb: 12,
|
|
27
|
+
/** The running median's reach on each side, seconds. */
|
|
28
|
+
medianSeconds: 0.05,
|
|
29
|
+
/** The highest normalized autocorrelation at the pitch lags an impulse may show. */
|
|
30
|
+
maxCorrelation: 0.3,
|
|
31
|
+
/** The pitch lags, seconds. */
|
|
32
|
+
minLagSeconds: 0.0025,
|
|
33
|
+
maxLagSeconds: 0.015,
|
|
34
|
+
/** The window the autocorrelation is measured over, centred on the impulse, seconds. */
|
|
35
|
+
correlationSeconds: 0.03,
|
|
36
|
+
/** The stretch after an impulse that must be quieter than its peak, seconds, and by how much, dB. */
|
|
37
|
+
afterSeconds: 0.01,
|
|
38
|
+
afterDropDb: 12,
|
|
39
|
+
/** The energy window of the word bounds and the loudness rule, seconds. */
|
|
40
|
+
energySeconds: 0.01,
|
|
41
|
+
/** A word's sound lasts while its 10 ms energy stays within this many dB of the word's peak. */
|
|
42
|
+
wordDropDb: 20,
|
|
43
|
+
/** The guard on each side of a word's bound, seconds. */
|
|
44
|
+
wordGuardSeconds: 0.08,
|
|
45
|
+
/** Nothing whose 10 ms energy is above this is a click to act on, dBFS (mean square). */
|
|
46
|
+
loudDbfs: -45,
|
|
47
|
+
} as const;
|
|
48
|
+
|
|
49
|
+
const FRAME_SECONDS = 0.002;
|
|
50
|
+
const HOP_SECONDS = 0.001;
|
|
51
|
+
/** The step the word bounds and the loudness rule are measured at. */
|
|
52
|
+
const STEP_SECONDS = 0.001;
|
|
53
|
+
/** An impulse reaches this far past its detected frames, for the band filter's delay and the frames' edges. */
|
|
54
|
+
const IMPULSE_MARGIN = 0.001;
|
|
55
|
+
/** A frame quieter than this (mean square, dB) is never an impulse: digital silence with a stray bit is not a click. */
|
|
56
|
+
const BAND_FLOOR_DB = -90;
|
|
57
|
+
|
|
58
|
+
/** The highest normalized autocorrelation of `band` at the pitch lags over the window centred on `center`. */
|
|
59
|
+
function getPeakCorrelation(band: Float64Array, center: number, sampleRate: number): number {
|
|
60
|
+
const half = Math.round((MOUTH_CLICK_RULES.correlationSeconds * sampleRate) / 2);
|
|
61
|
+
const from = Math.max(0, center - half);
|
|
62
|
+
const to = Math.min(band.length, center + half);
|
|
63
|
+
const minLag = Math.max(1, Math.round(MOUTH_CLICK_RULES.minLagSeconds * sampleRate));
|
|
64
|
+
const maxLag = Math.min(Math.round(MOUTH_CLICK_RULES.maxLagSeconds * sampleRate), to - from - 1);
|
|
65
|
+
let peak = 0;
|
|
66
|
+
for (let lag = minLag; lag <= maxLag; lag++) {
|
|
67
|
+
let product = 0;
|
|
68
|
+
let head = 0;
|
|
69
|
+
let tail = 0;
|
|
70
|
+
for (let index = from; index + lag < to; index++) {
|
|
71
|
+
const a = band[index] ?? 0;
|
|
72
|
+
const b = band[index + lag] ?? 0;
|
|
73
|
+
product += a * b;
|
|
74
|
+
head += a * a;
|
|
75
|
+
tail += b * b;
|
|
76
|
+
}
|
|
77
|
+
if (head > 0 && tail > 0) peak = Math.max(peak, product / Math.sqrt(head * tail));
|
|
78
|
+
}
|
|
79
|
+
return peak;
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
/**
|
|
83
|
+
* The impulses of a mono signal by `MOUTH_CLICK_RULES` (length, band jump, periodicity, end), anywhere in it. Times in
|
|
84
|
+
* seconds from sample 0, widened by a millisecond on each side.
|
|
85
|
+
*/
|
|
86
|
+
export function findClickCandidates(mono: Float32Array, sampleRate: number): TimeSpan[] {
|
|
87
|
+
const seconds = mono.length / sampleRate;
|
|
88
|
+
const band = getBand({ mono, sampleRate, lowHz: MOUTH_CLICK_RULES.bandLowHz, highHz: MOUTH_CLICK_RULES.bandHighHz });
|
|
89
|
+
const sums = getSquareSums(band);
|
|
90
|
+
const frameLength = Math.max(1, Math.round(FRAME_SECONDS * sampleRate));
|
|
91
|
+
const hop = Math.max(1, Math.round(HOP_SECONDS * sampleRate));
|
|
92
|
+
const after = Math.round(MOUTH_CLICK_RULES.afterSeconds * sampleRate);
|
|
93
|
+
const energies = getFrameEnergies(band, frameLength, hop);
|
|
94
|
+
const medians = getRunningMedian(energies, Math.round(MOUTH_CLICK_RULES.medianSeconds / HOP_SECONDS));
|
|
95
|
+
const candidates: TimeSpan[] = [];
|
|
96
|
+
const checkRun = (first: number, last: number): void => {
|
|
97
|
+
const start = (first * hop) / sampleRate;
|
|
98
|
+
const end = (last * hop + frameLength) / sampleRate;
|
|
99
|
+
if (end - start < MOUTH_CLICK_RULES.minSeconds || end - start > MOUTH_CLICK_RULES.maxSeconds) return;
|
|
100
|
+
const span = { start: Math.max(0, start - IMPULSE_MARGIN), end: Math.min(seconds, end + IMPULSE_MARGIN) };
|
|
101
|
+
let peakDb = -Infinity;
|
|
102
|
+
for (let frame = first; frame <= last; frame++) peakDb = Math.max(peakDb, energies[frame] ?? -Infinity);
|
|
103
|
+
const afterFrom = Math.round(span.end * sampleRate);
|
|
104
|
+
const afterTo = Math.min(band.length, afterFrom + after);
|
|
105
|
+
if (afterTo > afterFrom && toPowerDb(((sums[afterTo] ?? 0) - (sums[afterFrom] ?? 0)) / (afterTo - afterFrom)) > peakDb - MOUTH_CLICK_RULES.afterDropDb) return;
|
|
106
|
+
const center = Math.round(((span.start + span.end) / 2) * sampleRate);
|
|
107
|
+
if (getPeakCorrelation(band, center, sampleRate) >= MOUTH_CLICK_RULES.maxCorrelation) return;
|
|
108
|
+
candidates.push(span);
|
|
109
|
+
};
|
|
110
|
+
let runStart = -1;
|
|
111
|
+
for (let frame = 0; frame < energies.length; frame++) {
|
|
112
|
+
const energy = energies[frame] ?? 0;
|
|
113
|
+
const isJump = energy > BAND_FLOOR_DB && energy - (medians[frame] ?? 0) > MOUTH_CLICK_RULES.jumpDb;
|
|
114
|
+
if (isJump && runStart === -1) runStart = frame;
|
|
115
|
+
else if (!isJump && runStart !== -1) {
|
|
116
|
+
checkRun(runStart, frame - 1);
|
|
117
|
+
runStart = -1;
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
if (runStart !== -1) checkRun(runStart, energies.length - 1);
|
|
121
|
+
return candidates;
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
/** The analysis of one mono stretch in its own time (sample 0 at 0 s). */
|
|
125
|
+
export interface MouthClickAnalysis {
|
|
126
|
+
/** Every impulse by the rule, protected or not. */
|
|
127
|
+
candidates: TimeSpan[];
|
|
128
|
+
/** Each word's bound on the real sound (before the guard). */
|
|
129
|
+
bounds: TimeSpan[];
|
|
130
|
+
/** True when the span touches a word's bound with its guard, or a 10 ms energy above the loudness rule. */
|
|
131
|
+
isProtected: (span: TimeSpan) => boolean;
|
|
132
|
+
/** True when the stretch holds no 10 ms window above the loudness rule, the impulses' samples left out. */
|
|
133
|
+
isQuiet: (stretch: TimeSpan) => boolean;
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
/** The detector over a mono stretch whose words are in the stretch's own time. */
|
|
137
|
+
export function analyzeMouthClicks(mono: Float32Array, sampleRate: number, words: readonly TimeSpan[]): MouthClickAnalysis {
|
|
138
|
+
const length = mono.length;
|
|
139
|
+
const candidates = findClickCandidates(mono, sampleRate);
|
|
140
|
+
const sums = getSquareSums(mono);
|
|
141
|
+
const masked = Float64Array.from(mono);
|
|
142
|
+
const counts = new Float64Array(length + 1);
|
|
143
|
+
const isMasked = new Uint8Array(length);
|
|
144
|
+
for (const candidate of candidates) {
|
|
145
|
+
for (let index = Math.max(0, Math.floor(candidate.start * sampleRate)); index < Math.min(length, Math.ceil(candidate.end * sampleRate)); index++) {
|
|
146
|
+
isMasked[index] = 1;
|
|
147
|
+
masked[index] = 0;
|
|
148
|
+
}
|
|
149
|
+
}
|
|
150
|
+
for (let index = 0; index < length; index++) counts[index + 1] = (counts[index] ?? 0) + (isMasked[index] === 1 ? 0 : 1);
|
|
151
|
+
const maskedSums = getSquareSums(masked);
|
|
152
|
+
const window = Math.max(1, Math.round(MOUTH_CLICK_RULES.energySeconds * sampleRate));
|
|
153
|
+
const step = Math.max(1, Math.round(STEP_SECONDS * sampleRate));
|
|
154
|
+
const clamp = (index: number): number => Math.min(length, Math.max(0, index));
|
|
155
|
+
const getDb = (from: number, to: number): number => {
|
|
156
|
+
const [a, b] = [clamp(from), clamp(to)];
|
|
157
|
+
return b > a ? toPowerDb(((sums[b] ?? 0) - (sums[a] ?? 0)) / (b - a)) : -Infinity;
|
|
158
|
+
};
|
|
159
|
+
const getMaskedDb = (from: number, to: number): number => {
|
|
160
|
+
const [a, b] = [clamp(from), clamp(to)];
|
|
161
|
+
const count = (counts[b] ?? 0) - (counts[a] ?? 0);
|
|
162
|
+
return count > 0 ? toPowerDb(((maskedSums[b] ?? 0) - (maskedSums[a] ?? 0)) / count) : -Infinity;
|
|
163
|
+
};
|
|
164
|
+
const getBound = (word: TimeSpan): TimeSpan => {
|
|
165
|
+
const from = clamp(Math.round(word.start * sampleRate));
|
|
166
|
+
const to = clamp(Math.round(word.end * sampleRate));
|
|
167
|
+
if (to <= from) return { ...word };
|
|
168
|
+
let peakDb = -Infinity;
|
|
169
|
+
for (let index = from; index <= Math.max(from, to - window); index += step) peakDb = Math.max(peakDb, getDb(index, Math.min(to, index + window)));
|
|
170
|
+
const threshold = peakDb - MOUTH_CLICK_RULES.wordDropDb;
|
|
171
|
+
// The start walks back while the 10 ms before it hold the word's sound, the end forward while the 10 ms after it do.
|
|
172
|
+
let start = from;
|
|
173
|
+
while (start > 0 && getDb(start - window, start) >= threshold) start = clamp(start - step);
|
|
174
|
+
let end = to;
|
|
175
|
+
while (end < length && getDb(end, end + window) >= threshold) end = clamp(end + step);
|
|
176
|
+
return { start: Math.min(word.start, start / sampleRate), end: Math.max(word.end, end / sampleRate) };
|
|
177
|
+
};
|
|
178
|
+
const bounds = [...words].filter((word) => word.end > word.start).sort((a, b) => a.start - b.start).map(getBound);
|
|
179
|
+
const isLoud = (span: TimeSpan): boolean => {
|
|
180
|
+
const from = Math.floor(span.start * sampleRate);
|
|
181
|
+
const to = Math.ceil(span.end * sampleRate);
|
|
182
|
+
for (let center = from; ; center = Math.min(to, center + step)) {
|
|
183
|
+
if (getMaskedDb(center - (window >> 1), center + window - (window >> 1)) > MOUTH_CLICK_RULES.loudDbfs) return true;
|
|
184
|
+
if (center >= to) return false;
|
|
185
|
+
}
|
|
186
|
+
};
|
|
187
|
+
const guard = MOUTH_CLICK_RULES.wordGuardSeconds;
|
|
188
|
+
return {
|
|
189
|
+
candidates,
|
|
190
|
+
bounds,
|
|
191
|
+
isProtected: (span) => bounds.some((bound) => span.start < bound.end + guard && span.end > bound.start - guard) || isLoud(span),
|
|
192
|
+
isQuiet: (stretch) => stretch.end <= 0 || stretch.start >= length / sampleRate || !isLoud({ start: Math.max(0, stretch.start), end: Math.min(length / sampleRate, stretch.end) }),
|
|
193
|
+
};
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
/** What `findMouthClicks` found, in the film's time (the stretch's `offset` added). */
|
|
197
|
+
export interface MouthClicks {
|
|
198
|
+
/** The impulses outside the speech protection zone, in time order. */
|
|
199
|
+
clicks: TimeSpan[];
|
|
200
|
+
/** Each word's bound on the real sound (before the ±80 ms guard), in time order. */
|
|
201
|
+
words: TimeSpan[];
|
|
202
|
+
/** True when the stretch holds no 10 ms window above -45 dBFS, the impulses' samples left out. */
|
|
203
|
+
isQuiet: (stretch: TimeSpan) => boolean;
|
|
204
|
+
}
|
|
205
|
+
|
|
206
|
+
/**
|
|
207
|
+
* Mouth clicks of a mono stretch whose sample 0 plays at `offset` seconds, outside the speech protection zone
|
|
208
|
+
* (`MOUTH_CLICK_RULES`); `words` are in the same time as `offset`. No audio is changed: an app can move a segment's
|
|
209
|
+
* cut past a click before its first word or after its last (issue #1182).
|
|
210
|
+
*/
|
|
211
|
+
export function findMouthClicks(options: { mono: Float32Array; sampleRate: number; offset: number; words: readonly TimeSpan[] }): MouthClicks {
|
|
212
|
+
const { mono, sampleRate, offset } = options;
|
|
213
|
+
const shift = (span: TimeSpan, by: number): TimeSpan => ({ start: span.start + by, end: span.end + by });
|
|
214
|
+
const analysis = analyzeMouthClicks(
|
|
215
|
+
mono,
|
|
216
|
+
sampleRate,
|
|
217
|
+
options.words.map((word) => shift(word, -offset)),
|
|
218
|
+
);
|
|
219
|
+
return {
|
|
220
|
+
clicks: analysis.candidates.filter((candidate) => !analysis.isProtected(candidate)).map((candidate) => shift(candidate, offset)),
|
|
221
|
+
words: analysis.bounds.map((bound) => shift(bound, offset)),
|
|
222
|
+
isQuiet: (stretch) => analysis.isQuiet(shift(stretch, -offset)),
|
|
223
|
+
};
|
|
224
|
+
}
|
package/src/footage/pipeline.ts
CHANGED
|
@@ -12,7 +12,7 @@ import { copyFonts } from "../render/render.js";
|
|
|
12
12
|
import type { TtsResult } from "../voice/provider.js";
|
|
13
13
|
import type { TimedWord } from "../voice/voiceover.js";
|
|
14
14
|
import { checkMusicFileType, describeAudioProbe, probeAudio } from "./audio.js";
|
|
15
|
-
import { prepareClipSounds } from "./clip-sound.js";
|
|
15
|
+
import { prepareClipSounds, type SpeechPeaks, type SpeechPreGain } from "./clip-sound.js";
|
|
16
16
|
import { readCachedTranscript, writeCacheFile, writeTranscript } from "./cache.js";
|
|
17
17
|
import { composeFootage, getFootageFilmSeconds, getFootageSfxCues } from "./compose.js";
|
|
18
18
|
import { findMissingCaptionGlyphs, getCaptionBounds, getFootageCaptionLayout } from "./captions.js";
|
|
@@ -145,6 +145,11 @@ export interface FootageRunOptions {
|
|
|
145
145
|
* person to run again. Every failure also gets one `log` line with the step and its full command.
|
|
146
146
|
*/
|
|
147
147
|
onFfmpegFailure?: FfmpegFailureHook;
|
|
148
|
+
/**
|
|
149
|
+
* How the clip sound's pre-gain treats a true peak above `SPEECH_PRE_GAIN.peakCeiling` (issue #1181): `ceiling`
|
|
150
|
+
* (default) stops the gain there; `float` reaches the target and writes the clip sound as 32-bit float.
|
|
151
|
+
*/
|
|
152
|
+
speechPeaks?: SpeechPeaks;
|
|
148
153
|
}
|
|
149
154
|
|
|
150
155
|
export interface PreparedClip {
|
|
@@ -173,6 +178,11 @@ export type TranscribeOutcome =
|
|
|
173
178
|
* optional so an app's own `ready` outcome still compiles.
|
|
174
179
|
*/
|
|
175
180
|
speechGainDb?: Record<string, number>;
|
|
181
|
+
/**
|
|
182
|
+
* What each clip's pre-gain reached (`ClipSound.preGain`, issue #1181): `held_by_peak_ceiling` names a clip the
|
|
183
|
+
* speech chain hears under the target. Always answered; optional so an app's own `ready` outcome still compiles.
|
|
184
|
+
*/
|
|
185
|
+
speechPreGain?: Record<string, SpeechPreGain>;
|
|
176
186
|
};
|
|
177
187
|
|
|
178
188
|
export function getFrame(film: Pick<FootageFilm, "format">): Frame {
|
|
@@ -301,20 +311,29 @@ async function decodeIsolatedVoice(cached: string, clipId: string, buildDir: str
|
|
|
301
311
|
return output;
|
|
302
312
|
}
|
|
303
313
|
|
|
304
|
-
/** Each clip's lossless, pre-gained sound (`prepareClipSounds`)
|
|
305
|
-
async function getClipSoundFiles(
|
|
306
|
-
|
|
307
|
-
|
|
314
|
+
/** Each clip's lossless, pre-gained sound (`prepareClipSounds`), its pre-gain in dB and what it reached, by clip id. */
|
|
315
|
+
async function getClipSoundFiles(
|
|
316
|
+
clips: PreparedClip[],
|
|
317
|
+
options: Pick<FootageRunOptions, "buildDir" | "signal" | "speechPeaks" | "log">,
|
|
318
|
+
): Promise<{ speech: Record<string, string>; speechGainDb: Record<string, number>; speechPreGain: Record<string, SpeechPreGain> }> {
|
|
319
|
+
const { buildDir, signal, speechPeaks, log } = options;
|
|
320
|
+
const sounds = Object.entries(await prepareClipSounds(clips, { buildDir, log, ...(signal === undefined ? {} : { signal }), ...(speechPeaks === undefined ? {} : { peaks: speechPeaks }) }));
|
|
321
|
+
return {
|
|
322
|
+
speech: Object.fromEntries(sounds.map(([id, sound]) => [id, sound.file])),
|
|
323
|
+
speechGainDb: Object.fromEntries(sounds.map(([id, sound]) => [id, sound.gainDb])),
|
|
324
|
+
speechPreGain: Object.fromEntries(sounds.map(([id, sound]) => [id, sound.preGain])),
|
|
325
|
+
};
|
|
308
326
|
}
|
|
309
327
|
|
|
310
328
|
/** The free transcripts of `--placeholder`, written under `placeholder/` for a look; the speech is each clip's lossless sound. */
|
|
311
|
-
async function transcribePlaceholder(clips: PreparedClip[],
|
|
329
|
+
async function transcribePlaceholder(clips: PreparedClip[], options: FootageRunOptions): Promise<TranscribeOutcome> {
|
|
330
|
+
const { buildDir } = options;
|
|
312
331
|
const words: Record<string, TimedWord[]> = {};
|
|
313
332
|
for (const clip of clips) {
|
|
314
333
|
words[clip.clip.id] = clip.clip.role === "talk" ? makePlaceholderTranscript(clip.clip.script, clip.duration) : [];
|
|
315
334
|
writeTranscript(join(buildDir, "placeholder", `${clip.clip.id}.transcript.json`), words[clip.clip.id] ?? []);
|
|
316
335
|
}
|
|
317
|
-
return { kind: "ready", words, ...(await getClipSoundFiles(clips,
|
|
336
|
+
return { kind: "ready", words, ...(await getClipSoundFiles(clips, options)) };
|
|
318
337
|
}
|
|
319
338
|
|
|
320
339
|
/** The paid music step, when the film's music still has to be bought; null when it is cached or free. */
|
|
@@ -337,7 +356,7 @@ export function transcribeFootage(film: FootageFilm, clips: PreparedClip[], opti
|
|
|
337
356
|
/** The body of `transcribeFootage`, inside its ffmpeg failure reporting. */
|
|
338
357
|
async function runTranscribeFootage(film: FootageFilm, clips: PreparedClip[], options: FootageRunOptions): Promise<FootageResult<TranscribeOutcome>> {
|
|
339
358
|
const { buildDir, providers, log, signal } = options;
|
|
340
|
-
if (options.isPlaceholder) return ok(await transcribePlaceholder(clips,
|
|
359
|
+
if (options.isPlaceholder) return ok(await transcribePlaceholder(clips, options));
|
|
341
360
|
const keyTerms = getFilmKeyTerms(film);
|
|
342
361
|
if (!keyTerms.ok) return err(`footage film ${film.id}: ${keyTerms.errors.join(" ")}`, { kind: "input" });
|
|
343
362
|
const sttKeyTerms = keyTerms.terms.length === 0 ? {} : { keyTerms: keyTerms.terms };
|
|
@@ -363,7 +382,7 @@ async function runTranscribeFootage(film: FootageFilm, clips: PreparedClip[], op
|
|
|
363
382
|
const words: Record<string, TimedWord[]> = {};
|
|
364
383
|
for (const clip of clips) words[clip.clip.id] = [];
|
|
365
384
|
// Every clip's sound, lossless and pre-gained: what the isolator, the speech to text and the speech track hear (#1139).
|
|
366
|
-
const { speech, speechGainDb } = await getClipSoundFiles(clips,
|
|
385
|
+
const { speech, speechGainDb, speechPreGain } = await getClipSoundFiles(clips, options);
|
|
367
386
|
const isolating = plan.filter(({ isolated }) => isolated !== null && !existsSync(isolated.path));
|
|
368
387
|
const transcribing = plan.filter(({ transcript }) => !existsSync(transcript.path));
|
|
369
388
|
const isolateProgress = createStageProgress(options.onProgress, "isolate", isolating.map(({ clip }) => ({ item: clip.clip.id, seconds: clip.duration })));
|
|
@@ -416,7 +435,7 @@ async function runTranscribeFootage(film: FootageFilm, clips: PreparedClip[], op
|
|
|
416
435
|
transcribeProgress(index, clip.duration);
|
|
417
436
|
words[id] = result.value.words;
|
|
418
437
|
}
|
|
419
|
-
return ok({ kind: "ready", words, speech, speechGainDb });
|
|
438
|
+
return ok({ kind: "ready", words, speech, speechGainDb, speechPreGain });
|
|
420
439
|
}
|
|
421
440
|
|
|
422
441
|
/** The talking clips' sentences with their ids (`s1`…), as `buildEdl` names them, for choosing what to keep. */
|
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Sound analysis shared by the speech declick stage and the mouth-click detector (issues #1170, #1182, #1183): a
|
|
3
|
+
* fourth-order band pass, frame energies and a running median.
|
|
4
|
+
*/
|
|
5
|
+
|
|
6
|
+
/** A stretch of time in seconds. */
|
|
7
|
+
export interface TimeSpan {
|
|
8
|
+
start: number;
|
|
9
|
+
end: number;
|
|
10
|
+
}
|
|
11
|
+
|
|
12
|
+
/** One second-order section (RBJ cookbook), run in place over the samples. */
|
|
13
|
+
function runBiquad(samples: Float64Array, kind: "highpass" | "lowpass", frequency: number, sampleRate: number): void {
|
|
14
|
+
const w0 = (2 * Math.PI * Math.min(frequency, sampleRate * 0.45)) / sampleRate;
|
|
15
|
+
const cos = Math.cos(w0);
|
|
16
|
+
const alpha = Math.sin(w0) / (2 * Math.SQRT1_2);
|
|
17
|
+
const a0 = 1 + alpha;
|
|
18
|
+
const b0 = (kind === "lowpass" ? (1 - cos) / 2 : (1 + cos) / 2) / a0;
|
|
19
|
+
const b1 = (kind === "lowpass" ? 1 - cos : -(1 + cos)) / a0;
|
|
20
|
+
const b2 = b0;
|
|
21
|
+
const a1 = (-2 * cos) / a0;
|
|
22
|
+
const a2 = (1 - alpha) / a0;
|
|
23
|
+
let x1 = 0;
|
|
24
|
+
let x2 = 0;
|
|
25
|
+
let y1 = 0;
|
|
26
|
+
let y2 = 0;
|
|
27
|
+
for (let index = 0; index < samples.length; index++) {
|
|
28
|
+
const x = samples[index] ?? 0;
|
|
29
|
+
const y = b0 * x + b1 * x1 + b2 * x2 - a1 * y1 - a2 * y2;
|
|
30
|
+
x2 = x1;
|
|
31
|
+
x1 = x;
|
|
32
|
+
y2 = y1;
|
|
33
|
+
y1 = y;
|
|
34
|
+
samples[index] = y;
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
/** The band of the signal between `lowHz` and `highHz`: two high-pass and two low-pass sections (fourth order on each side). */
|
|
39
|
+
export function getBand(options: { mono: Float32Array; sampleRate: number; lowHz: number; highHz: number }): Float64Array {
|
|
40
|
+
const { mono, sampleRate, lowHz, highHz } = options;
|
|
41
|
+
const band = Float64Array.from(mono);
|
|
42
|
+
runBiquad(band, "highpass", lowHz, sampleRate);
|
|
43
|
+
runBiquad(band, "highpass", lowHz, sampleRate);
|
|
44
|
+
runBiquad(band, "lowpass", highHz, sampleRate);
|
|
45
|
+
runBiquad(band, "lowpass", highHz, sampleRate);
|
|
46
|
+
return band;
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
/** The running sums of the squared samples: `sums[i]` is the energy of samples 0 to i - 1. */
|
|
50
|
+
export function getSquareSums(samples: ArrayLike<number>): Float64Array {
|
|
51
|
+
const sums = new Float64Array(samples.length + 1);
|
|
52
|
+
for (let index = 0; index < samples.length; index++) {
|
|
53
|
+
const value = samples[index] ?? 0;
|
|
54
|
+
sums[index + 1] = (sums[index] ?? 0) + value * value;
|
|
55
|
+
}
|
|
56
|
+
return sums;
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
/** A mean square in dB (10·log10), with a floor so silence stays finite. */
|
|
60
|
+
export const toPowerDb = (meanSquare: number): number => 10 * Math.log10(Math.max(meanSquare, 1e-20));
|
|
61
|
+
|
|
62
|
+
/** Each frame's mean square in dB, frames of `frameLength` samples every `hop` samples. */
|
|
63
|
+
export function getFrameEnergies(band: Float64Array, frameLength: number, hop: number): Float64Array {
|
|
64
|
+
const sums = getSquareSums(band);
|
|
65
|
+
const count = band.length < frameLength ? 0 : Math.floor((band.length - frameLength) / hop) + 1;
|
|
66
|
+
const energies = new Float64Array(count);
|
|
67
|
+
for (let frame = 0; frame < count; frame++) {
|
|
68
|
+
const from = frame * hop;
|
|
69
|
+
energies[frame] = toPowerDb(((sums[from + frameLength] ?? 0) - (sums[from] ?? 0)) / frameLength);
|
|
70
|
+
}
|
|
71
|
+
return energies;
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
/** The index of the first entry of the sorted list not below `value`. */
|
|
75
|
+
function findInsertIndex(sorted: number[], value: number): number {
|
|
76
|
+
let low = 0;
|
|
77
|
+
let high = sorted.length;
|
|
78
|
+
while (low < high) {
|
|
79
|
+
const middle = (low + high) >> 1;
|
|
80
|
+
if ((sorted[middle] ?? 0) < value) low = middle + 1;
|
|
81
|
+
else high = middle;
|
|
82
|
+
}
|
|
83
|
+
return low;
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
/** The running median of the values over `reach` entries on each side (fewer at the ends). */
|
|
87
|
+
export function getRunningMedian(values: Float64Array, reach: number): Float64Array {
|
|
88
|
+
const medians = new Float64Array(values.length);
|
|
89
|
+
const window: number[] = [];
|
|
90
|
+
let from = 0;
|
|
91
|
+
let to = 0;
|
|
92
|
+
for (let index = 0; index < values.length; index++) {
|
|
93
|
+
const wantedTo = Math.min(values.length, index + reach + 1);
|
|
94
|
+
for (; to < wantedTo; to++) {
|
|
95
|
+
const value = values[to] ?? 0;
|
|
96
|
+
window.splice(findInsertIndex(window, value), 0, value);
|
|
97
|
+
}
|
|
98
|
+
const wantedFrom = Math.max(0, index - reach);
|
|
99
|
+
for (; from < wantedFrom; from++) window.splice(findInsertIndex(window, values[from] ?? 0), 1);
|
|
100
|
+
const middle = window.length >> 1;
|
|
101
|
+
medians[index] = window.length % 2 === 1 ? (window[middle] ?? 0) : ((window[middle - 1] ?? 0) + (window[middle] ?? 0)) / 2;
|
|
102
|
+
}
|
|
103
|
+
return medians;
|
|
104
|
+
}
|
package/src/index.ts
CHANGED
|
@@ -252,6 +252,7 @@ export {
|
|
|
252
252
|
DECLICK_CROSSFADE,
|
|
253
253
|
DECLICK_LEVELS,
|
|
254
254
|
DECLICK_MAX_IMPULSE,
|
|
255
|
+
DECLICK_REPAIR_RULES,
|
|
255
256
|
DECLICK_SETTINGS,
|
|
256
257
|
DECLICK_WORD_GUARD,
|
|
257
258
|
buildDeclickDecodeArgs,
|
|
@@ -260,6 +261,7 @@ export {
|
|
|
260
261
|
declickSpeech,
|
|
261
262
|
describeDeclick,
|
|
262
263
|
findImpulses,
|
|
264
|
+
findRepairs,
|
|
263
265
|
getArPolynomial,
|
|
264
266
|
getRepairZones,
|
|
265
267
|
getRunningMedian,
|
|
@@ -269,8 +271,10 @@ export {
|
|
|
269
271
|
type DeclickReport,
|
|
270
272
|
type DeclickSettings,
|
|
271
273
|
type DeclickSpeechOptions,
|
|
274
|
+
type DetectorDeclickLevel,
|
|
272
275
|
type TimeSpan as DeclickSpan,
|
|
273
276
|
} from "./footage/declick.js";
|
|
277
|
+
export { MOUTH_CLICK_RULES, findMouthClicks, type MouthClicks } from "./footage/mouth-clicks.js";
|
|
274
278
|
export {
|
|
275
279
|
MIN_CLIP_SECONDS,
|
|
276
280
|
describeProbe,
|
|
@@ -464,10 +468,15 @@ export {
|
|
|
464
468
|
buildClipSoundMeasureArgs,
|
|
465
469
|
getAudioCodecArgs,
|
|
466
470
|
getClipSoundPath,
|
|
471
|
+
describeSpeechPreGain,
|
|
467
472
|
getPreGainDb,
|
|
473
|
+
getSpeechPreGain,
|
|
468
474
|
prepareClipSounds,
|
|
469
475
|
type ClipSound,
|
|
470
476
|
type ClipSoundInput,
|
|
477
|
+
type MeasuredPreGain,
|
|
478
|
+
type SpeechPeaks,
|
|
479
|
+
type SpeechPreGain,
|
|
471
480
|
} from "./footage/clip-sound.js";
|
|
472
481
|
export {
|
|
473
482
|
AUDIO_BITRATE_RANGE_KBPS,
|