@softure-ai/marketing-kit 0.19.0 → 0.21.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +23 -0
- package/README.md +23 -0
- package/dist/config/schema.d.ts +1 -0
- package/dist/config/schema.d.ts.map +1 -1
- package/dist/config/schema.js +1 -1
- package/dist/config/schema.js.map +1 -1
- package/dist/footage/declick.d.ts +35 -9
- package/dist/footage/declick.d.ts.map +1 -1
- package/dist/footage/declick.js +57 -88
- package/dist/footage/declick.js.map +1 -1
- package/dist/footage/film.d.ts +5 -2
- package/dist/footage/film.d.ts.map +1 -1
- package/dist/footage/film.js.map +1 -1
- package/dist/footage/mouth-clicks.d.ts +85 -0
- package/dist/footage/mouth-clicks.d.ts.map +1 -0
- package/dist/footage/mouth-clicks.js +207 -0
- package/dist/footage/mouth-clicks.js.map +1 -0
- package/dist/footage/sound-analysis.d.ts +25 -0
- package/dist/footage/sound-analysis.d.ts.map +1 -0
- package/dist/footage/sound-analysis.js +95 -0
- package/dist/footage/sound-analysis.js.map +1 -0
- package/dist/index.d.ts +2 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +2 -1
- package/dist/index.js.map +1 -1
- package/dist/og/index.d.ts +1 -1
- package/dist/og/index.d.ts.map +1 -1
- package/dist/og/index.js +1 -1
- package/dist/og/index.js.map +1 -1
- package/dist/og/photo-cover.d.ts +8 -0
- package/dist/og/photo-cover.d.ts.map +1 -1
- package/dist/og/photo-cover.js +50 -10
- package/dist/og/photo-cover.js.map +1 -1
- package/package.json +1 -1
- package/schema/marketing.schema.json +3 -2
- package/src/config/schema.ts +1 -1
- package/src/footage/declick.ts +64 -97
- package/src/footage/film.ts +5 -2
- package/src/footage/mouth-clicks.ts +224 -0
- package/src/footage/sound-analysis.ts +104 -0
- package/src/index.ts +4 -0
- package/src/og/index.ts +1 -0
- package/src/og/photo-cover.ts +68 -12
|
@@ -0,0 +1,224 @@
|
|
|
1
|
+
import { getBand, getFrameEnergies, getRunningMedian, getSquareSums, toPowerDb, type TimeSpan } from "./sound-analysis.js";
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* The mouth-click detector shared by moving a segment's cut past a click (stage A, issue #1182) and the declick
|
|
5
|
+
* stage's `repair` level (stage B, issue #1183), so a click that moves a cut and a click that is repaired are the same
|
|
6
|
+
* thing. Its input is the speech as the chain hears it: the clips' sound after the -23 LUFS pre-gain.
|
|
7
|
+
*
|
|
8
|
+
* - **An impulse** lasts 1 to 25 ms, its 2-10 kHz energy rises more than 12 dB above the band's ±50 ms running median,
|
|
9
|
+
* it is not periodic (the band's normalized autocorrelation stays under 0.3 at the pitch lags, 2.5 to 15 ms, over a
|
|
10
|
+
* 30 ms window centred on it) and it is over within its length (the 10 ms after it at least 12 dB under its peak, so
|
|
11
|
+
* the onset of a breath or a fricative is not one).
|
|
12
|
+
* - **The speech protection zone:** each word's bound on the real sound (the earlier start and the later end of the
|
|
13
|
+
* transcript's bound and the point where the 10 ms energy falls 20 dB under the word's peak), widened by 80 ms; and
|
|
14
|
+
* every stretch whose 10 ms energy is above -45 dBFS, measured with the impulses' own samples left out (a click's
|
|
15
|
+
* own energy must not protect it).
|
|
16
|
+
*/
|
|
17
|
+
export const MOUTH_CLICK_RULES = {
|
|
18
|
+
/** The shortest impulse, seconds. */
|
|
19
|
+
minSeconds: 0.001,
|
|
20
|
+
/** The longest impulse, seconds. */
|
|
21
|
+
maxSeconds: 0.025,
|
|
22
|
+
/** The band the impulse is measured in. */
|
|
23
|
+
bandLowHz: 2000,
|
|
24
|
+
bandHighHz: 10_000,
|
|
25
|
+
/** How far an impulse's band energy rises above the band's running median, dB. */
|
|
26
|
+
jumpDb: 12,
|
|
27
|
+
/** The running median's reach on each side, seconds. */
|
|
28
|
+
medianSeconds: 0.05,
|
|
29
|
+
/** The highest normalized autocorrelation at the pitch lags an impulse may show. */
|
|
30
|
+
maxCorrelation: 0.3,
|
|
31
|
+
/** The pitch lags, seconds. */
|
|
32
|
+
minLagSeconds: 0.0025,
|
|
33
|
+
maxLagSeconds: 0.015,
|
|
34
|
+
/** The window the autocorrelation is measured over, centred on the impulse, seconds. */
|
|
35
|
+
correlationSeconds: 0.03,
|
|
36
|
+
/** The stretch after an impulse that must be quieter than its peak, seconds, and by how much, dB. */
|
|
37
|
+
afterSeconds: 0.01,
|
|
38
|
+
afterDropDb: 12,
|
|
39
|
+
/** The energy window of the word bounds and the loudness rule, seconds. */
|
|
40
|
+
energySeconds: 0.01,
|
|
41
|
+
/** A word's sound lasts while its 10 ms energy stays within this many dB of the word's peak. */
|
|
42
|
+
wordDropDb: 20,
|
|
43
|
+
/** The guard on each side of a word's bound, seconds. */
|
|
44
|
+
wordGuardSeconds: 0.08,
|
|
45
|
+
/** Nothing whose 10 ms energy is above this is a click to act on, dBFS (mean square). */
|
|
46
|
+
loudDbfs: -45,
|
|
47
|
+
} as const;
|
|
48
|
+
|
|
49
|
+
const FRAME_SECONDS = 0.002;
|
|
50
|
+
const HOP_SECONDS = 0.001;
|
|
51
|
+
/** The step the word bounds and the loudness rule are measured at. */
|
|
52
|
+
const STEP_SECONDS = 0.001;
|
|
53
|
+
/** An impulse reaches this far past its detected frames, for the band filter's delay and the frames' edges. */
|
|
54
|
+
const IMPULSE_MARGIN = 0.001;
|
|
55
|
+
/** A frame quieter than this (mean square, dB) is never an impulse: digital silence with a stray bit is not a click. */
|
|
56
|
+
const BAND_FLOOR_DB = -90;
|
|
57
|
+
|
|
58
|
+
/** The highest normalized autocorrelation of `band` at the pitch lags over the window centred on `center`. */
|
|
59
|
+
function getPeakCorrelation(band: Float64Array, center: number, sampleRate: number): number {
|
|
60
|
+
const half = Math.round((MOUTH_CLICK_RULES.correlationSeconds * sampleRate) / 2);
|
|
61
|
+
const from = Math.max(0, center - half);
|
|
62
|
+
const to = Math.min(band.length, center + half);
|
|
63
|
+
const minLag = Math.max(1, Math.round(MOUTH_CLICK_RULES.minLagSeconds * sampleRate));
|
|
64
|
+
const maxLag = Math.min(Math.round(MOUTH_CLICK_RULES.maxLagSeconds * sampleRate), to - from - 1);
|
|
65
|
+
let peak = 0;
|
|
66
|
+
for (let lag = minLag; lag <= maxLag; lag++) {
|
|
67
|
+
let product = 0;
|
|
68
|
+
let head = 0;
|
|
69
|
+
let tail = 0;
|
|
70
|
+
for (let index = from; index + lag < to; index++) {
|
|
71
|
+
const a = band[index] ?? 0;
|
|
72
|
+
const b = band[index + lag] ?? 0;
|
|
73
|
+
product += a * b;
|
|
74
|
+
head += a * a;
|
|
75
|
+
tail += b * b;
|
|
76
|
+
}
|
|
77
|
+
if (head > 0 && tail > 0) peak = Math.max(peak, product / Math.sqrt(head * tail));
|
|
78
|
+
}
|
|
79
|
+
return peak;
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
/**
|
|
83
|
+
* The impulses of a mono signal by `MOUTH_CLICK_RULES` (length, band jump, periodicity, end), anywhere in it. Times in
|
|
84
|
+
* seconds from sample 0, widened by a millisecond on each side.
|
|
85
|
+
*/
|
|
86
|
+
export function findClickCandidates(mono: Float32Array, sampleRate: number): TimeSpan[] {
|
|
87
|
+
const seconds = mono.length / sampleRate;
|
|
88
|
+
const band = getBand({ mono, sampleRate, lowHz: MOUTH_CLICK_RULES.bandLowHz, highHz: MOUTH_CLICK_RULES.bandHighHz });
|
|
89
|
+
const sums = getSquareSums(band);
|
|
90
|
+
const frameLength = Math.max(1, Math.round(FRAME_SECONDS * sampleRate));
|
|
91
|
+
const hop = Math.max(1, Math.round(HOP_SECONDS * sampleRate));
|
|
92
|
+
const after = Math.round(MOUTH_CLICK_RULES.afterSeconds * sampleRate);
|
|
93
|
+
const energies = getFrameEnergies(band, frameLength, hop);
|
|
94
|
+
const medians = getRunningMedian(energies, Math.round(MOUTH_CLICK_RULES.medianSeconds / HOP_SECONDS));
|
|
95
|
+
const candidates: TimeSpan[] = [];
|
|
96
|
+
const checkRun = (first: number, last: number): void => {
|
|
97
|
+
const start = (first * hop) / sampleRate;
|
|
98
|
+
const end = (last * hop + frameLength) / sampleRate;
|
|
99
|
+
if (end - start < MOUTH_CLICK_RULES.minSeconds || end - start > MOUTH_CLICK_RULES.maxSeconds) return;
|
|
100
|
+
const span = { start: Math.max(0, start - IMPULSE_MARGIN), end: Math.min(seconds, end + IMPULSE_MARGIN) };
|
|
101
|
+
let peakDb = -Infinity;
|
|
102
|
+
for (let frame = first; frame <= last; frame++) peakDb = Math.max(peakDb, energies[frame] ?? -Infinity);
|
|
103
|
+
const afterFrom = Math.round(span.end * sampleRate);
|
|
104
|
+
const afterTo = Math.min(band.length, afterFrom + after);
|
|
105
|
+
if (afterTo > afterFrom && toPowerDb(((sums[afterTo] ?? 0) - (sums[afterFrom] ?? 0)) / (afterTo - afterFrom)) > peakDb - MOUTH_CLICK_RULES.afterDropDb) return;
|
|
106
|
+
const center = Math.round(((span.start + span.end) / 2) * sampleRate);
|
|
107
|
+
if (getPeakCorrelation(band, center, sampleRate) >= MOUTH_CLICK_RULES.maxCorrelation) return;
|
|
108
|
+
candidates.push(span);
|
|
109
|
+
};
|
|
110
|
+
let runStart = -1;
|
|
111
|
+
for (let frame = 0; frame < energies.length; frame++) {
|
|
112
|
+
const energy = energies[frame] ?? 0;
|
|
113
|
+
const isJump = energy > BAND_FLOOR_DB && energy - (medians[frame] ?? 0) > MOUTH_CLICK_RULES.jumpDb;
|
|
114
|
+
if (isJump && runStart === -1) runStart = frame;
|
|
115
|
+
else if (!isJump && runStart !== -1) {
|
|
116
|
+
checkRun(runStart, frame - 1);
|
|
117
|
+
runStart = -1;
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
if (runStart !== -1) checkRun(runStart, energies.length - 1);
|
|
121
|
+
return candidates;
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
/** The analysis of one mono stretch in its own time (sample 0 at 0 s). */
|
|
125
|
+
export interface MouthClickAnalysis {
|
|
126
|
+
/** Every impulse by the rule, protected or not. */
|
|
127
|
+
candidates: TimeSpan[];
|
|
128
|
+
/** Each word's bound on the real sound (before the guard). */
|
|
129
|
+
bounds: TimeSpan[];
|
|
130
|
+
/** True when the span touches a word's bound with its guard, or a 10 ms energy above the loudness rule. */
|
|
131
|
+
isProtected: (span: TimeSpan) => boolean;
|
|
132
|
+
/** True when the stretch holds no 10 ms window above the loudness rule, the impulses' samples left out. */
|
|
133
|
+
isQuiet: (stretch: TimeSpan) => boolean;
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
/** The detector over a mono stretch whose words are in the stretch's own time. */
|
|
137
|
+
export function analyzeMouthClicks(mono: Float32Array, sampleRate: number, words: readonly TimeSpan[]): MouthClickAnalysis {
|
|
138
|
+
const length = mono.length;
|
|
139
|
+
const candidates = findClickCandidates(mono, sampleRate);
|
|
140
|
+
const sums = getSquareSums(mono);
|
|
141
|
+
const masked = Float64Array.from(mono);
|
|
142
|
+
const counts = new Float64Array(length + 1);
|
|
143
|
+
const isMasked = new Uint8Array(length);
|
|
144
|
+
for (const candidate of candidates) {
|
|
145
|
+
for (let index = Math.max(0, Math.floor(candidate.start * sampleRate)); index < Math.min(length, Math.ceil(candidate.end * sampleRate)); index++) {
|
|
146
|
+
isMasked[index] = 1;
|
|
147
|
+
masked[index] = 0;
|
|
148
|
+
}
|
|
149
|
+
}
|
|
150
|
+
for (let index = 0; index < length; index++) counts[index + 1] = (counts[index] ?? 0) + (isMasked[index] === 1 ? 0 : 1);
|
|
151
|
+
const maskedSums = getSquareSums(masked);
|
|
152
|
+
const window = Math.max(1, Math.round(MOUTH_CLICK_RULES.energySeconds * sampleRate));
|
|
153
|
+
const step = Math.max(1, Math.round(STEP_SECONDS * sampleRate));
|
|
154
|
+
const clamp = (index: number): number => Math.min(length, Math.max(0, index));
|
|
155
|
+
const getDb = (from: number, to: number): number => {
|
|
156
|
+
const [a, b] = [clamp(from), clamp(to)];
|
|
157
|
+
return b > a ? toPowerDb(((sums[b] ?? 0) - (sums[a] ?? 0)) / (b - a)) : -Infinity;
|
|
158
|
+
};
|
|
159
|
+
const getMaskedDb = (from: number, to: number): number => {
|
|
160
|
+
const [a, b] = [clamp(from), clamp(to)];
|
|
161
|
+
const count = (counts[b] ?? 0) - (counts[a] ?? 0);
|
|
162
|
+
return count > 0 ? toPowerDb(((maskedSums[b] ?? 0) - (maskedSums[a] ?? 0)) / count) : -Infinity;
|
|
163
|
+
};
|
|
164
|
+
const getBound = (word: TimeSpan): TimeSpan => {
|
|
165
|
+
const from = clamp(Math.round(word.start * sampleRate));
|
|
166
|
+
const to = clamp(Math.round(word.end * sampleRate));
|
|
167
|
+
if (to <= from) return { ...word };
|
|
168
|
+
let peakDb = -Infinity;
|
|
169
|
+
for (let index = from; index <= Math.max(from, to - window); index += step) peakDb = Math.max(peakDb, getDb(index, Math.min(to, index + window)));
|
|
170
|
+
const threshold = peakDb - MOUTH_CLICK_RULES.wordDropDb;
|
|
171
|
+
// The start walks back while the 10 ms before it hold the word's sound, the end forward while the 10 ms after it do.
|
|
172
|
+
let start = from;
|
|
173
|
+
while (start > 0 && getDb(start - window, start) >= threshold) start = clamp(start - step);
|
|
174
|
+
let end = to;
|
|
175
|
+
while (end < length && getDb(end, end + window) >= threshold) end = clamp(end + step);
|
|
176
|
+
return { start: Math.min(word.start, start / sampleRate), end: Math.max(word.end, end / sampleRate) };
|
|
177
|
+
};
|
|
178
|
+
const bounds = [...words].filter((word) => word.end > word.start).sort((a, b) => a.start - b.start).map(getBound);
|
|
179
|
+
const isLoud = (span: TimeSpan): boolean => {
|
|
180
|
+
const from = Math.floor(span.start * sampleRate);
|
|
181
|
+
const to = Math.ceil(span.end * sampleRate);
|
|
182
|
+
for (let center = from; ; center = Math.min(to, center + step)) {
|
|
183
|
+
if (getMaskedDb(center - (window >> 1), center + window - (window >> 1)) > MOUTH_CLICK_RULES.loudDbfs) return true;
|
|
184
|
+
if (center >= to) return false;
|
|
185
|
+
}
|
|
186
|
+
};
|
|
187
|
+
const guard = MOUTH_CLICK_RULES.wordGuardSeconds;
|
|
188
|
+
return {
|
|
189
|
+
candidates,
|
|
190
|
+
bounds,
|
|
191
|
+
isProtected: (span) => bounds.some((bound) => span.start < bound.end + guard && span.end > bound.start - guard) || isLoud(span),
|
|
192
|
+
isQuiet: (stretch) => stretch.end <= 0 || stretch.start >= length / sampleRate || !isLoud({ start: Math.max(0, stretch.start), end: Math.min(length / sampleRate, stretch.end) }),
|
|
193
|
+
};
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
/** What `findMouthClicks` found, in the film's time (the stretch's `offset` added). */
|
|
197
|
+
export interface MouthClicks {
|
|
198
|
+
/** The impulses outside the speech protection zone, in time order. */
|
|
199
|
+
clicks: TimeSpan[];
|
|
200
|
+
/** Each word's bound on the real sound (before the ±80 ms guard), in time order. */
|
|
201
|
+
words: TimeSpan[];
|
|
202
|
+
/** True when the stretch holds no 10 ms window above -45 dBFS, the impulses' samples left out. */
|
|
203
|
+
isQuiet: (stretch: TimeSpan) => boolean;
|
|
204
|
+
}
|
|
205
|
+
|
|
206
|
+
/**
|
|
207
|
+
* Mouth clicks of a mono stretch whose sample 0 plays at `offset` seconds, outside the speech protection zone
|
|
208
|
+
* (`MOUTH_CLICK_RULES`); `words` are in the same time as `offset`. No audio is changed: an app can move a segment's
|
|
209
|
+
* cut past a click before its first word or after its last (issue #1182).
|
|
210
|
+
*/
|
|
211
|
+
export function findMouthClicks(options: { mono: Float32Array; sampleRate: number; offset: number; words: readonly TimeSpan[] }): MouthClicks {
|
|
212
|
+
const { mono, sampleRate, offset } = options;
|
|
213
|
+
const shift = (span: TimeSpan, by: number): TimeSpan => ({ start: span.start + by, end: span.end + by });
|
|
214
|
+
const analysis = analyzeMouthClicks(
|
|
215
|
+
mono,
|
|
216
|
+
sampleRate,
|
|
217
|
+
options.words.map((word) => shift(word, -offset)),
|
|
218
|
+
);
|
|
219
|
+
return {
|
|
220
|
+
clicks: analysis.candidates.filter((candidate) => !analysis.isProtected(candidate)).map((candidate) => shift(candidate, offset)),
|
|
221
|
+
words: analysis.bounds.map((bound) => shift(bound, offset)),
|
|
222
|
+
isQuiet: (stretch) => analysis.isQuiet(shift(stretch, -offset)),
|
|
223
|
+
};
|
|
224
|
+
}
|
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Sound analysis shared by the speech declick stage and the mouth-click detector (issues #1170, #1182, #1183): a
|
|
3
|
+
* fourth-order band pass, frame energies and a running median.
|
|
4
|
+
*/
|
|
5
|
+
|
|
6
|
+
/** A stretch of time in seconds. */
|
|
7
|
+
export interface TimeSpan {
|
|
8
|
+
start: number;
|
|
9
|
+
end: number;
|
|
10
|
+
}
|
|
11
|
+
|
|
12
|
+
/** One second-order section (RBJ cookbook), run in place over the samples. */
|
|
13
|
+
function runBiquad(samples: Float64Array, kind: "highpass" | "lowpass", frequency: number, sampleRate: number): void {
|
|
14
|
+
const w0 = (2 * Math.PI * Math.min(frequency, sampleRate * 0.45)) / sampleRate;
|
|
15
|
+
const cos = Math.cos(w0);
|
|
16
|
+
const alpha = Math.sin(w0) / (2 * Math.SQRT1_2);
|
|
17
|
+
const a0 = 1 + alpha;
|
|
18
|
+
const b0 = (kind === "lowpass" ? (1 - cos) / 2 : (1 + cos) / 2) / a0;
|
|
19
|
+
const b1 = (kind === "lowpass" ? 1 - cos : -(1 + cos)) / a0;
|
|
20
|
+
const b2 = b0;
|
|
21
|
+
const a1 = (-2 * cos) / a0;
|
|
22
|
+
const a2 = (1 - alpha) / a0;
|
|
23
|
+
let x1 = 0;
|
|
24
|
+
let x2 = 0;
|
|
25
|
+
let y1 = 0;
|
|
26
|
+
let y2 = 0;
|
|
27
|
+
for (let index = 0; index < samples.length; index++) {
|
|
28
|
+
const x = samples[index] ?? 0;
|
|
29
|
+
const y = b0 * x + b1 * x1 + b2 * x2 - a1 * y1 - a2 * y2;
|
|
30
|
+
x2 = x1;
|
|
31
|
+
x1 = x;
|
|
32
|
+
y2 = y1;
|
|
33
|
+
y1 = y;
|
|
34
|
+
samples[index] = y;
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
/** The band of the signal between `lowHz` and `highHz`: two high-pass and two low-pass sections (fourth order on each side). */
|
|
39
|
+
export function getBand(options: { mono: Float32Array; sampleRate: number; lowHz: number; highHz: number }): Float64Array {
|
|
40
|
+
const { mono, sampleRate, lowHz, highHz } = options;
|
|
41
|
+
const band = Float64Array.from(mono);
|
|
42
|
+
runBiquad(band, "highpass", lowHz, sampleRate);
|
|
43
|
+
runBiquad(band, "highpass", lowHz, sampleRate);
|
|
44
|
+
runBiquad(band, "lowpass", highHz, sampleRate);
|
|
45
|
+
runBiquad(band, "lowpass", highHz, sampleRate);
|
|
46
|
+
return band;
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
/** The running sums of the squared samples: `sums[i]` is the energy of samples 0 to i - 1. */
|
|
50
|
+
export function getSquareSums(samples: ArrayLike<number>): Float64Array {
|
|
51
|
+
const sums = new Float64Array(samples.length + 1);
|
|
52
|
+
for (let index = 0; index < samples.length; index++) {
|
|
53
|
+
const value = samples[index] ?? 0;
|
|
54
|
+
sums[index + 1] = (sums[index] ?? 0) + value * value;
|
|
55
|
+
}
|
|
56
|
+
return sums;
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
/** A mean square in dB (10·log10), with a floor so silence stays finite. */
|
|
60
|
+
export const toPowerDb = (meanSquare: number): number => 10 * Math.log10(Math.max(meanSquare, 1e-20));
|
|
61
|
+
|
|
62
|
+
/** Each frame's mean square in dB, frames of `frameLength` samples every `hop` samples. */
|
|
63
|
+
export function getFrameEnergies(band: Float64Array, frameLength: number, hop: number): Float64Array {
|
|
64
|
+
const sums = getSquareSums(band);
|
|
65
|
+
const count = band.length < frameLength ? 0 : Math.floor((band.length - frameLength) / hop) + 1;
|
|
66
|
+
const energies = new Float64Array(count);
|
|
67
|
+
for (let frame = 0; frame < count; frame++) {
|
|
68
|
+
const from = frame * hop;
|
|
69
|
+
energies[frame] = toPowerDb(((sums[from + frameLength] ?? 0) - (sums[from] ?? 0)) / frameLength);
|
|
70
|
+
}
|
|
71
|
+
return energies;
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
/** The index of the first entry of the sorted list not below `value`. */
|
|
75
|
+
function findInsertIndex(sorted: number[], value: number): number {
|
|
76
|
+
let low = 0;
|
|
77
|
+
let high = sorted.length;
|
|
78
|
+
while (low < high) {
|
|
79
|
+
const middle = (low + high) >> 1;
|
|
80
|
+
if ((sorted[middle] ?? 0) < value) low = middle + 1;
|
|
81
|
+
else high = middle;
|
|
82
|
+
}
|
|
83
|
+
return low;
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
/** The running median of the values over `reach` entries on each side (fewer at the ends). */
|
|
87
|
+
export function getRunningMedian(values: Float64Array, reach: number): Float64Array {
|
|
88
|
+
const medians = new Float64Array(values.length);
|
|
89
|
+
const window: number[] = [];
|
|
90
|
+
let from = 0;
|
|
91
|
+
let to = 0;
|
|
92
|
+
for (let index = 0; index < values.length; index++) {
|
|
93
|
+
const wantedTo = Math.min(values.length, index + reach + 1);
|
|
94
|
+
for (; to < wantedTo; to++) {
|
|
95
|
+
const value = values[to] ?? 0;
|
|
96
|
+
window.splice(findInsertIndex(window, value), 0, value);
|
|
97
|
+
}
|
|
98
|
+
const wantedFrom = Math.max(0, index - reach);
|
|
99
|
+
for (; from < wantedFrom; from++) window.splice(findInsertIndex(window, values[from] ?? 0), 1);
|
|
100
|
+
const middle = window.length >> 1;
|
|
101
|
+
medians[index] = window.length % 2 === 1 ? (window[middle] ?? 0) : ((window[middle - 1] ?? 0) + (window[middle] ?? 0)) / 2;
|
|
102
|
+
}
|
|
103
|
+
return medians;
|
|
104
|
+
}
|
package/src/index.ts
CHANGED
|
@@ -252,6 +252,7 @@ export {
|
|
|
252
252
|
DECLICK_CROSSFADE,
|
|
253
253
|
DECLICK_LEVELS,
|
|
254
254
|
DECLICK_MAX_IMPULSE,
|
|
255
|
+
DECLICK_REPAIR_RULES,
|
|
255
256
|
DECLICK_SETTINGS,
|
|
256
257
|
DECLICK_WORD_GUARD,
|
|
257
258
|
buildDeclickDecodeArgs,
|
|
@@ -260,6 +261,7 @@ export {
|
|
|
260
261
|
declickSpeech,
|
|
261
262
|
describeDeclick,
|
|
262
263
|
findImpulses,
|
|
264
|
+
findRepairs,
|
|
263
265
|
getArPolynomial,
|
|
264
266
|
getRepairZones,
|
|
265
267
|
getRunningMedian,
|
|
@@ -269,8 +271,10 @@ export {
|
|
|
269
271
|
type DeclickReport,
|
|
270
272
|
type DeclickSettings,
|
|
271
273
|
type DeclickSpeechOptions,
|
|
274
|
+
type DetectorDeclickLevel,
|
|
272
275
|
type TimeSpan as DeclickSpan,
|
|
273
276
|
} from "./footage/declick.js";
|
|
277
|
+
export { MOUTH_CLICK_RULES, findMouthClicks, type MouthClicks } from "./footage/mouth-clicks.js";
|
|
274
278
|
export {
|
|
275
279
|
MIN_CLIP_SECONDS,
|
|
276
280
|
describeProbe,
|
package/src/og/index.ts
CHANGED
package/src/og/photo-cover.ts
CHANGED
|
@@ -62,6 +62,8 @@ const PILL_ALPHA = 0xd9;
|
|
|
62
62
|
const DRAWN_ALPHA = 16;
|
|
63
63
|
/** The scrim's opacity at the band's lower edge and below it. */
|
|
64
64
|
const SCRIM_ALPHA = 0x8c;
|
|
65
|
+
/** The least contrast the accent keeps against what it stands on: the WCAG bound for large text. */
|
|
66
|
+
export const MIN_COVER_ACCENT_CONTRAST = 3;
|
|
65
67
|
|
|
66
68
|
/** The centre 4:5 band of a cover, in pixels. */
|
|
67
69
|
export interface CoverBand {
|
|
@@ -104,6 +106,7 @@ const coverOptionsSchema = z.object({
|
|
|
104
106
|
.max(MAX_COVER_KICKER_LENGTH, `the kicker is longer than ${String(MAX_COVER_KICKER_LENGTH)} characters`)
|
|
105
107
|
.optional(),
|
|
106
108
|
tone: z.enum(["light", "dark"]).optional(),
|
|
109
|
+
accent: z.boolean().optional(),
|
|
107
110
|
quality: z.number().int().min(1).max(100).default(PHOTO_COVER_QUALITY),
|
|
108
111
|
});
|
|
109
112
|
|
|
@@ -122,6 +125,12 @@ export interface PhotoCoverInput {
|
|
|
122
125
|
kicker?: string;
|
|
123
126
|
/** `light` text over a dark scrim, or `dark` text over a light one; `light` by default. */
|
|
124
127
|
tone?: PhotoCoverTone;
|
|
128
|
+
/**
|
|
129
|
+
* Draws the brand's `accent` colour: the kicker of `kicker-title` in it, or the `pill` box in it (opaque) with the
|
|
130
|
+
* label in `foreground` or `background`, whichever reads better. Refused by the other layouts, and refused when the
|
|
131
|
+
* accent's contrast against what it stands on is under 3:1. Off by default.
|
|
132
|
+
*/
|
|
133
|
+
accent?: boolean;
|
|
125
134
|
/** The light colour is the lighter of `foreground` and `background`, the dark colour the darker. */
|
|
126
135
|
colors: BrandColors;
|
|
127
136
|
/** The brand's fonts (`loadOgFonts`); the text is set in the heading (display) family. */
|
|
@@ -184,9 +193,12 @@ function getRegionRows(band: CoverBand, region: CoverRegion): { top: number; bot
|
|
|
184
193
|
}
|
|
185
194
|
|
|
186
195
|
/** The text the options ask for, or an error when the layout and the texts do not go together. */
|
|
187
|
-
function resolveCoverText(options: { layout?: PhotoCoverLayout | undefined; caption?: string | undefined; kicker?: string | undefined }): OgResult<CoverText | null> {
|
|
196
|
+
function resolveCoverText(options: { layout?: PhotoCoverLayout | undefined; caption?: string | undefined; kicker?: string | undefined; accent?: boolean | undefined }): OgResult<CoverText | null> {
|
|
188
197
|
const { caption, kicker } = options;
|
|
189
198
|
const layout = options.layout ?? (caption === undefined ? "photo" : "title-bottom");
|
|
199
|
+
if (options.accent === true && layout !== "kicker-title" && layout !== "pill") {
|
|
200
|
+
return err(`the accent is drawn only by the layouts "kicker-title" and "pill", not "${layout}"; leave it out`);
|
|
201
|
+
}
|
|
190
202
|
if (kicker !== undefined && layout !== "kicker-title") return err(`the kicker is drawn only by the layout "kicker-title", not "${layout}"; leave it out`);
|
|
191
203
|
if (layout === "photo") return caption === undefined ? ok(null) : err('the layout "photo" draws no text; leave the caption out or pick a text layout');
|
|
192
204
|
if (caption === undefined) return err(`the layout "${layout}" needs a caption`);
|
|
@@ -220,12 +232,54 @@ function getLuminance(color: string): number {
|
|
|
220
232
|
return 0.2126 * channel(1) + 0.7152 * channel(3) + 0.0722 * channel(5);
|
|
221
233
|
}
|
|
222
234
|
|
|
223
|
-
/** The
|
|
224
|
-
function
|
|
235
|
+
/** The WCAG contrast ratio of two opaque colours, 1 to 21. */
|
|
236
|
+
function getContrast(first: string, second: string): number {
|
|
237
|
+
const [lighter, darker] = [getLuminance(first), getLuminance(second)].sort((a, b) => b - a) as [number, number];
|
|
238
|
+
return (lighter + 0.05) / (darker + 0.05);
|
|
239
|
+
}
|
|
240
|
+
|
|
241
|
+
/** A contrast ratio as errors write it, rounded down so a ratio just under the bound never reads as the bound. */
|
|
242
|
+
const describeContrast = (ratio: number) => `${(Math.floor(ratio * 100) / 100).toFixed(2)}:1`;
|
|
243
|
+
|
|
244
|
+
/** The colours a cover's text is drawn in. */
|
|
245
|
+
interface CoverInk {
|
|
246
|
+
/** The caption (and the kicker without the accent). */
|
|
247
|
+
text: string;
|
|
248
|
+
/** The scrim and the shadow. */
|
|
249
|
+
shade: string;
|
|
250
|
+
kicker: string;
|
|
251
|
+
pill: { box: string; alpha: number; text: string };
|
|
252
|
+
}
|
|
253
|
+
|
|
254
|
+
/**
|
|
255
|
+
* The cover's colours: the text the brand's light colour and the scrim the dark one, swapped for a `dark` tone; with
|
|
256
|
+
* the accent, the kicker or the pill box in `colors.accent`. An error when the accent reads under 3:1.
|
|
257
|
+
*/
|
|
258
|
+
function getCoverInk(colors: BrandColors, options: { tone: PhotoCoverTone; layout: CoverText["layout"] | null; accent: boolean }): OgResult<CoverInk> {
|
|
225
259
|
const foreground = toOpaqueHex(colors.foreground);
|
|
226
260
|
const background = toOpaqueHex(colors.background);
|
|
227
261
|
const [light, dark] = getLuminance(foreground) >= getLuminance(background) ? [foreground, background] : [background, foreground];
|
|
228
|
-
|
|
262
|
+
const [text, shade] = options.tone === "light" ? [light, dark] : [dark, light];
|
|
263
|
+
const plain: CoverInk = { text, shade, kicker: text, pill: { box: shade, alpha: PILL_ALPHA, text } };
|
|
264
|
+
if (!options.accent) return ok(plain);
|
|
265
|
+
const accent = toOpaqueHex(colors.accent);
|
|
266
|
+
if (options.layout === "kicker-title") {
|
|
267
|
+
const ratio = getContrast(accent, shade);
|
|
268
|
+
if (ratio < MIN_COVER_ACCENT_CONTRAST) {
|
|
269
|
+
return err(`the accent ${accent} has a contrast of ${describeContrast(ratio)} against the scrim ${shade}, under 3:1; pick a ${getLuminance(accent) < getLuminance(shade) ? "darker" : "lighter"} accent or the other tone`);
|
|
270
|
+
}
|
|
271
|
+
return ok({ ...plain, kicker: accent });
|
|
272
|
+
}
|
|
273
|
+
if (options.layout === "pill") {
|
|
274
|
+
const label = getContrast(accent, light) >= getContrast(accent, dark) ? light : dark;
|
|
275
|
+
const ratio = getContrast(accent, label);
|
|
276
|
+
if (ratio < MIN_COVER_ACCENT_CONTRAST) {
|
|
277
|
+
return err(`the accent ${accent} has a contrast of ${describeContrast(ratio)} against the pill label ${label}, under 3:1; pick an accent that reads with the foreground or the background`);
|
|
278
|
+
}
|
|
279
|
+
// Opaque, so the contrast checked is the contrast drawn whatever the photo under it.
|
|
280
|
+
return ok({ ...plain, pill: { box: accent, alpha: 0xff, text: label } });
|
|
281
|
+
}
|
|
282
|
+
return ok(plain);
|
|
229
283
|
}
|
|
230
284
|
|
|
231
285
|
interface TextLayout {
|
|
@@ -233,7 +287,7 @@ interface TextLayout {
|
|
|
233
287
|
band: CoverBand;
|
|
234
288
|
text: CoverText;
|
|
235
289
|
fonts: OgFonts;
|
|
236
|
-
ink:
|
|
290
|
+
ink: CoverInk;
|
|
237
291
|
}
|
|
238
292
|
|
|
239
293
|
/** Where the text block sits: its foot above the band's lower edge, its head below the band's top, or centred in the middle third. */
|
|
@@ -268,10 +322,10 @@ function buildTextTree(layout: TextLayout): OgNode {
|
|
|
268
322
|
maxWidth: "100%",
|
|
269
323
|
padding: `${String(Math.round(PILL_PADDING[0] * scale))}px ${String(Math.round(PILL_PADDING[1] * scale))}px`,
|
|
270
324
|
borderRadius: Math.round(PILL_SIZE * scale),
|
|
271
|
-
backgroundColor: withAlpha(ink.
|
|
325
|
+
backgroundColor: withAlpha(ink.pill.box, ink.pill.alpha),
|
|
272
326
|
textAlign: "center",
|
|
273
327
|
wordBreak: "break-word",
|
|
274
|
-
color: ink.text,
|
|
328
|
+
color: ink.pill.text,
|
|
275
329
|
fontSize: Math.round(PILL_SIZE * scale),
|
|
276
330
|
lineHeight: 1.15,
|
|
277
331
|
...font,
|
|
@@ -282,7 +336,7 @@ function buildTextTree(layout: TextLayout): OgNode {
|
|
|
282
336
|
]
|
|
283
337
|
: [
|
|
284
338
|
...(text.layout === "kicker-title"
|
|
285
|
-
? [line(text.kicker, { fontSize: Math.round(KICKER_SIZE * scale), lineHeight: 1.2, letterSpacing: Math.round(3 * scale), marginBottom: Math.round(KICKER_GAP * scale), textShadow: shadow })]
|
|
339
|
+
? [line(text.kicker, { color: ink.kicker, fontSize: Math.round(KICKER_SIZE * scale), lineHeight: 1.2, letterSpacing: Math.round(3 * scale), marginBottom: Math.round(KICKER_GAP * scale), textShadow: shadow })]
|
|
286
340
|
: []),
|
|
287
341
|
line(text.caption, { fontSize: Math.round(getCoverCaptionSize(text.caption) * scale), lineHeight: 1.08, textShadow: shadow }),
|
|
288
342
|
];
|
|
@@ -418,6 +472,7 @@ export async function renderPhotoCover(input: PhotoCoverInput): Promise<OgResult
|
|
|
418
472
|
caption: input.caption,
|
|
419
473
|
kicker: input.kicker,
|
|
420
474
|
tone: input.tone,
|
|
475
|
+
accent: input.accent,
|
|
421
476
|
quality: input.quality,
|
|
422
477
|
});
|
|
423
478
|
if (!parsed.success) {
|
|
@@ -435,8 +490,9 @@ export async function renderPhotoCover(input: PhotoCoverInput): Promise<OgResult
|
|
|
435
490
|
const size = options.size;
|
|
436
491
|
const [width, height] = size;
|
|
437
492
|
const band = getCoverBand(size);
|
|
438
|
-
const ink = getCoverInk(input.colors, options.tone ?? "light");
|
|
439
|
-
|
|
493
|
+
const ink = getCoverInk(input.colors, { tone: options.tone ?? "light", layout: text.value?.layout ?? null, accent: options.accent ?? false });
|
|
494
|
+
if (!ink.ok) return err(`${name}: ${ink.error}.`);
|
|
495
|
+
const drawn = text.value === null ? null : await layOutText({ size, band, text: text.value, fonts: input.fonts, ink: ink.value });
|
|
440
496
|
if (drawn !== null && !drawn.ok) return err(`${name}: ${drawn.error}.`);
|
|
441
497
|
|
|
442
498
|
const oriented = getOrientedSize(info.value);
|
|
@@ -444,13 +500,13 @@ export async function renderPhotoCover(input: PhotoCoverInput): Promise<OgResult
|
|
|
444
500
|
const focusPoint = { x: crop.x + options.focus.x * oriented.width * crop.scale, y: crop.y + options.focus.y * oriented.height * crop.scale };
|
|
445
501
|
const matrix = getOrientationMatrix(info.value.orientation, info.value.width, info.value.height).join(" ");
|
|
446
502
|
const layers = [
|
|
447
|
-
`<rect width="${String(width)}" height="${String(height)}" fill="${ink.shade}"/>`,
|
|
503
|
+
`<rect width="${String(width)}" height="${String(height)}" fill="${ink.value.shade}"/>`,
|
|
448
504
|
`<image href="${toDataUri(`image/${info.value.format}`, bytes.value)}" width="${String(info.value.width)}" height="${String(info.value.height)}" preserveAspectRatio="none" ` +
|
|
449
505
|
`transform="translate(${String(crop.x)} ${String(crop.y)}) scale(${String(crop.scale)}) matrix(${matrix})"/>`,
|
|
450
506
|
];
|
|
451
507
|
if (drawn !== null && text.value !== null) {
|
|
452
508
|
layers.push(
|
|
453
|
-
buildScrim({ size, band, layout: text.value.layout, text: { top: drawn.value.box.top, bottom: drawn.value.box.bottom - 1 }, shade: ink.shade }),
|
|
509
|
+
buildScrim({ size, band, layout: text.value.layout, text: { top: drawn.value.box.top, bottom: drawn.value.box.bottom - 1 }, shade: ink.value.shade }),
|
|
454
510
|
`<image href="${toDataUri("image/svg+xml", drawn.value.svg)}" width="${String(width)}" height="${String(height)}"/>`,
|
|
455
511
|
);
|
|
456
512
|
}
|