@ignix-social/editor 0.1.7 → 0.1.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOTICE +7 -0
- package/licenses/torch_audioset-MIT.txt +21 -0
- package/package.json +1 -1
- package/patches/index.mjs +29 -0
- package/skills/ignix-editor/SKILL.md +3 -0
- package/src/analysis/audio-events.mjs +223 -0
- package/src/analysis/audioset-classes.json +1 -0
- package/src/analysis/runtime.mjs +9 -2
- package/src/analysis/serve.mjs +10 -0
- package/src/analysis/style.mjs +381 -0
- package/src/binaries.json +14 -0
- package/src/cli.mjs +25 -0
- package/src/runtime/ignix-frame.js +1 -1
- package/src/validate.mjs +10 -0
- package/vendor/hyperframes/UPSTREAM.json +2 -1
- package/vendor/hyperframes/dist/checkBrowser-RPMO4OW5.js +5 -5
- package/vendor/hyperframes/dist/chunk-2YN64IN5.js +1 -1
- package/vendor/hyperframes/dist/chunk-4PFZPP2N.js +3 -3
- package/vendor/hyperframes/dist/chunk-4THIAPBL.js +1 -1
- package/vendor/hyperframes/dist/chunk-EOVYQ3IG.js +1 -1
- package/vendor/hyperframes/dist/chunk-HWKCIWQT.js +2 -2
- package/vendor/hyperframes/dist/chunk-K2FTXJYH.js +6 -6
- package/vendor/hyperframes/dist/chunk-L3YOGIPG.js +1 -1
- package/vendor/hyperframes/dist/chunk-S7NPLFWV.js +1 -1
- package/vendor/hyperframes/dist/chunk-XUXGIA7H.js +1 -1
- package/vendor/hyperframes/dist/chunk-YQZNILVH.js +1 -1
- package/vendor/hyperframes/dist/chunk-Z6B37SNM.js +1 -1
- package/vendor/hyperframes/dist/chunk-ZMXUQLI2.js +1 -1
- package/vendor/hyperframes/dist/cli.js +1 -1
- package/vendor/hyperframes/dist/hyperframe-runtime.js +4 -4
- package/vendor/hyperframes/dist/hyperframe.manifest.json +1 -1
- package/vendor/hyperframes/dist/hyperframe.runtime.iife.js +4 -4
- package/vendor/hyperframes/dist/hyperframes-player.global.js +1 -1
package/NOTICE
CHANGED
|
@@ -84,6 +84,13 @@ CLIP ViT-B/32 (openai/clip-vit-base-patch32; ONNX export and tokenizer files fro
|
|
|
84
84
|
Xenova/clip-vit-base-patch32@d15189d) — MIT License, Copyright (c) 2021 OpenAI.
|
|
85
85
|
The vision model is stored in two parts that Ignix Editor joins and checks.
|
|
86
86
|
|
|
87
|
+
YAMNet (sound event classifier, Google's AudioSet model) as ported to PyTorch by
|
|
88
|
+
torch_audioset (w-hc/torch_audioset) — MIT License, Copyright (c) 2021 Haochen
|
|
89
|
+
(licenses/torch_audioset-MIT.txt); ONNX export by Qualcomm AI Hub (qualcomm/YamNet
|
|
90
|
+
v0.63.0, MIT). The AudioSet class names (src/analysis/audioset-classes.json) come from
|
|
91
|
+
tensorflow/models (research/audioset/yamnet/yamnet_class_map.csv) — Apache License 2.0,
|
|
92
|
+
Copyright The TensorFlow Authors.
|
|
93
|
+
|
|
87
94
|
Inter (font, from @fontsource/inter) — SIL Open Font License 1.1, Copyright (c) 2016
|
|
88
95
|
The Inter Project Authors. Copied into the projects Ignix Creator's templates render
|
|
89
96
|
(src/creator), unmodified.
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2021 Haochen
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
package/package.json
CHANGED
package/patches/index.mjs
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
// Parches sobre el dist compilado de HyperFrames. Los nombres de fichero llevan hash y
|
|
2
2
|
// cambian entre versiones, así que cada parche busca su ancla en TODO dist/ y exige que
|
|
3
3
|
// aparezca exactamente las veces esperadas. Si no, la actualización se para aquí.
|
|
4
|
+
import { createHash } from 'node:crypto';
|
|
4
5
|
import { readFileSync, readdirSync, writeFileSync } from 'node:fs';
|
|
5
6
|
import { join } from 'node:path';
|
|
6
7
|
|
|
@@ -17,6 +18,20 @@ function replaceAnchor(dist, { anchor, replacement, expect = 1 }) {
|
|
|
17
18
|
if (hits !== expect) throw new Error(`Ancla encontrada ${hits} veces (se esperaban ${expect}): ${anchor.slice(0, 80)}…`);
|
|
18
19
|
}
|
|
19
20
|
|
|
21
|
+
/** Sustituye TODAS las apariciones de `anchor` en dist/ y exige al menos `min`. */
|
|
22
|
+
function replaceAll(dist, { anchor, replacement, min = 1 }) {
|
|
23
|
+
let hits = 0;
|
|
24
|
+
for (const f of readdirSync(dist).filter((n) => n.endsWith('.js'))) {
|
|
25
|
+
const file = join(dist, f);
|
|
26
|
+
const src = readFileSync(file, 'utf8');
|
|
27
|
+
const n = src.split(anchor).length - 1;
|
|
28
|
+
if (!n) continue;
|
|
29
|
+
hits += n;
|
|
30
|
+
writeFileSync(file, src.split(anchor).join(replacement));
|
|
31
|
+
}
|
|
32
|
+
if (hits < min) throw new Error(`Ancla encontrada ${hits} veces (se esperaban al menos ${min}): ${anchor.slice(0, 80)}…`);
|
|
33
|
+
}
|
|
34
|
+
|
|
20
35
|
export const patches = [
|
|
21
36
|
{
|
|
22
37
|
id: 'no-telemetry',
|
|
@@ -88,6 +103,20 @@ export const patches = [
|
|
|
88
103
|
replacement: ' return null; // ignix-editor: sin aviso de la app de escritorio\n if (ready && (installed ?? desktopInstalled({ platform, env })))',
|
|
89
104
|
}),
|
|
90
105
|
},
|
|
106
|
+
{
|
|
107
|
+
id: 'log-prefix',
|
|
108
|
+
// Lo que ve el cliente (y su agente) dice Ignix Editor: el motor firmaba sus mensajes con
|
|
109
|
+
// «[hyperframes]» y el agente acababa nombrando HyperFrames.
|
|
110
|
+
apply: (dist) => {
|
|
111
|
+
replaceAll(dist, { anchor: '[hyperframes]', replacement: '[ignix-editor]', min: 20 });
|
|
112
|
+
// El motor comprueba la huella de su runtime (hyperframe.manifest.json): se recalcula.
|
|
113
|
+
const mf = join(dist, 'hyperframe.manifest.json');
|
|
114
|
+
const manifest = JSON.parse(readFileSync(mf, 'utf8'));
|
|
115
|
+
const runtime = readFileSync(join(dist, manifest.artifacts.iife), 'utf8');
|
|
116
|
+
manifest.sha256 = createHash('sha256').update(runtime, 'utf8').digest('hex');
|
|
117
|
+
writeFileSync(mf, `${JSON.stringify(manifest, null, 2)}\n`);
|
|
118
|
+
},
|
|
119
|
+
},
|
|
91
120
|
{
|
|
92
121
|
id: 'chrome-first-launch',
|
|
93
122
|
// Antes de renderizar, el motor pregunta `chrome --version` con 5 s de margen. La
|
|
@@ -31,6 +31,7 @@ One project = one folder with `index.html` at its root and everything it uses in
|
|
|
31
31
|
- `data-duration` (seconds) is the length of the video. Set it from the real voice: sum of the lines + air + a ~0.6 s tail.
|
|
32
32
|
- Size the root with `width/height:100%`; the canvas size comes from `data-width/height`.
|
|
33
33
|
- Exactly one timeline, registered by `IgnixFrame.run({ id, duration, fps }, render)` (section 2). `id` = `data-composition-id`.
|
|
34
|
+
- Keep the two scripts `init` puts in `<head>`, in this order: `<script src="assets/gsap.min.js"></script>` then `<script src="ignix-frame.js"></script>`. If you rewrite `index.html`, copy them over: without GSAP first nothing moves (`check` reports `gsap_before_frame`).
|
|
34
35
|
|
|
35
36
|
### Media: written in the HTML, never created by JS
|
|
36
37
|
|
|
@@ -138,6 +139,8 @@ Re-run it whenever the voice or the cuts change; the animation code reads `windo
|
|
|
138
139
|
|
|
139
140
|
## 7. Review before showing anything
|
|
140
141
|
|
|
142
|
+
`check`, `snapshot` and `render` start a headless Chrome. A sandboxed shell can block it (Codex: `MachPortRendezvousServer: Permission denied`, or `Chrome cannot start`): run those commands with the host's escalated permissions, outside its sandbox — in Codex, request escalated permissions for the command. If the host cannot, give the user ONE line to paste into Terminal, e.g. `cd ~/ignix-videos/<name> && ~/.ignix-editor/cli/node_modules/.bin/ignix-editor render`, never the contents of a script file (a `#!` line breaks zsh), then continue when they say it finished.
|
|
143
|
+
|
|
141
144
|
1. `ignix-editor check` → 0 errors.
|
|
142
145
|
2. `ignix-editor snapshot --at <every ~0.5 s and every cue>` and LOOK at the frames (or tile them: `ignix-editor ffmpeg -pattern_type glob -i 'snapshots/frame-*.png' -vf "scale=270:480,tile=8x2" sheet.jpg`). Check: hook readable at frame 0, nothing still for >1.5 s, every named thing pointed at on its word, captions never over what is pointed at, nothing outside the safe zone.
|
|
143
146
|
3. `ignix-editor render` → open the MP4's first frame and a few cues again; then deliver as the playbook says.
|
|
@@ -0,0 +1,223 @@
|
|
|
1
|
+
// Qué SUENA en un vídeo, por tramos de 0,96 s: voz, música, efectos (whoosh, pop, ding,
|
|
2
|
+
// golpe…). YAMNet (Google, AudioSet: 521 clases), en el port a PyTorch de torch_audioset (MIT)
|
|
3
|
+
// que Qualcomm publica en ONNX; las clases son las de AudioSet (yamnet_class_map.csv de
|
|
4
|
+
// tensorflow/models, Apache-2.0). Para el estilo de edición (ADR-0023 E3): qué efectos usa
|
|
5
|
+
// alguien y cuánta música lleva.
|
|
6
|
+
//
|
|
7
|
+
// La entrada del modelo es un parche de 96 × 64 del espectrograma log-mel, calculado como lo
|
|
8
|
+
// hace YAMNet: 16 kHz mono, ventana de Hann periódica de 25 ms cada 10 ms, FFT de 512, 64
|
|
9
|
+
// bandas mel (escala HTK) de 125 a 7500 Hz, log(mel + 0,001); parches de 0,96 s cada 0,48 s.
|
|
10
|
+
import { readFileSync } from 'node:fs';
|
|
11
|
+
import { dirname, join } from 'node:path';
|
|
12
|
+
import { fileURLToPath } from 'node:url';
|
|
13
|
+
import { pcm } from './media.mjs';
|
|
14
|
+
import { session } from './runtime.mjs';
|
|
15
|
+
|
|
16
|
+
const RATE = 16000;
|
|
17
|
+
const WIN = 400;
|
|
18
|
+
const HOP = 160;
|
|
19
|
+
const NFFT = 512;
|
|
20
|
+
const BINS = NFFT / 2 + 1;
|
|
21
|
+
const MELS = 64;
|
|
22
|
+
const PATCH = 96;
|
|
23
|
+
const PATCH_HOP = 48;
|
|
24
|
+
export const PATCH_SEC = (PATCH * HOP) / RATE; // 0,96
|
|
25
|
+
export const PATCH_HOP_SEC = (PATCH_HOP * HOP) / RATE; // 0,48
|
|
26
|
+
|
|
27
|
+
export const CLASSES = JSON.parse(readFileSync(join(dirname(fileURLToPath(import.meta.url)), 'audioset-classes.json'), 'utf8'));
|
|
28
|
+
|
|
29
|
+
const hzToMel = (f) => 1127 * Math.log(1 + f / 700);
|
|
30
|
+
|
|
31
|
+
/** Matriz de pesos lineal → mel como tf.signal.linear_to_mel_weight_matrix (sin la banda DC). */
|
|
32
|
+
function melMatrix() {
|
|
33
|
+
const nyq = RATE / 2;
|
|
34
|
+
const lo = hzToMel(125);
|
|
35
|
+
const hi = hzToMel(7500);
|
|
36
|
+
const edges = Array.from({ length: MELS + 2 }, (_, i) => lo + ((hi - lo) * i) / (MELS + 1));
|
|
37
|
+
const w = Array.from({ length: BINS }, () => new Float32Array(MELS));
|
|
38
|
+
for (let b = 1; b < BINS; b++) {
|
|
39
|
+
const mel = hzToMel((nyq * b) / (BINS - 1));
|
|
40
|
+
for (let m = 0; m < MELS; m++) {
|
|
41
|
+
const lower = (mel - edges[m]) / (edges[m + 1] - edges[m]);
|
|
42
|
+
const upper = (edges[m + 2] - mel) / (edges[m + 2] - edges[m + 1]);
|
|
43
|
+
w[b][m] = Math.max(0, Math.min(lower, upper));
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
return w;
|
|
47
|
+
}
|
|
48
|
+
const MEL = melMatrix();
|
|
49
|
+
const HANN = Float32Array.from({ length: WIN }, (_, n) => 0.5 - 0.5 * Math.cos((2 * Math.PI * n) / WIN));
|
|
50
|
+
|
|
51
|
+
/** Magnitud de la FFT real de 512 puntos (radix 2, en el sitio). */
|
|
52
|
+
function magnitude(frame, re, im) {
|
|
53
|
+
re.fill(0);
|
|
54
|
+
im.fill(0);
|
|
55
|
+
re.set(frame);
|
|
56
|
+
for (let i = 1, j = 0; i < NFFT; i++) {
|
|
57
|
+
let bit = NFFT >> 1;
|
|
58
|
+
for (; j & bit; bit >>= 1) j ^= bit;
|
|
59
|
+
j ^= bit;
|
|
60
|
+
if (i < j) {
|
|
61
|
+
[re[i], re[j]] = [re[j], re[i]];
|
|
62
|
+
[im[i], im[j]] = [im[j], im[i]];
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
for (let len = 2; len <= NFFT; len <<= 1) {
|
|
66
|
+
const ang = (-2 * Math.PI) / len;
|
|
67
|
+
for (let i = 0; i < NFFT; i += len) {
|
|
68
|
+
for (let k = 0; k < len / 2; k++) {
|
|
69
|
+
const cr = Math.cos(ang * k);
|
|
70
|
+
const ci = Math.sin(ang * k);
|
|
71
|
+
const a = i + k;
|
|
72
|
+
const b = a + len / 2;
|
|
73
|
+
const tr = re[b] * cr - im[b] * ci;
|
|
74
|
+
const ti = re[b] * ci + im[b] * cr;
|
|
75
|
+
re[b] = re[a] - tr;
|
|
76
|
+
im[b] = im[a] - ti;
|
|
77
|
+
re[a] += tr;
|
|
78
|
+
im[a] += ti;
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
const out = new Float32Array(BINS);
|
|
83
|
+
for (let k = 0; k < BINS; k++) out[k] = Math.hypot(re[k], im[k]);
|
|
84
|
+
return out;
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
/** Espectrograma log-mel (frames × 64) de muestras mono a 16 kHz. */
|
|
88
|
+
export function logMel(samples) {
|
|
89
|
+
const frames = samples.length >= WIN ? 1 + Math.floor((samples.length - WIN) / HOP) : 0;
|
|
90
|
+
const out = new Float32Array(frames * MELS);
|
|
91
|
+
const buf = new Float32Array(WIN);
|
|
92
|
+
const re = new Float64Array(NFFT);
|
|
93
|
+
const im = new Float64Array(NFFT);
|
|
94
|
+
for (let f = 0; f < frames; f++) {
|
|
95
|
+
for (let n = 0; n < WIN; n++) buf[n] = samples[f * HOP + n] * HANN[n];
|
|
96
|
+
const mag = magnitude(buf, re, im);
|
|
97
|
+
for (let m = 0; m < MELS; m++) {
|
|
98
|
+
let s = 0;
|
|
99
|
+
for (let b = 1; b < BINS; b++) s += mag[b] * MEL[b][m];
|
|
100
|
+
out[f * MELS + m] = Math.log(s + 0.001);
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
return { frames, data: out };
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
const sigmoid = (x) => 1 / (1 + Math.exp(-x));
|
|
107
|
+
|
|
108
|
+
/**
|
|
109
|
+
* Clasifica el sonido de un fichero por parches de 0,96 s (cada 0,48 s). Devuelve, por parche,
|
|
110
|
+
* su instante y las puntuaciones (0..1) de las 521 clases.
|
|
111
|
+
*/
|
|
112
|
+
export async function classify(ffmpeg, path, { maxSec = 600, log = () => {} } = {}) {
|
|
113
|
+
const chunks = [];
|
|
114
|
+
let total = 0;
|
|
115
|
+
const has = await pcm(ffmpeg, path, {
|
|
116
|
+
rate: RATE,
|
|
117
|
+
end: maxSec,
|
|
118
|
+
onSamples: (s) => {
|
|
119
|
+
chunks.push(Float32Array.from(s));
|
|
120
|
+
total += s.length;
|
|
121
|
+
},
|
|
122
|
+
});
|
|
123
|
+
if (has === false || !total) return [];
|
|
124
|
+
// Como YAMNet: con silencio hasta completar el último parche (un efecto de 0,3 s es un
|
|
125
|
+
// parche, no ninguno).
|
|
126
|
+
const need = (PATCH - 1) * HOP + WIN;
|
|
127
|
+
const len = total < need ? need : need + Math.ceil((total - need) / (PATCH_HOP * HOP)) * PATCH_HOP * HOP;
|
|
128
|
+
const samples = new Float32Array(len);
|
|
129
|
+
let o = 0;
|
|
130
|
+
for (const c of chunks) {
|
|
131
|
+
samples.set(c, o);
|
|
132
|
+
o += c.length;
|
|
133
|
+
}
|
|
134
|
+
const { frames, data } = logMel(samples);
|
|
135
|
+
const s = await session('yamnet', log);
|
|
136
|
+
const ort = await import('./runtime.mjs').then((m) => m.loadOrt(log));
|
|
137
|
+
const patches = [];
|
|
138
|
+
for (let start = 0; start + PATCH <= frames; start += PATCH_HOP) {
|
|
139
|
+
const input = new ort.Tensor('float32', data.slice(start * MELS, (start + PATCH) * MELS), [1, 1, PATCH, MELS]);
|
|
140
|
+
const res = await s.run({ [s.inputNames[0]]: input });
|
|
141
|
+
const raw = res[s.outputNames[0]].data;
|
|
142
|
+
// El port da logits; YAMNet da probabilidades. Se normaliza a 0..1 en los dos casos.
|
|
143
|
+
const logits = raw.some((v) => v < 0 || v > 1);
|
|
144
|
+
const scores = Float32Array.from(raw, (v) => (logits ? sigmoid(v) : v));
|
|
145
|
+
patches.push({ t: +((start * HOP) / RATE).toFixed(2), scores });
|
|
146
|
+
}
|
|
147
|
+
return patches;
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
/** Las `n` clases más probables de un parche. */
|
|
151
|
+
export function top(scores, n = 3) {
|
|
152
|
+
return [...scores]
|
|
153
|
+
.map((p, i) => ({ name: CLASSES[i], p }))
|
|
154
|
+
.sort((a, b) => b.p - a.p)
|
|
155
|
+
.slice(0, n);
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
/**
|
|
159
|
+
* Familias de efectos de sonido, por las clases de AudioSet que las delatan. Comprobado con
|
|
160
|
+
* la biblioteca de efectos de Ignix (2026-10-06): los whoosh dan «Whoosh, swoosh, swish»
|
|
161
|
+
* 0,58–0,78; el pop «Plop» 0,95; el ding «Ding» 0,97; la cámara «Camera».
|
|
162
|
+
*/
|
|
163
|
+
export const SFX_FAMILIES = {
|
|
164
|
+
whoosh: ['Whoosh, swoosh, swish', 'Arrow'],
|
|
165
|
+
pop: ['Plop', 'Boing'],
|
|
166
|
+
ding: ['Ding', 'Chime', 'Bell', 'Jingle, tinkle', 'Ding-dong', 'Clang', 'Cash register'],
|
|
167
|
+
click: ['Tick', 'Clicking', 'Tap', 'Typing', 'Camera'],
|
|
168
|
+
impact: ['Thump, thud', 'Whack, thwack', 'Bang', 'Slap, smack', 'Boom', 'Explosion'],
|
|
169
|
+
notification: ['Telephone', 'Ringtone', 'Beep, bleep', 'Alarm'],
|
|
170
|
+
crowd: ['Applause', 'Clapping', 'Cheering', 'Laughter'],
|
|
171
|
+
};
|
|
172
|
+
const FAMILY_IDX = Object.fromEntries(Object.entries(SFX_FAMILIES).map(([k, names]) => [k, names.map((n) => CLASSES.indexOf(n)).filter((i) => i >= 0)]));
|
|
173
|
+
const IDX = (name) => CLASSES.indexOf(name);
|
|
174
|
+
const MUSIC = IDX('Music');
|
|
175
|
+
const SPEECH = IDX('Speech');
|
|
176
|
+
const GENERIC = IDX('Sound effect');
|
|
177
|
+
export const SFX_MIN = 0.3;
|
|
178
|
+
|
|
179
|
+
/**
|
|
180
|
+
* Lo que suena, resumido: cuánta música y cuánta voz (parte de los parches), y los efectos:
|
|
181
|
+
* cuántos por minuto, de qué familias y cuántos cortes llevan uno (±0,5 s).
|
|
182
|
+
*/
|
|
183
|
+
export function soundProfile(patches, { duration, cutTimes = [] } = {}) {
|
|
184
|
+
if (!patches.length) return null;
|
|
185
|
+
const events = [];
|
|
186
|
+
let music = 0;
|
|
187
|
+
let speech = 0;
|
|
188
|
+
for (const p of patches) {
|
|
189
|
+
if (p.scores[MUSIC] >= 0.3) music += 1;
|
|
190
|
+
if (p.scores[SPEECH] >= 0.3) speech += 1;
|
|
191
|
+
let best = null;
|
|
192
|
+
for (const [family, idx] of Object.entries(FAMILY_IDX)) {
|
|
193
|
+
const score = Math.max(...idx.map((i) => p.scores[i]));
|
|
194
|
+
if (score >= SFX_MIN && (!best || score > best.score)) best = { family, score };
|
|
195
|
+
}
|
|
196
|
+
if (!best && p.scores[GENERIC] >= SFX_MIN) best = { family: 'other', score: p.scores[GENERIC] };
|
|
197
|
+
if (!best) continue;
|
|
198
|
+
const t = p.t + PATCH_SEC / 2;
|
|
199
|
+
const last = events[events.length - 1];
|
|
200
|
+
// Parches que se solapan (cada 0,48 s) son el mismo sonido aunque lo vean como familias
|
|
201
|
+
// distintas (un ding también parece un teléfono): se queda la de más puntuación.
|
|
202
|
+
if (last && t - last.at < PATCH_SEC - 0.05) {
|
|
203
|
+
last.at = t;
|
|
204
|
+
if (best.score > last.score) Object.assign(last, { family: best.family, score: best.score });
|
|
205
|
+
continue;
|
|
206
|
+
}
|
|
207
|
+
events.push({ t: +t.toFixed(2), at: t, family: best.family, score: best.score });
|
|
208
|
+
}
|
|
209
|
+
const counts = {};
|
|
210
|
+
for (const e of events) counts[e.family] = (counts[e.family] ?? 0) + 1;
|
|
211
|
+
const minutes = Math.max(duration ?? patches.length * PATCH_HOP_SEC, 1) / 60;
|
|
212
|
+
const onCuts = cutTimes.filter((c) => events.some((e) => Math.abs(e.t - c) <= 0.5)).length;
|
|
213
|
+
return {
|
|
214
|
+
musicShare: +(music / patches.length).toFixed(2),
|
|
215
|
+
speechShare: +(speech / patches.length).toFixed(2),
|
|
216
|
+
sfxPerMinute: +(events.length / minutes).toFixed(2),
|
|
217
|
+
sfxOnCutsShare: cutTimes.length ? +(onCuts / cutTimes.length).toFixed(2) : null,
|
|
218
|
+
sfxTypes: Object.entries(counts)
|
|
219
|
+
.sort((a, b) => b[1] - a[1])
|
|
220
|
+
.map(([family, count]) => ({ family, count })),
|
|
221
|
+
events: events.slice(0, 200).map((e) => ({ t: e.t, family: e.family })),
|
|
222
|
+
};
|
|
223
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
["Speech", "Child speech, kid speaking", "Conversation", "Narration, monologue", "Babbling", "Speech synthesizer", "Shout", "Bellow", "Whoop", "Yell", "Children shouting", "Screaming", "Whispering", "Laughter", "Baby laughter", "Giggle", "Snicker", "Belly laugh", "Chuckle, chortle", "Crying, sobbing", "Baby cry, infant cry", "Whimper", "Wail, moan", "Sigh", "Singing", "Choir", "Yodeling", "Chant", "Mantra", "Child singing", "Synthetic singing", "Rapping", "Humming", "Groan", "Grunt", "Whistling", "Breathing", "Wheeze", "Snoring", "Gasp", "Pant", "Snort", "Cough", "Throat clearing", "Sneeze", "Sniff", "Run", "Shuffle", "Walk, footsteps", "Chewing, mastication", "Biting", "Gargling", "Stomach rumble", "Burping, eructation", "Hiccup", "Fart", "Hands", "Finger snapping", "Clapping", "Heart sounds, heartbeat", "Heart murmur", "Cheering", "Applause", "Chatter", "Crowd", "Hubbub, speech noise, speech babble", "Children playing", "Animal", "Domestic animals, pets", "Dog", "Bark", "Yip", "Howl", "Bow-wow", "Growling", "Whimper (dog)", "Cat", "Purr", "Meow", "Hiss", "Caterwaul", "Livestock, farm animals, working animals", "Horse", "Clip-clop", "Neigh, whinny", "Cattle, bovinae", "Moo", "Cowbell", "Pig", "Oink", "Goat", "Bleat", "Sheep", "Fowl", "Chicken, rooster", "Cluck", "Crowing, cock-a-doodle-doo", "Turkey", "Gobble", "Duck", "Quack", "Goose", "Honk", "Wild animals", "Roaring cats (lions, tigers)", "Roar", "Bird", "Bird vocalization, bird call, bird song", "Chirp, tweet", "Squawk", "Pigeon, dove", "Coo", "Crow", "Caw", "Owl", "Hoot", "Bird flight, flapping wings", "Canidae, dogs, wolves", "Rodents, rats, mice", "Mouse", "Patter", "Insect", "Cricket", "Mosquito", "Fly, housefly", "Buzz", "Bee, wasp, etc.", "Frog", "Croak", "Snake", "Rattle", "Whale vocalization", "Music", "Musical instrument", "Plucked string instrument", "Guitar", "Electric guitar", "Bass guitar", "Acoustic guitar", "Steel guitar, slide guitar", "Tapping (guitar technique)", "Strum", "Banjo", "Sitar", "Mandolin", "Zither", "Ukulele", "Keyboard (musical)", "Piano", "Electric piano", "Organ", "Electronic organ", "Hammond organ", "Synthesizer", "Sampler", "Harpsichord", "Percussion", "Drum kit", "Drum machine", "Drum", "Snare drum", "Rimshot", "Drum roll", "Bass drum", "Timpani", "Tabla", "Cymbal", "Hi-hat", "Wood block", "Tambourine", "Rattle (instrument)", "Maraca", "Gong", "Tubular bells", "Mallet percussion", "Marimba, xylophone", "Glockenspiel", "Vibraphone", "Steelpan", "Orchestra", "Brass instrument", "French horn", "Trumpet", "Trombone", "Bowed string instrument", "String section", "Violin, fiddle", "Pizzicato", "Cello", "Double bass", "Wind instrument, woodwind instrument", "Flute", "Saxophone", "Clarinet", "Harp", "Bell", "Church bell", "Jingle bell", "Bicycle bell", "Tuning fork", "Chime", "Wind chime", "Change ringing (campanology)", "Harmonica", "Accordion", "Bagpipes", "Didgeridoo", "Shofar", "Theremin", "Singing bowl", "Scratching (performance technique)", "Pop music", "Hip hop music", "Beatboxing", "Rock music", "Heavy metal", "Punk rock", "Grunge", "Progressive rock", "Rock and roll", "Psychedelic rock", "Rhythm and blues", "Soul music", "Reggae", "Country", "Swing music", "Bluegrass", "Funk", "Folk music", "Middle Eastern music", "Jazz", "Disco", "Classical music", "Opera", "Electronic music", "House music", "Techno", "Dubstep", "Drum and bass", "Electronica", "Electronic dance music", "Ambient music", "Trance music", "Music of Latin America", "Salsa music", "Flamenco", "Blues", "Music for children", "New-age music", "Vocal music", "A capella", "Music of Africa", "Afrobeat", "Christian music", "Gospel music", "Music of Asia", "Carnatic music", "Music of Bollywood", "Ska", "Traditional music", "Independent music", "Song", "Background music", "Theme music", "Jingle (music)", "Soundtrack music", "Lullaby", "Video game music", "Christmas music", "Dance music", "Wedding music", "Happy music", "Sad music", "Tender music", "Exciting music", "Angry music", "Scary music", "Wind", "Rustling leaves", "Wind noise (microphone)", "Thunderstorm", "Thunder", "Water", "Rain", "Raindrop", "Rain on surface", "Stream", "Waterfall", "Ocean", "Waves, surf", "Steam", "Gurgling", "Fire", "Crackle", "Vehicle", "Boat, Water vehicle", "Sailboat, sailing ship", "Rowboat, canoe, kayak", "Motorboat, speedboat", "Ship", "Motor vehicle (road)", "Car", "Vehicle horn, car horn, honking", "Toot", "Car alarm", "Power windows, electric windows", "Skidding", "Tire squeal", "Car passing by", "Race car, auto racing", "Truck", "Air brake", "Air horn, truck horn", "Reversing beeps", "Ice cream truck, ice cream van", "Bus", "Emergency vehicle", "Police car (siren)", "Ambulance (siren)", "Fire engine, fire truck (siren)", "Motorcycle", "Traffic noise, roadway noise", "Rail transport", "Train", "Train whistle", "Train horn", "Railroad car, train wagon", "Train wheels squealing", "Subway, metro, underground", "Aircraft", "Aircraft engine", "Jet engine", "Propeller, airscrew", "Helicopter", "Fixed-wing aircraft, airplane", "Bicycle", "Skateboard", "Engine", "Light engine (high frequency)", "Dental drill, dentist's drill", "Lawn mower", "Chainsaw", "Medium engine (mid frequency)", "Heavy engine (low frequency)", "Engine knocking", "Engine starting", "Idling", "Accelerating, revving, vroom", "Door", "Doorbell", "Ding-dong", "Sliding door", "Slam", "Knock", "Tap", "Squeak", "Cupboard open or close", "Drawer open or close", "Dishes, pots, and pans", "Cutlery, silverware", "Chopping (food)", "Frying (food)", "Microwave oven", "Blender", "Water tap, faucet", "Sink (filling or washing)", "Bathtub (filling or washing)", "Hair dryer", "Toilet flush", "Toothbrush", "Electric toothbrush", "Vacuum cleaner", "Zipper (clothing)", "Keys jangling", "Coin (dropping)", "Scissors", "Electric shaver, electric razor", "Shuffling cards", "Typing", "Typewriter", "Computer keyboard", "Writing", "Alarm", "Telephone", "Telephone bell ringing", "Ringtone", "Telephone dialing, DTMF", "Dial tone", "Busy signal", "Alarm clock", "Siren", "Civil defense siren", "Buzzer", "Smoke detector, smoke alarm", "Fire alarm", "Foghorn", "Whistle", "Steam whistle", "Mechanisms", "Ratchet, pawl", "Clock", "Tick", "Tick-tock", "Gears", "Pulleys", "Sewing machine", "Mechanical fan", "Air conditioning", "Cash register", "Printer", "Camera", "Single-lens reflex camera", "Tools", "Hammer", "Jackhammer", "Sawing", "Filing (rasp)", "Sanding", "Power tool", "Drill", "Explosion", "Gunshot, gunfire", "Machine gun", "Fusillade", "Artillery fire", "Cap gun", "Fireworks", "Firecracker", "Burst, pop", "Eruption", "Boom", "Wood", "Chop", "Splinter", "Crack", "Glass", "Chink, clink", "Shatter", "Liquid", "Splash, splatter", "Slosh", "Squish", "Drip", "Pour", "Trickle, dribble", "Gush", "Fill (with liquid)", "Spray", "Pump (liquid)", "Stir", "Boiling", "Sonar", "Arrow", "Whoosh, swoosh, swish", "Thump, thud", "Thunk", "Electronic tuner", "Effects unit", "Chorus effect", "Basketball bounce", "Bang", "Slap, smack", "Whack, thwack", "Smash, crash", "Breaking", "Bouncing", "Whip", "Flap", "Scratch", "Scrape", "Rub", "Roll", "Crushing", "Crumpling, crinkling", "Tearing", "Beep, bleep", "Ping", "Ding", "Clang", "Squeal", "Creak", "Rustle", "Whir", "Clatter", "Sizzle", "Clicking", "Clickety-clack", "Rumble", "Plop", "Jingle, tinkle", "Hum", "Zing", "Boing", "Crunch", "Silence", "Sine wave", "Harmonic", "Chirp tone", "Sound effect", "Pulse", "Inside, small room", "Inside, large room or hall", "Inside, public space", "Outside, urban or manmade", "Outside, rural or natural", "Reverberation", "Echo", "Noise", "Environmental noise", "Static", "Mains hum", "Distortion", "Sidetone", "Cacophony", "White noise", "Pink noise", "Throbbing", "Vibration", "Television", "Radio", "Field recording"]
|
package/src/analysis/runtime.mjs
CHANGED
|
@@ -70,8 +70,13 @@ export async function ensureModel(name, log = () => {}) {
|
|
|
70
70
|
return p;
|
|
71
71
|
}
|
|
72
72
|
const dest = join(HOME, 'models', 'analysis', manifest.version, entry.file);
|
|
73
|
-
if (existsSync(dest)) return dest;
|
|
74
73
|
const base = `${manifest.baseUrl}/${manifest.version}/models`;
|
|
74
|
+
// Ficheros que el modelo lee a su lado (pesos en un fichero aparte), con su sha256.
|
|
75
|
+
for (const x of entry.extra ?? []) {
|
|
76
|
+
const p = join(dirname(dest), x.file);
|
|
77
|
+
if (!existsSync(p)) await fetchChecked(`${base}/${x.file}`, p, { ...x, label: `${entry.label ?? entry.file} (weights)` }, log);
|
|
78
|
+
}
|
|
79
|
+
if (existsSync(dest)) return dest;
|
|
75
80
|
if (!entry.parts) {
|
|
76
81
|
await fetchChecked(`${base}/${entry.file}`, dest, entry, log);
|
|
77
82
|
return dest;
|
|
@@ -122,7 +127,9 @@ export function analysisMissing() {
|
|
|
122
127
|
if (!process.env.IGNIX_EDITOR_ORT && rt && !existsSync(join(rtDir, 'onnxruntime-node', 'package.json'))) out.push({ item: 'runtime', size: rt.size });
|
|
123
128
|
for (const [name, entry] of Object.entries(manifest.models)) {
|
|
124
129
|
if (process.env.IGNIX_EDITOR_MODELS) continue;
|
|
125
|
-
|
|
130
|
+
const dir = join(HOME, 'models', 'analysis', manifest.version);
|
|
131
|
+
const missing = [entry, ...(entry.extra ?? [])].filter((f) => !existsSync(join(dir, f.file)));
|
|
132
|
+
if (missing.length) out.push({ item: name, size: missing.reduce((n, f) => n + f.size, 0) });
|
|
126
133
|
}
|
|
127
134
|
return out;
|
|
128
135
|
}
|
package/src/analysis/serve.mjs
CHANGED
|
@@ -16,6 +16,7 @@ import { frames, probe } from './media.mjs';
|
|
|
16
16
|
import { analyze, edit, review } from './montage.mjs';
|
|
17
17
|
import { RUNTIME_VERSION, loadOrt } from './runtime.mjs';
|
|
18
18
|
import { cleanVoice, structure } from './sound.mjs';
|
|
19
|
+
import { measureStyle } from './style.mjs';
|
|
19
20
|
|
|
20
21
|
const MAX_SAMPLES = 2400;
|
|
21
22
|
const log = (...a) => console.error(...a);
|
|
@@ -99,6 +100,15 @@ export function createOps({ version, engine }) {
|
|
|
99
100
|
return structure({ ffmpeg, path: req.path });
|
|
100
101
|
},
|
|
101
102
|
|
|
103
|
+
// El estilo de edición medido en 1–5 vídeos (ADR-0023 E3); con `speech`, también el habla.
|
|
104
|
+
async style(req) {
|
|
105
|
+
const { ffmpeg, ffprobe } = await tools();
|
|
106
|
+
const speech = req.speech
|
|
107
|
+
? { whisper: await ensureTool('whisper', log), model: await ensureWhisperModel('small', log), language: req.language ?? null }
|
|
108
|
+
: null;
|
|
109
|
+
return measureStyle({ ffmpeg, ffprobe, paths: req.paths, speech, stills: req.stills ?? null, log });
|
|
110
|
+
},
|
|
111
|
+
|
|
102
112
|
async analyze(req) {
|
|
103
113
|
const { ffmpeg, ffprobe } = await tools();
|
|
104
114
|
return analyze({ ffmpeg, ffprobe, workdir: req.workdir, media: req.media ?? [], log });
|