@ignix-social/editor 0.1.3 → 0.1.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOTICE +30 -3
- package/README.md +3 -2
- package/licenses/onnxruntime-MIT.txt +21 -0
- package/licenses/yunet-MIT.txt +21 -0
- package/package.json +1 -1
- package/patches/index.mjs +38 -0
- package/scripts/build-analysis-runtime.mjs +45 -0
- package/scripts/build-ffmpeg-windows.sh +44 -0
- package/src/analysis/clip.mjs +142 -0
- package/src/analysis/detect.mjs +185 -0
- package/src/analysis/image.mjs +123 -0
- package/src/analysis/media.mjs +152 -0
- package/src/analysis/montage.mjs +356 -0
- package/src/analysis/runtime.mjs +115 -0
- package/src/analysis/serve.mjs +149 -0
- package/src/analysis/sound.mjs +99 -0
- package/src/binaries.json +142 -6
- package/src/cli.mjs +35 -24
- package/src/prepare.mjs +13 -2
- package/src/transcribe.mjs +65 -1
- package/vendor/hyperframes/UPSTREAM.json +1 -0
- package/vendor/hyperframes/dist/chunk-K2FTXJYH.js +16 -2
- package/vendor/hyperframes/dist/chunk-TVCRUG2S.js +1 -1
- package/vendor/hyperframes/dist/render-LNO62LEL.js +1 -1
- package/src/native/faces.swift +0 -60
|
@@ -0,0 +1,123 @@
|
|
|
1
|
+
// Operaciones de imagen en RGB (uint8, intercalado) para preparar la entrada de los modelos.
|
|
2
|
+
|
|
3
|
+
/** Reescalado bilineal de un RGB intercalado. */
|
|
4
|
+
export function resize(rgb, w, h, W, H) {
|
|
5
|
+
const out = new Uint8Array(W * H * 3);
|
|
6
|
+
const sx = w / W;
|
|
7
|
+
const sy = h / H;
|
|
8
|
+
for (let y = 0; y < H; y++) {
|
|
9
|
+
const fy = Math.max(0, (y + 0.5) * sy - 0.5);
|
|
10
|
+
const y0 = Math.min(h - 1, Math.floor(fy));
|
|
11
|
+
const y1 = Math.min(h - 1, y0 + 1);
|
|
12
|
+
const dy = fy - y0;
|
|
13
|
+
for (let x = 0; x < W; x++) {
|
|
14
|
+
const fx = Math.max(0, (x + 0.5) * sx - 0.5);
|
|
15
|
+
const x0 = Math.min(w - 1, Math.floor(fx));
|
|
16
|
+
const x1 = Math.min(w - 1, x0 + 1);
|
|
17
|
+
const dx = fx - x0;
|
|
18
|
+
const o = (y * W + x) * 3;
|
|
19
|
+
for (let c = 0; c < 3; c++) {
|
|
20
|
+
const a = rgb[(y0 * w + x0) * 3 + c];
|
|
21
|
+
const b = rgb[(y0 * w + x1) * 3 + c];
|
|
22
|
+
const d = rgb[(y1 * w + x0) * 3 + c];
|
|
23
|
+
const e = rgb[(y1 * w + x1) * 3 + c];
|
|
24
|
+
out[o + c] = Math.round((a * (1 - dx) + b * dx) * (1 - dy) + (d * (1 - dx) + e * dx) * dy);
|
|
25
|
+
}
|
|
26
|
+
}
|
|
27
|
+
}
|
|
28
|
+
return out;
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
// Núcleo cúbico de Keys (a = -0.5), el del reescalado «bicúbico» de PIL que usa CLIP.
|
|
32
|
+
const cubic = (x) => {
|
|
33
|
+
const a = -0.5;
|
|
34
|
+
x = Math.abs(x);
|
|
35
|
+
if (x <= 1) return ((a + 2) * x - (a + 3)) * x * x + 1;
|
|
36
|
+
if (x < 2) return (((x - 5) * x + 8) * x - 4) * a;
|
|
37
|
+
return 0;
|
|
38
|
+
};
|
|
39
|
+
|
|
40
|
+
/** Reescalado bicúbico separable con antialias al reducir (como PIL). */
|
|
41
|
+
export function resizeBicubic(rgb, w, h, W, H) {
|
|
42
|
+
const pass = (src, sw, sh, dw, horizontal) => {
|
|
43
|
+
const len = horizontal ? sw : sh;
|
|
44
|
+
const out = new Float32Array(horizontal ? dw * sh * 3 : sw * dw * 3);
|
|
45
|
+
const scale = len / dw;
|
|
46
|
+
const support = 2 * Math.max(1, scale);
|
|
47
|
+
const fscale = Math.max(1, scale);
|
|
48
|
+
for (let i = 0; i < dw; i++) {
|
|
49
|
+
const center = (i + 0.5) * scale;
|
|
50
|
+
const lo = Math.max(0, Math.floor(center - support));
|
|
51
|
+
const hi = Math.min(len, Math.ceil(center + support));
|
|
52
|
+
const ws = [];
|
|
53
|
+
let sum = 0;
|
|
54
|
+
for (let j = lo; j < hi; j++) {
|
|
55
|
+
const wgt = cubic((j + 0.5 - center) / fscale);
|
|
56
|
+
ws.push(wgt);
|
|
57
|
+
sum += wgt;
|
|
58
|
+
}
|
|
59
|
+
for (let k = 0; k < ws.length; k++) ws[k] /= sum || 1;
|
|
60
|
+
const lines = horizontal ? sh : sw;
|
|
61
|
+
for (let l = 0; l < lines; l++) {
|
|
62
|
+
for (let c = 0; c < 3; c++) {
|
|
63
|
+
let acc = 0;
|
|
64
|
+
for (let j = lo, k = 0; j < hi; j++, k++) {
|
|
65
|
+
const idx = horizontal ? (l * sw + j) * 3 + c : (j * sw + l) * 3 + c;
|
|
66
|
+
acc += src[idx] * ws[k];
|
|
67
|
+
}
|
|
68
|
+
const o = horizontal ? (l * dw + i) * 3 + c : (i * sw + l) * 3 + c;
|
|
69
|
+
out[o] = acc;
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
return out;
|
|
74
|
+
};
|
|
75
|
+
const a = pass(rgb, w, h, W, true); // W × h
|
|
76
|
+
const b = pass(a, W, h, H, false); // W × H
|
|
77
|
+
const out = new Uint8Array(W * H * 3);
|
|
78
|
+
for (let i = 0; i < out.length; i++) out[i] = Math.max(0, Math.min(255, Math.round(b[i])));
|
|
79
|
+
return out;
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
/** Tensor CHW float32 con normalización por canal (`mean`, `std` en la escala dada). */
|
|
83
|
+
export function toCHW(rgb, w, h, { scale = 1, mean = [0, 0, 0], std = [1, 1, 1], bgr = false } = {}) {
|
|
84
|
+
const out = new Float32Array(3 * w * h);
|
|
85
|
+
const plane = w * h;
|
|
86
|
+
for (let i = 0; i < plane; i++) {
|
|
87
|
+
for (let c = 0; c < 3; c++) {
|
|
88
|
+
const src = rgb[i * 3 + (bgr ? 2 - c : c)];
|
|
89
|
+
out[c * plane + i] = (src * scale - mean[c]) / std[c];
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
return out;
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
/** Pega la imagen arriba a la izquierda de un lienzo `S`×`S` relleno de `fill` (letterbox). */
|
|
96
|
+
export function letterbox(rgb, w, h, S, fill = 114) {
|
|
97
|
+
const r = Math.min(S / w, S / h);
|
|
98
|
+
const nw = Math.max(1, Math.round(w * r));
|
|
99
|
+
const nh = Math.max(1, Math.round(h * r));
|
|
100
|
+
const small = resize(rgb, w, h, nw, nh);
|
|
101
|
+
const out = new Uint8Array(S * S * 3).fill(fill);
|
|
102
|
+
for (let y = 0; y < nh; y++) out.set(small.subarray(y * nw * 3, (y + 1) * nw * 3), y * S * 3);
|
|
103
|
+
return { rgb: out, r };
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
/** Supresión de no máximos sobre cajas {x,y,w,h,score}. */
|
|
107
|
+
export function nms(boxes, iou = 0.5) {
|
|
108
|
+
const sorted = [...boxes].sort((a, b) => b.score - a.score);
|
|
109
|
+
const keep = [];
|
|
110
|
+
for (const b of sorted) {
|
|
111
|
+
if (keep.every((k) => overlap(k, b) <= iou)) keep.push(b);
|
|
112
|
+
}
|
|
113
|
+
return keep;
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
function overlap(a, b) {
|
|
117
|
+
const x0 = Math.max(a.x, b.x);
|
|
118
|
+
const y0 = Math.max(a.y, b.y);
|
|
119
|
+
const x1 = Math.min(a.x + a.w, b.x + b.w);
|
|
120
|
+
const y1 = Math.min(a.y + a.h, b.y + b.h);
|
|
121
|
+
const inter = Math.max(0, x1 - x0) * Math.max(0, y1 - y0);
|
|
122
|
+
return inter / (a.w * a.h + b.w * b.h - inter || 1);
|
|
123
|
+
}
|
|
@@ -0,0 +1,152 @@
|
|
|
1
|
+
// Leer vídeo, fotos y audio con el ffmpeg del Editor: datos del fichero, fotogramas
|
|
2
|
+
// muestreados en RGB (para los modelos) y el sonido como muestras (para medirlo).
|
|
3
|
+
import { spawn, spawnSync } from 'node:child_process';
|
|
4
|
+
|
|
5
|
+
/** Duración, fotogramas por segundo, tamaño y si tiene audio. */
|
|
6
|
+
export function probe(ffprobe, path) {
|
|
7
|
+
const r = spawnSync(ffprobe, ['-v', 'error', '-print_format', 'json', '-show_format', '-show_streams', path], { encoding: 'utf8', maxBuffer: 16 * 1024 * 1024 });
|
|
8
|
+
if (r.status !== 0) throw new Error(`Could not read ${path}: ${(r.stderr || '').slice(-200)}`);
|
|
9
|
+
const j = JSON.parse(r.stdout);
|
|
10
|
+
const v = (j.streams ?? []).find((s) => s.codec_type === 'video');
|
|
11
|
+
const rate = (s) => {
|
|
12
|
+
const [a, b] = String(s ?? '0/1').split('/').map(Number);
|
|
13
|
+
return b ? a / b : 0;
|
|
14
|
+
};
|
|
15
|
+
// Vídeo grabado con el móvil en vertical: el giro va en los metadatos.
|
|
16
|
+
const rot = Math.abs(Number(v?.tags?.rotate ?? (v?.side_data_list ?? []).find((d) => d.rotation !== undefined)?.rotation ?? 0)) % 180 === 90;
|
|
17
|
+
return {
|
|
18
|
+
duration: Number(j.format?.duration ?? v?.duration ?? 0) || 0,
|
|
19
|
+
fps: rate(v?.avg_frame_rate) || rate(v?.r_frame_rate) || 30,
|
|
20
|
+
width: rot ? v?.height ?? 0 : v?.width ?? 0,
|
|
21
|
+
height: rot ? v?.width ?? 0 : v?.height ?? 0,
|
|
22
|
+
hasVideo: !!v,
|
|
23
|
+
hasAudio: (j.streams ?? []).some((s) => s.codec_type === 'audio'),
|
|
24
|
+
};
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
/** Tamaño de trabajo que cabe en `max` px de lado manteniendo la proporción (par). */
|
|
28
|
+
export function fitSize(w, h, max) {
|
|
29
|
+
const s = Math.min(1, max / Math.max(w, h));
|
|
30
|
+
return { w: Math.max(2, Math.round((w * s) / 2) * 2), h: Math.max(2, Math.round((h * s) / 2) * 2) };
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
/**
|
|
34
|
+
* Fotogramas del vídeo a `fps` por segundo (o uno, si es una foto), en RGB y a un tamaño de
|
|
35
|
+
* trabajo de como mucho `max` px. Llama a `onFrame({ t, w, h, rgb })` en orden; si devuelve
|
|
36
|
+
* `false`, para. `start`/`end` acotan un tramo (segundos).
|
|
37
|
+
*/
|
|
38
|
+
export function frames(ffmpeg, path, { fps = 2, max = 640, width, height, start = 0, end = null, still = false, onFrame }) {
|
|
39
|
+
const { w, h } = fitSize(width, height, max);
|
|
40
|
+
const args = ['-nostdin', '-hide_banner', '-loglevel', 'error'];
|
|
41
|
+
if (start > 0) args.push('-ss', start.toFixed(3));
|
|
42
|
+
args.push('-i', path);
|
|
43
|
+
if (end !== null) args.push('-t', Math.max(0.05, end - start).toFixed(3));
|
|
44
|
+
const vf = still ? `scale=${w}:${h}` : `fps=${fps},scale=${w}:${h}`;
|
|
45
|
+
args.push('-an', '-vf', vf, ...(still ? ['-frames:v', '1'] : []), '-f', 'rawvideo', '-pix_fmt', 'rgb24', 'pipe:1');
|
|
46
|
+
const size = w * h * 3;
|
|
47
|
+
return new Promise((resolve, reject) => {
|
|
48
|
+
const p = spawn(ffmpeg, args, { stdio: ['ignore', 'pipe', 'pipe'] });
|
|
49
|
+
let buf = Buffer.alloc(0);
|
|
50
|
+
let k = 0;
|
|
51
|
+
let stopped = false;
|
|
52
|
+
let err = '';
|
|
53
|
+
let chain = Promise.resolve();
|
|
54
|
+
p.stderr.on('data', (d) => (err = (err + d).slice(-400)));
|
|
55
|
+
p.stdout.on('data', (d) => {
|
|
56
|
+
if (stopped) return;
|
|
57
|
+
buf = buf.length ? Buffer.concat([buf, d]) : d;
|
|
58
|
+
while (buf.length >= size && !stopped) {
|
|
59
|
+
const rgb = buf.subarray(0, size);
|
|
60
|
+
buf = buf.subarray(size);
|
|
61
|
+
// El fps de ffmpeg toma el fotograma de cada tramo de 1/fps: su instante es el centro.
|
|
62
|
+
const t = still ? 0 : start + (k + 0.5) / fps;
|
|
63
|
+
k += 1;
|
|
64
|
+
const copy = Buffer.from(rgb);
|
|
65
|
+
chain = chain.then(async () => {
|
|
66
|
+
if (stopped) return;
|
|
67
|
+
if ((await onFrame({ t, w, h, rgb: copy })) === false) {
|
|
68
|
+
stopped = true;
|
|
69
|
+
p.kill();
|
|
70
|
+
}
|
|
71
|
+
});
|
|
72
|
+
// Con muchos fotogramas en cola, que ffmpeg espere (memoria acotada).
|
|
73
|
+
p.stdout.pause();
|
|
74
|
+
chain.then(() => p.stdout.resume(), () => p.stdout.resume());
|
|
75
|
+
}
|
|
76
|
+
});
|
|
77
|
+
p.on('error', reject);
|
|
78
|
+
p.on('close', (code) => {
|
|
79
|
+
chain.then(() => {
|
|
80
|
+
if (code !== 0 && !stopped && k === 0) reject(new Error(`Could not read the frames of ${path}: ${err.trim()}`));
|
|
81
|
+
else resolve({ count: k, w, h });
|
|
82
|
+
}, reject);
|
|
83
|
+
});
|
|
84
|
+
});
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
/**
|
|
88
|
+
* El sonido como muestras float32 mono a `rate` Hz, en bloques. `start`/`end` acotan un tramo.
|
|
89
|
+
* Devuelve false si el fichero no tiene audio.
|
|
90
|
+
*/
|
|
91
|
+
export function pcm(ffmpeg, path, { rate = 16000, start = 0, end = null, onSamples }) {
|
|
92
|
+
const args = ['-nostdin', '-hide_banner', '-loglevel', 'error'];
|
|
93
|
+
if (start > 0) args.push('-ss', start.toFixed(3));
|
|
94
|
+
args.push('-i', path);
|
|
95
|
+
if (end !== null) args.push('-t', Math.max(0.05, end - start).toFixed(3));
|
|
96
|
+
args.push('-vn', '-ac', '1', '-ar', String(rate), '-f', 'f32le', 'pipe:1');
|
|
97
|
+
return new Promise((resolve, reject) => {
|
|
98
|
+
const p = spawn(ffmpeg, args, { stdio: ['ignore', 'pipe', 'pipe'] });
|
|
99
|
+
let carry = Buffer.alloc(0);
|
|
100
|
+
let err = '';
|
|
101
|
+
let any = false;
|
|
102
|
+
p.stderr.on('data', (d) => (err = (err + d).slice(-400)));
|
|
103
|
+
p.stdout.on('data', (d) => {
|
|
104
|
+
any = true;
|
|
105
|
+
const all = carry.length ? Buffer.concat([carry, d]) : d;
|
|
106
|
+
const n = Math.floor(all.length / 4);
|
|
107
|
+
const copy = new Float32Array(n);
|
|
108
|
+
for (let i = 0; i < n; i++) copy[i] = all.readFloatLE(i * 4);
|
|
109
|
+
carry = all.subarray(n * 4);
|
|
110
|
+
try {
|
|
111
|
+
onSamples(copy);
|
|
112
|
+
} catch (e) {
|
|
113
|
+
p.kill();
|
|
114
|
+
reject(e);
|
|
115
|
+
}
|
|
116
|
+
});
|
|
117
|
+
p.on('error', reject);
|
|
118
|
+
p.on('close', (code) => {
|
|
119
|
+
if (!any && /does not contain any stream|Output file .* does not contain|matches no streams/i.test(err)) resolve(false);
|
|
120
|
+
else if (code !== 0 && !any) reject(new Error(`Could not read the audio of ${path}: ${err.trim()}`));
|
|
121
|
+
else resolve(true);
|
|
122
|
+
});
|
|
123
|
+
});
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
/** Niveles en dBFS de ventanas de `windowSec` (RMS). */
|
|
127
|
+
export async function levels(ffmpeg, path, { windowSec = 0.05, start = 0, end = null, maxSec = 3600 } = {}) {
|
|
128
|
+
const rate = 16000;
|
|
129
|
+
const win = Math.round(rate * windowSec);
|
|
130
|
+
const out = [];
|
|
131
|
+
let acc = 0;
|
|
132
|
+
let n = 0;
|
|
133
|
+
let seen = 0;
|
|
134
|
+
const ok = await pcm(ffmpeg, path, {
|
|
135
|
+
rate,
|
|
136
|
+
start,
|
|
137
|
+
end,
|
|
138
|
+
onSamples: (x) => {
|
|
139
|
+
seen += x.length;
|
|
140
|
+
if (seen > maxSec * rate) throw new Error('too long');
|
|
141
|
+
for (let i = 0; i < x.length; i++) {
|
|
142
|
+
acc += x[i] * x[i];
|
|
143
|
+
if (++n === win) {
|
|
144
|
+
out.push(10 * Math.log10(acc / win + 1e-12));
|
|
145
|
+
acc = 0;
|
|
146
|
+
n = 0;
|
|
147
|
+
}
|
|
148
|
+
}
|
|
149
|
+
},
|
|
150
|
+
});
|
|
151
|
+
return ok ? out : null;
|
|
152
|
+
}
|
|
@@ -0,0 +1,356 @@
|
|
|
1
|
+
// El material del cliente como un ÍNDICE DE PLANOS, y el montaje que lo usa (ADR-0023 E1):
|
|
2
|
+
// · analyze: cada vídeo partido en sus planos reales (cambios de escena; una toma larga en
|
|
3
|
+
// trozos de ~4 s), cada plano con tres fotogramas, su movimiento, dónde está la acción o la
|
|
4
|
+
// cara, su tamaño de plano y su vector CLIP. Lo ya analizado no se repite.
|
|
5
|
+
// · edit: para cada momento del guion, el plano que ENSEÑA lo que se dice (CLIP imagen↔texto),
|
|
6
|
+
// sin repetir fuente seguida, variando el tamaño de plano, y el trozo del plano que mejor
|
|
7
|
+
// encaja; más planos alternativos para los subplanos.
|
|
8
|
+
// · review: la línea de tiempo terminada, juzgada (¿parece un pase de diapositivas?, ¿varía?).
|
|
9
|
+
// Reglas propias de Ignix (no se copia código de OpenMontage).
|
|
10
|
+
import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs';
|
|
11
|
+
import { join } from 'node:path';
|
|
12
|
+
import { dot, embedImage, embedTexts, pool } from './clip.mjs';
|
|
13
|
+
import { faces as detectFaces } from './detect.mjs';
|
|
14
|
+
import { frames, probe } from './media.mjs';
|
|
15
|
+
import { cuts } from './sound.mjs';
|
|
16
|
+
|
|
17
|
+
const DIM = 512;
|
|
18
|
+
const SHOT_SIZES = ['close-up', 'medium', 'wide'];
|
|
19
|
+
// Frases fijas con las que CLIP compara cada plano (tamaño de plano sin entrenar nada).
|
|
20
|
+
const SIZE_PROMPTS = {
|
|
21
|
+
'close-up': 'an extreme close-up photo of a detail, a face or hands filling the frame',
|
|
22
|
+
medium: 'a medium shot photo of a person from the waist up',
|
|
23
|
+
wide: 'a wide shot photo of a whole room, place or scene',
|
|
24
|
+
};
|
|
25
|
+
const MAX_WINDOWS = 12;
|
|
26
|
+
const TAG_WEIGHT = 0.25;
|
|
27
|
+
const NEAR_DUPLICATE = 0.92;
|
|
28
|
+
|
|
29
|
+
// ── el índice: index.json + un .f32 por plano con [conjunto, etiqueta, fotogramas…]
|
|
30
|
+
function store(workdir) {
|
|
31
|
+
const dir = join(workdir, 'editor-corpus');
|
|
32
|
+
mkdirSync(dir, { recursive: true });
|
|
33
|
+
const indexPath = join(dir, 'index.json');
|
|
34
|
+
const index = existsSync(indexPath) ? JSON.parse(readFileSync(indexPath, 'utf8')) : { clips: {} };
|
|
35
|
+
const vecs = new Map();
|
|
36
|
+
const file = (id) => join(dir, `${id.replace(/[^\w.-]/g, '_')}.f32`);
|
|
37
|
+
return {
|
|
38
|
+
index,
|
|
39
|
+
save: () => writeFileSync(indexPath, JSON.stringify(index)),
|
|
40
|
+
put(id, meta, pooled, tag, frameVecs) {
|
|
41
|
+
const all = new Float32Array(DIM * (2 + frameVecs.length));
|
|
42
|
+
all.set(pooled, 0);
|
|
43
|
+
all.set(tag, DIM);
|
|
44
|
+
frameVecs.forEach((v, i) => all.set(v, DIM * (2 + i)));
|
|
45
|
+
writeFileSync(file(id), Buffer.from(all.buffer));
|
|
46
|
+
index.clips[id] = meta;
|
|
47
|
+
vecs.set(id, all);
|
|
48
|
+
},
|
|
49
|
+
vectors(id) {
|
|
50
|
+
if (!vecs.has(id)) {
|
|
51
|
+
const b = readFileSync(file(id));
|
|
52
|
+
vecs.set(id, new Float32Array(b.buffer, b.byteOffset, b.length / 4));
|
|
53
|
+
}
|
|
54
|
+
const all = vecs.get(id);
|
|
55
|
+
const n = all.length / DIM - 2;
|
|
56
|
+
return {
|
|
57
|
+
pooled: all.subarray(0, DIM),
|
|
58
|
+
tag: all.subarray(DIM, 2 * DIM),
|
|
59
|
+
frames: Array.from({ length: n }, (_, i) => all.subarray(DIM * (2 + i), DIM * (3 + i))),
|
|
60
|
+
};
|
|
61
|
+
},
|
|
62
|
+
};
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
/** Movimiento (diferencia media de grises, 0–255) entre dos fotogramas y dónde ocurre (0..1). */
|
|
66
|
+
export function motionOf(a, b, w, h) {
|
|
67
|
+
if (!a || !b) return { motion: 0, focus: null };
|
|
68
|
+
const cols = new Float64Array(w);
|
|
69
|
+
let sum = 0;
|
|
70
|
+
for (let i = 0; i < w * h; i++) {
|
|
71
|
+
const ga = 0.299 * a[i * 3] + 0.587 * a[i * 3 + 1] + 0.114 * a[i * 3 + 2];
|
|
72
|
+
const gb = 0.299 * b[i * 3] + 0.587 * b[i * 3 + 1] + 0.114 * b[i * 3 + 2];
|
|
73
|
+
const d = Math.abs(ga - gb);
|
|
74
|
+
sum += d;
|
|
75
|
+
cols[i % w] += d;
|
|
76
|
+
}
|
|
77
|
+
const motion = sum / (w * h);
|
|
78
|
+
let focus = null;
|
|
79
|
+
if (motion > 2) {
|
|
80
|
+
let tot = 0;
|
|
81
|
+
let acc = 0;
|
|
82
|
+
for (let x = 0; x < w; x++) {
|
|
83
|
+
tot += cols[x];
|
|
84
|
+
acc += cols[x] * x;
|
|
85
|
+
}
|
|
86
|
+
if (tot > 0) focus = acc / tot / w;
|
|
87
|
+
}
|
|
88
|
+
return { motion, focus };
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
export async function analyze({ ffmpeg, ffprobe, workdir, media, log = () => {} }) {
|
|
92
|
+
const db = store(workdir);
|
|
93
|
+
const sizeVecs = await embedTexts(SHOT_SIZES.map((s) => SIZE_PROMPTS[s]), log);
|
|
94
|
+
const shots = [];
|
|
95
|
+
for (const m of media) {
|
|
96
|
+
const { id: asset, kind, path } = m;
|
|
97
|
+
const label = String(m.label ?? '').trim();
|
|
98
|
+
let windows = [];
|
|
99
|
+
let faceSamples = [];
|
|
100
|
+
let info = null;
|
|
101
|
+
try {
|
|
102
|
+
info = probe(ffprobe, path);
|
|
103
|
+
} catch (e) {
|
|
104
|
+
log(`analyze ${asset}: ${e.message}`);
|
|
105
|
+
continue;
|
|
106
|
+
}
|
|
107
|
+
if (kind === 'video') {
|
|
108
|
+
const duration = Number(m.duration) || info.duration;
|
|
109
|
+
// Dónde está la cara (la más grande) dos veces por segundo: encuadre y tamaño de plano.
|
|
110
|
+
await frames(ffmpeg, path, {
|
|
111
|
+
fps: 2,
|
|
112
|
+
max: 640,
|
|
113
|
+
width: info.width,
|
|
114
|
+
height: info.height,
|
|
115
|
+
onFrame: async (f) => {
|
|
116
|
+
const found = await detectFaces(f, log);
|
|
117
|
+
const big = found.sort((a, b) => b.w * b.h - a.w * a.h)[0];
|
|
118
|
+
if (big) faceSamples.push({ t: f.t, x: big.x + big.w / 2, area: big.w * big.h });
|
|
119
|
+
},
|
|
120
|
+
}).catch((e) => log(`faces ${asset}: ${e.message}`));
|
|
121
|
+
const found = await cuts(ffmpeg, path, { minSec: 0.8, maxSec: 24 * 3600 }).catch(() => []);
|
|
122
|
+
const bounds = [0, ...found, duration];
|
|
123
|
+
windows = [];
|
|
124
|
+
for (let i = 0; i + 1 < bounds.length; i++) if (bounds[i + 1] - bounds[i] >= 0.8) windows.push([bounds[i], bounds[i + 1]]);
|
|
125
|
+
if (!windows.length && duration > 0) windows = [[0, duration]];
|
|
126
|
+
// Una toma larga: trozos de ~4 s, para poder elegir cada uno.
|
|
127
|
+
const split = [];
|
|
128
|
+
for (const [a, b] of windows) {
|
|
129
|
+
const n = Math.max(1, Math.floor((b - a) / 4));
|
|
130
|
+
for (let k = 0; k < n; k++) split.push([a + (k * (b - a)) / n, a + ((k + 1) * (b - a)) / n]);
|
|
131
|
+
}
|
|
132
|
+
windows = split.slice(0, MAX_WINDOWS);
|
|
133
|
+
} else windows = [[0, 0]];
|
|
134
|
+
|
|
135
|
+
for (const [k, [a, b]] of windows.entries()) {
|
|
136
|
+
const clip = `${asset}~${k}`;
|
|
137
|
+
const known = db.index.clips[clip];
|
|
138
|
+
if (known) {
|
|
139
|
+
shots.push({ clip, asset, start: a, end: b, motion: known.motion, size: known.size, focus: known.focus });
|
|
140
|
+
continue;
|
|
141
|
+
}
|
|
142
|
+
const grabbed = [];
|
|
143
|
+
if (kind === 'video') {
|
|
144
|
+
await frames(ffmpeg, path, { fps: 3 / Math.max(0.1, b - a), max: 448, width: info.width, height: info.height, start: a, end: b, onFrame: (f) => void grabbed.push(f) }).catch(() => {});
|
|
145
|
+
} else {
|
|
146
|
+
await frames(ffmpeg, path, { still: true, max: 448, width: info.width, height: info.height, onFrame: (f) => void grabbed.push(f) }).catch(() => {});
|
|
147
|
+
}
|
|
148
|
+
if (!grabbed.length) continue;
|
|
149
|
+
const g = grabbed.slice(0, 3);
|
|
150
|
+
let { motion, focus } = kind === 'video' && g.length >= 2 ? motionOf(g[0].rgb, g[Math.floor(g.length / 2)].rgb, g[0].w, g[0].h) : { motion: 0, focus: null };
|
|
151
|
+
let faceSize = null;
|
|
152
|
+
const inside = faceSamples.filter((s) => s.t >= a && s.t <= b);
|
|
153
|
+
if (inside.length) {
|
|
154
|
+
// Una cara en el plano lo encuadra; si no, dónde está la acción.
|
|
155
|
+
const tot = inside.reduce((s, x) => s + x.area, 0);
|
|
156
|
+
focus = inside.reduce((s, x) => s + x.x * x.area, 0) / tot;
|
|
157
|
+
// El tamaño de la cara en el cuadro ES el tamaño de plano (gramática de montaje).
|
|
158
|
+
const side = Math.sqrt(Math.max(...inside.map((x) => x.area)));
|
|
159
|
+
faceSize = side > 0.3 ? 'close-up' : side < 0.1 ? 'wide' : 'medium';
|
|
160
|
+
} else if (kind !== 'video') {
|
|
161
|
+
const f = (await detectFaces(g[0], log)).sort((p, q) => q.w * q.h - p.w * p.h)[0];
|
|
162
|
+
if (f) focus = f.x + f.w / 2;
|
|
163
|
+
}
|
|
164
|
+
const frameVecs = [];
|
|
165
|
+
for (const f of g) frameVecs.push(await embedImage(f, log));
|
|
166
|
+
const pooled = pool(frameVecs);
|
|
167
|
+
const [tag] = await embedTexts([label || (kind === 'video' ? 'a video clip' : 'a photo')], log);
|
|
168
|
+
const size = faceSize ?? SHOT_SIZES[sizeVecs.map((v) => dot(pooled, v)).reduce((bi, v, i, arr) => (v > arr[bi] ? i : bi), 0)];
|
|
169
|
+
const fx = focus === null || focus === undefined ? null : +Math.min(1, Math.max(0, focus)).toFixed(3);
|
|
170
|
+
const times = g.map((f) => +f.t.toFixed(3));
|
|
171
|
+
db.put(clip, { asset, kind, label, start: a, end: b, duration: kind === 'video' ? +(b - a).toFixed(3) : 0, motion, size, focus: fx, times }, pooled, tag, frameVecs);
|
|
172
|
+
shots.push({ clip, asset, start: a, end: b, motion, size, focus: fx });
|
|
173
|
+
}
|
|
174
|
+
db.save();
|
|
175
|
+
}
|
|
176
|
+
db.save();
|
|
177
|
+
return { shots, corpus: Object.keys(db.index.clips).length };
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
/** El trozo del plano cuyos fotogramas se parecen más al momento. */
|
|
181
|
+
function windowFor(shot, meta, frameVecs, seconds, q) {
|
|
182
|
+
const a = Number(shot.start);
|
|
183
|
+
const b = Number(shot.end);
|
|
184
|
+
if (b - a <= seconds + 0.2) return [a, b];
|
|
185
|
+
let bestT = (a + b) / 2;
|
|
186
|
+
if (meta?.times?.length && frameVecs.length) {
|
|
187
|
+
let best = -Infinity;
|
|
188
|
+
frameVecs.forEach((v, i) => {
|
|
189
|
+
const s = dot(v, q);
|
|
190
|
+
if (s > best) {
|
|
191
|
+
best = s;
|
|
192
|
+
bestT = meta.times[i];
|
|
193
|
+
}
|
|
194
|
+
});
|
|
195
|
+
}
|
|
196
|
+
const start = Math.min(Math.max(a + 0.1, bestT - seconds / 2), b - seconds - 0.05);
|
|
197
|
+
return [+start.toFixed(2), +(start + seconds).toFixed(2)];
|
|
198
|
+
}
|
|
199
|
+
|
|
200
|
+
export async function edit({ workdir, shots: given = [], beats, allowed = null, preferred = [], minScore = 0.3, minVisual = 0.24, log = () => {} }) {
|
|
201
|
+
const db = store(workdir);
|
|
202
|
+
const shotsById = new Map(given.map((s) => [s.clip, s]));
|
|
203
|
+
const allow = allowed ? new Set(allowed) : null;
|
|
204
|
+
const prefer = new Set(preferred);
|
|
205
|
+
const queries = beats.map((b) => String(b.query ?? '').trim().slice(0, 300) || 'a scene');
|
|
206
|
+
const qvecs = await embedTexts(queries, log);
|
|
207
|
+
const clips = Object.entries(db.index.clips).map(([id, meta]) => ({ id, ...meta }));
|
|
208
|
+
const visual = (id, i) => dot(db.vectors(id).pooled, qvecs[i]);
|
|
209
|
+
|
|
210
|
+
// Para cada momento: los planos ordenados por lo que ENSEÑAN (CLIP) y, menos, por su etiqueta.
|
|
211
|
+
const ranked = beats.map((beat, i) => {
|
|
212
|
+
const fixed = beat.asset;
|
|
213
|
+
const floor = Number(beat.min_visual ?? minVisual);
|
|
214
|
+
return clips
|
|
215
|
+
.filter((c) => (beat.kind === 'video' || beat.kind === 'image' ? c.kind === beat.kind : true))
|
|
216
|
+
.filter((c) => !allow || allow.has(c.asset))
|
|
217
|
+
.filter((c) => !fixed || c.asset === fixed)
|
|
218
|
+
.map((c) => {
|
|
219
|
+
const v = db.vectors(c.id);
|
|
220
|
+
const vis = dot(v.pooled, qvecs[i]);
|
|
221
|
+
return { score: (1 - TAG_WEIGHT) * vis + TAG_WEIGHT * dot(v.tag, qvecs[i]), visual: vis, clip: c };
|
|
222
|
+
})
|
|
223
|
+
.filter((x) => fixed || (x.score >= minScore && x.visual >= floor))
|
|
224
|
+
.sort((p, q) => q.score - p.score);
|
|
225
|
+
});
|
|
226
|
+
|
|
227
|
+
const used = new Set();
|
|
228
|
+
const picks = new Array(beats.length).fill(null);
|
|
229
|
+
const choose = (i, avoid) => {
|
|
230
|
+
const prev = i > 0 ? picks[i - 1]?.clip : null;
|
|
231
|
+
const next = i + 1 < picks.length ? picks[i + 1]?.clip : null;
|
|
232
|
+
const fixed = beats[i].asset;
|
|
233
|
+
let cands = ranked[i].filter((x) => !used.has(x.clip.id) && !avoid.has(x.clip.id) && (fixed || ((!prev || x.clip.asset !== prev.asset) && (!next || x.clip.asset !== next.asset))));
|
|
234
|
+
if (!cands.length) return null;
|
|
235
|
+
// Lo que adjuntaron para esta pieza antes que el resto de su biblioteca.
|
|
236
|
+
const mine = cands.filter((x) => prefer.has(x.clip.asset));
|
|
237
|
+
if (mine.length) cands = [...mine, ...cands.filter((x) => !prefer.has(x.clip.asset))];
|
|
238
|
+
let best = cands[0];
|
|
239
|
+
// Variar el tamaño de plano cuando hay otro casi igual de bueno.
|
|
240
|
+
if (prev?.size && best.clip.size === prev.size) {
|
|
241
|
+
const alt = cands.find((x) => x.score >= best.score - 0.02 && x.clip.size !== prev.size);
|
|
242
|
+
if (alt) best = alt;
|
|
243
|
+
}
|
|
244
|
+
return best;
|
|
245
|
+
};
|
|
246
|
+
for (let i = 0; i < beats.length; i++) {
|
|
247
|
+
picks[i] = choose(i, new Set());
|
|
248
|
+
if (picks[i]) used.add(picks[i].clip.id);
|
|
249
|
+
}
|
|
250
|
+
// Casi duplicados (dos planos que se ven iguales): el segundo se cambia por otro distinto.
|
|
251
|
+
let swapped = 0;
|
|
252
|
+
const kept = [];
|
|
253
|
+
for (let i = 0; i < picks.length; i++) {
|
|
254
|
+
const p = picks[i];
|
|
255
|
+
if (!p) continue;
|
|
256
|
+
const pv = db.vectors(p.clip.id).pooled;
|
|
257
|
+
const dup = kept.some((id) => dot(db.vectors(id).pooled, pv) > NEAR_DUPLICATE);
|
|
258
|
+
if (dup && !beats[i].asset) {
|
|
259
|
+
used.delete(p.clip.id);
|
|
260
|
+
const alt = choose(i, new Set([p.clip.id, ...kept]));
|
|
261
|
+
picks[i] = alt ?? p;
|
|
262
|
+
used.add(picks[i].clip.id);
|
|
263
|
+
if (alt) swapped += 1;
|
|
264
|
+
}
|
|
265
|
+
kept.push(picks[i].clip.id);
|
|
266
|
+
}
|
|
267
|
+
|
|
268
|
+
const cut = (i, x, seconds) => {
|
|
269
|
+
const c = x.clip;
|
|
270
|
+
const shot = shotsById.get(c.id) ?? { start: c.start ?? 0, end: c.end ?? c.duration ?? 0 };
|
|
271
|
+
const [a, b] = c.kind === 'video' ? windowFor(shot, c, db.vectors(c.id).frames, seconds, qvecs[i]) : [0, 0];
|
|
272
|
+
return {
|
|
273
|
+
beat: beats[i].beat,
|
|
274
|
+
asset: c.asset,
|
|
275
|
+
clip: c.id,
|
|
276
|
+
kind: c.kind,
|
|
277
|
+
in: a,
|
|
278
|
+
out: b,
|
|
279
|
+
score: +x.score.toFixed(4),
|
|
280
|
+
visual: +visual(c.id, i).toFixed(4),
|
|
281
|
+
size: c.size || null,
|
|
282
|
+
motion: +Number(c.motion || 0).toFixed(2),
|
|
283
|
+
focus: c.focus ?? null,
|
|
284
|
+
};
|
|
285
|
+
};
|
|
286
|
+
const cutsOut = [];
|
|
287
|
+
const unmatched = [];
|
|
288
|
+
picks.forEach((p, i) => (p ? cutsOut.push(cut(i, p, Number(beats[i].seconds))) : unmatched.push(beats[i].beat)));
|
|
289
|
+
// Planos alternativos (subplanos): fuentes distintas, planos sin usar.
|
|
290
|
+
const alternates = [];
|
|
291
|
+
beats.forEach((beat, i) => {
|
|
292
|
+
const want = Number(beat.alternates ?? 0);
|
|
293
|
+
if (want <= 0 || beat.asset) return;
|
|
294
|
+
const seen = new Set([picks[i]?.clip.asset]);
|
|
295
|
+
const part = Number(beat.seconds) / (want + 1);
|
|
296
|
+
for (const x of ranked[i]) {
|
|
297
|
+
if (alternates.filter((a) => a.beat === beat.beat).length >= want) break;
|
|
298
|
+
if (used.has(x.clip.id) || seen.has(x.clip.asset)) continue;
|
|
299
|
+
used.add(x.clip.id);
|
|
300
|
+
seen.add(x.clip.asset);
|
|
301
|
+
alternates.push(cut(i, x, Math.max(1.2, part)));
|
|
302
|
+
}
|
|
303
|
+
});
|
|
304
|
+
return { cuts: cutsOut, alternates, unmatched, swapped };
|
|
305
|
+
}
|
|
306
|
+
|
|
307
|
+
/**
|
|
308
|
+
* Juicio de la línea de tiempo terminada. Riesgo de «pase de diapositivas» (0..1, más alto =
|
|
309
|
+
* peor) por dimensiones, y variedad con sus fallos concretos.
|
|
310
|
+
*/
|
|
311
|
+
export function review({ scenes }) {
|
|
312
|
+
const n = scenes.length || 1;
|
|
313
|
+
const pairs = Math.max(1, scenes.length - 1);
|
|
314
|
+
const same = (f) => scenes.slice(1).filter((s, i) => f(s) && f(s) === f(scenes[i])).length / pairs;
|
|
315
|
+
const longestRun = (f) => {
|
|
316
|
+
let best = 0;
|
|
317
|
+
let run = 0;
|
|
318
|
+
scenes.forEach((s, i) => {
|
|
319
|
+
run = i && f(s) === f(scenes[i - 1]) ? run + 1 : 1;
|
|
320
|
+
best = Math.max(best, run);
|
|
321
|
+
});
|
|
322
|
+
return best;
|
|
323
|
+
};
|
|
324
|
+
const dimensions = {
|
|
325
|
+
still_share: scenes.filter((s) => (s.type === 'still' || s.type === 'text_card') && s.shot_language?.camera_movement === 'static').length / n,
|
|
326
|
+
text_cards: scenes.filter((s) => s.type === 'text_card').length / n,
|
|
327
|
+
camera_monotony: same((s) => s.shot_language?.camera_movement),
|
|
328
|
+
size_monotony: same((s) => (s.shot_language?.shot_size === 'none' ? null : s.shot_language?.shot_size)),
|
|
329
|
+
type_monotony: longestRun((s) => s.type) / n,
|
|
330
|
+
};
|
|
331
|
+
const values = Object.values(dimensions);
|
|
332
|
+
const average = +(values.reduce((a, b) => a + b, 0) / values.length).toFixed(3);
|
|
333
|
+
const violations = [];
|
|
334
|
+
const runs = (label, f, min) => {
|
|
335
|
+
let run = 1;
|
|
336
|
+
for (let i = 1; i <= scenes.length; i++) {
|
|
337
|
+
if (i < scenes.length && f(scenes[i]) && f(scenes[i]) === f(scenes[i - 1])) run += 1;
|
|
338
|
+
else {
|
|
339
|
+
if (run >= min) violations.push(`${run} consecutive scenes with the same ${label} (${f(scenes[i - 1])}), ending at ${scenes[i - 1].id}`);
|
|
340
|
+
run = 1;
|
|
341
|
+
}
|
|
342
|
+
}
|
|
343
|
+
};
|
|
344
|
+
runs('shot size', (s) => (s.shot_language?.shot_size === 'none' ? null : s.shot_language?.shot_size), 3);
|
|
345
|
+
runs('camera movement', (s) => s.shot_language?.camera_movement, 3);
|
|
346
|
+
runs('type', (s) => (s.type === 'text_card' ? 'text_card' : null), 3);
|
|
347
|
+
const score = +Math.max(0, 1 - violations.length / n).toFixed(3);
|
|
348
|
+
return {
|
|
349
|
+
slideshow_risk: {
|
|
350
|
+
average,
|
|
351
|
+
verdict: average < 0.35 ? 'low' : average < 0.6 ? 'medium' : 'high',
|
|
352
|
+
dimensions: Object.fromEntries(Object.entries(dimensions).map(([k, v]) => [k, +v.toFixed(3)])),
|
|
353
|
+
},
|
|
354
|
+
variation: { score, verdict: violations.length ? 'needs_variety' : 'varied', violations: violations.slice(0, 6), metadata_only: 0 },
|
|
355
|
+
};
|
|
356
|
+
}
|