@ignix-social/editor 0.1.3 → 0.1.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOTICE +37 -3
- package/README.md +3 -2
- package/licenses/onnxruntime-MIT.txt +21 -0
- package/licenses/yunet-MIT.txt +21 -0
- package/package.json +2 -1
- package/patches/index.mjs +38 -0
- package/scripts/build-analysis-runtime.mjs +45 -0
- package/scripts/build-ffmpeg-windows.sh +44 -0
- package/src/analysis/clip.mjs +142 -0
- package/src/analysis/detect.mjs +185 -0
- package/src/analysis/image.mjs +123 -0
- package/src/analysis/media.mjs +152 -0
- package/src/analysis/montage.mjs +356 -0
- package/src/analysis/runtime.mjs +128 -0
- package/src/analysis/serve.mjs +149 -0
- package/src/analysis/sound.mjs +99 -0
- package/src/binaries.json +142 -6
- package/src/binaries.mjs +17 -4
- package/src/cli.mjs +64 -71
- package/src/creator/build.mjs +344 -0
- package/src/creator/fit.mjs +58 -0
- package/src/creator/job.mjs +163 -0
- package/src/creator/render.mjs +22 -0
- package/src/creator/runtime/creator.js +1026 -0
- package/src/engine.mjs +98 -0
- package/src/fetch.mjs +30 -0
- package/src/prepare.mjs +13 -2
- package/src/transcribe.mjs +65 -1
- package/vendor/hyperframes/UPSTREAM.json +1 -0
- package/vendor/hyperframes/dist/chunk-K2FTXJYH.js +16 -2
- package/vendor/hyperframes/dist/chunk-TVCRUG2S.js +1 -1
- package/vendor/hyperframes/dist/render-LNO62LEL.js +1 -1
- package/src/native/faces.swift +0 -60
package/NOTICE
CHANGED
|
@@ -20,9 +20,11 @@ this package)
|
|
|
20
20
|
FFmpeg 7.1.2 (ffmpeg, ffprobe) — https://ffmpeg.org
|
|
21
21
|
Licensed under the GNU General Public License, version 2 or later. Its full text
|
|
22
22
|
is in licenses/ffmpeg-GPL-2.0.txt. Built with --enable-gpl --disable-nonfree and
|
|
23
|
-
without libx264 (it encodes with Apple's VideoToolbox
|
|
24
|
-
scripts/build-ffmpeg-macos.sh
|
|
25
|
-
|
|
23
|
+
without libx264 (it encodes with Apple's VideoToolbox on macOS and Microsoft
|
|
24
|
+
Media Foundation on Windows), using scripts/build-ffmpeg-macos.sh and
|
|
25
|
+
scripts/build-ffmpeg-windows.sh (cross-compiled with MinGW-w64; it statically
|
|
26
|
+
includes zlib, under the zlib licence) in this repository, which record the
|
|
27
|
+
exact configuration. The corresponding source code is available at
|
|
26
28
|
https://media.ignix.social/editor/bin/2/source/ffmpeg-7.1.2.tar.xz
|
|
27
29
|
and from https://ffmpeg.org/releases/ffmpeg-7.1.2.tar.xz.
|
|
28
30
|
FFmpeg is a separate program: Ignix Editor runs it as its own process and
|
|
@@ -56,3 +58,35 @@ whisper.cpp v1.7.6 (whisper-cli) — https://github.com/ggml-org/whisper.cpp
|
|
|
56
58
|
|
|
57
59
|
Whisper models (ggml-*.bin) are downloaded from
|
|
58
60
|
https://huggingface.co/ggerganov/whisper.cpp under the MIT license.
|
|
61
|
+
|
|
62
|
+
Analysis runtime and models (downloaded the first time `ignix-editor serve`, `faces`
|
|
63
|
+
or another analysis command needs them, from https://media.ignix.social/editor/analysis,
|
|
64
|
+
each checked against its SHA-256; listed in src/binaries.json):
|
|
65
|
+
|
|
66
|
+
ONNX Runtime 1.23.2 (onnxruntime-node and onnxruntime-common, from npm) —
|
|
67
|
+
https://github.com/microsoft/onnxruntime — MIT License, Copyright (c) Microsoft
|
|
68
|
+
Corporation. Repackaged per platform without changes by
|
|
69
|
+
scripts/build-analysis-runtime.mjs; each package carries its LICENSE and
|
|
70
|
+
ThirdPartyNotices.txt (licenses/onnxruntime-MIT.txt here).
|
|
71
|
+
|
|
72
|
+
YOLOX-S (person detector, ONNX export from OpenCV Zoo,
|
|
73
|
+
opencv/object_detection_yolox@78c368f) — Apache License 2.0, Copyright (c)
|
|
74
|
+
Megvii Inc.
|
|
75
|
+
|
|
76
|
+
YuNet (face detector, face_detection_yunet_2023mar.onnx from OpenCV Zoo,
|
|
77
|
+
opencv/face_detection_yunet@3cc26e7) — MIT License, Copyright (c) 2020 Shiqi Yu
|
|
78
|
+
(licenses/yunet-MIT.txt).
|
|
79
|
+
|
|
80
|
+
PP-OCRv3 text detector (ONNX from OpenCV Zoo, opencv/text_detection_ppocr@c299e81)
|
|
81
|
+
— Apache License 2.0, Copyright (c) PaddlePaddle Authors.
|
|
82
|
+
|
|
83
|
+
CLIP ViT-B/32 (openai/clip-vit-base-patch32; ONNX export and tokenizer files from
|
|
84
|
+
Xenova/clip-vit-base-patch32@d15189d) — MIT License, Copyright (c) 2021 OpenAI.
|
|
85
|
+
The vision model is stored in two parts that Ignix Editor joins and checks.
|
|
86
|
+
|
|
87
|
+
Inter (font, from @fontsource/inter) — SIL Open Font License 1.1, Copyright (c) 2016
|
|
88
|
+
The Inter Project Authors. Copied into the projects Ignix Creator's templates render
|
|
89
|
+
(src/creator), unmodified.
|
|
90
|
+
|
|
91
|
+
Ignix Creator's composition templates (src/creator) were ported from Ignix Creator's own
|
|
92
|
+
code (Copyright (c) 2026 Ignix); no Remotion code is included.
|
package/README.md
CHANGED
|
@@ -8,15 +8,16 @@ npx @ignix-social/editor init my-reel # creates ~/ignix-videos/my-reel
|
|
|
8
8
|
npx @ignix-social/editor render ~/ignix-videos/my-reel
|
|
9
9
|
```
|
|
10
10
|
|
|
11
|
-
Commands: `doctor`, `init`, `check`, `snapshot`, `render`, `preview`, `transcribe`, `ffmpeg`, `ffprobe`, `skills add`, `help`.
|
|
11
|
+
Commands: `doctor`, `init`, `check`, `snapshot`, `render`, `preview`, `transcribe`, `faces`, `ffmpeg`, `ffprobe`, `skills add`, `fetch`, `serve`, `creator-job`, `help`.
|
|
12
12
|
|
|
13
13
|
- Everything runs locally; your footage never leaves your machine.
|
|
14
14
|
- ffmpeg (GPL build, without libx264) and whisper.cpp are downloaded once per machine into `~/.ignix-editor`, checked against their SHA-256. Chrome for rendering is downloaded once from Google's official channel.
|
|
15
15
|
- No telemetry and no automatic updates: the version you install is the version that runs.
|
|
16
16
|
- The final audio is normalised to −14 LUFS.
|
|
17
|
+
- Analysis (`faces`, and `serve`, the local service Ignix Creator talks to: shots, people, faces, burned-in text, voice clean-up, transcription) runs on ONNX Runtime with open models (YOLOX, YuNet, PP-OCR, CLIP), downloaded the first time they are needed and checked against their SHA-256.
|
|
17
18
|
- `ignix-editor skills add` installs the `ignix-editor` skill for Claude.
|
|
18
19
|
|
|
19
|
-
Requires Node.js 22 or newer. macOS
|
|
20
|
+
Requires Node.js 22 or newer. Runs on macOS (Intel and Apple Silicon) and Windows 10/11 (x64).
|
|
20
21
|
|
|
21
22
|
## License
|
|
22
23
|
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) Microsoft Corporation
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2020 Shiqi Yu <shiqi.yu@gmail.com>
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@ignix-social/editor",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.6",
|
|
4
4
|
"description": "Ignix Editor: render HTML video compositions (reels, edits, clips) on your own computer. Based on HyperFrames.",
|
|
5
5
|
"license": "Apache-2.0",
|
|
6
6
|
"type": "module",
|
|
@@ -27,6 +27,7 @@
|
|
|
27
27
|
"test": "node --test test/*.test.mjs"
|
|
28
28
|
},
|
|
29
29
|
"dependencies": {
|
|
30
|
+
"@fontsource/inter": "^5.3.0",
|
|
30
31
|
"@fontsource/montserrat": "^5.3.0",
|
|
31
32
|
"@hono/node-server": "^2.0.5",
|
|
32
33
|
"@puppeteer/browsers": "^3.2.2",
|
package/patches/index.mjs
CHANGED
|
@@ -41,6 +41,44 @@ export const patches = [
|
|
|
41
41
|
replacement: 'if (false) { // ignix-editor: sin comprobación ni instalación automática de versiones',
|
|
42
42
|
}),
|
|
43
43
|
},
|
|
44
|
+
{
|
|
45
|
+
id: 'mf-encoder',
|
|
46
|
+
// Windows sin libx264 (patentes) ni tarjeta gráfica con codificador: Media Foundation
|
|
47
|
+
// (h264_mf), el codificador del propio Windows, como VideoToolbox en el Mac. Va el
|
|
48
|
+
// ÚLTIMO de los de hardware: si hay NVIDIA, Intel o AMD, se prefieren.
|
|
49
|
+
apply: (dist) => {
|
|
50
|
+
replaceAnchor(dist, {
|
|
51
|
+
anchor: ' "qsv",\n "amf"\n];',
|
|
52
|
+
replacement: ' "qsv",\n "amf",\n "mf" // ignix-editor: Media Foundation (Windows)\n];',
|
|
53
|
+
});
|
|
54
|
+
replaceAnchor(dist, {
|
|
55
|
+
anchor: ' amf: "h264_amf"\n};',
|
|
56
|
+
replacement: ' amf: "h264_amf",\n mf: "h264_mf"\n};',
|
|
57
|
+
});
|
|
58
|
+
replaceAnchor(dist, {
|
|
59
|
+
anchor: ' case "amf":\n return codec === "h264" ? "h264_amf" : "hevc_amf";\n',
|
|
60
|
+
replacement: ' case "amf":\n return codec === "h264" ? "h264_amf" : "hevc_amf";\n case "mf":\n return codec === "h264" ? "h264_mf" : "hevc_mf";\n',
|
|
61
|
+
});
|
|
62
|
+
// Calidad por bitrate, con la misma cuenta que VideoToolbox (sin -allow_sw, que es solo de Apple).
|
|
63
|
+
replaceAnchor(dist, {
|
|
64
|
+
anchor: ' case "amf":\n if (bitrate) args.push("-b:v", bitrate);\n else args.push("-rc", "cqp", "-qp_i", String(quality), "-qp_p", String(quality));\n break;\n',
|
|
65
|
+
replacement:
|
|
66
|
+
' case "amf":\n if (bitrate) args.push("-b:v", bitrate);\n else args.push("-rc", "cqp", "-qp_i", String(quality), "-qp_p", String(quality));\n break;\n' +
|
|
67
|
+
' case "mf": {\n const vt = buildVideoToolboxRateControlArgs({ bitrate, width: options.width, height: options.height, fps: fpsToNumber(fps), quality });\n args.push("-b:v", vt[1], "-rate_control", "u_vbr");\n break;\n }\n',
|
|
68
|
+
expect: 2,
|
|
69
|
+
});
|
|
70
|
+
// El aviso decía VideoToolbox también en Windows.
|
|
71
|
+
replaceAnchor(dist, {
|
|
72
|
+
anchor: 'FFmpeg does not include libx264; falling back to VideoToolbox H.264 encoding.',
|
|
73
|
+
replacement: "FFmpeg does not include libx264; using the system's hardware H.264 encoder (VideoToolbox on macOS, Media Foundation on Windows).",
|
|
74
|
+
});
|
|
75
|
+
// Un ffmpeg sin libx264 pero con h264_mf también sirve (antes: solo VideoToolbox).
|
|
76
|
+
replaceAnchor(dist, {
|
|
77
|
+
anchor: 'if (/\\bh264_videotoolbox\\b/.test(ffmpegEncodersOutput)) return "gpu";',
|
|
78
|
+
replacement: 'if (/\\bh264_(videotoolbox|mf)\\b/.test(ffmpegEncodersOutput)) return "gpu";',
|
|
79
|
+
});
|
|
80
|
+
},
|
|
81
|
+
},
|
|
44
82
|
{
|
|
45
83
|
id: 'no-desktop-promo',
|
|
46
84
|
// Sin publicidad de la app de escritorio de HeyGen al terminar un render.
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
// Monta los paquetes de onnxruntime por sistema para el análisis (ADR-0023 E1) a partir de los
|
|
2
|
+
// paquetes oficiales de npm, sin compilar nada: un zip por plataforma con solo SU binario
|
|
3
|
+
// (el paquete completo trae los seis sistemas, ~250 MB).
|
|
4
|
+
//
|
|
5
|
+
// node scripts/build-analysis-runtime.mjs <dir-con-onnxruntime-node-y-common> <salida>
|
|
6
|
+
//
|
|
7
|
+
// <dir> contiene `onnxruntime-node-<v>.tgz` y `onnxruntime-common-<v>.tgz` (npm pack), además de
|
|
8
|
+
// LICENSE y ThirdPartyNotices.txt del repositorio de ONNX Runtime en esa versión (MIT).
|
|
9
|
+
// Imprime tamaño y sha256 de cada zip para src/binaries.json → analysis.runtime.
|
|
10
|
+
import { spawnSync } from 'node:child_process';
|
|
11
|
+
import { createHash } from 'node:crypto';
|
|
12
|
+
import { cpSync, mkdirSync, mkdtempSync, readFileSync, rmSync, statSync } from 'node:fs';
|
|
13
|
+
import { tmpdir } from 'node:os';
|
|
14
|
+
import { join } from 'node:path';
|
|
15
|
+
import AdmZip from 'adm-zip';
|
|
16
|
+
|
|
17
|
+
const [src, out] = process.argv.slice(2);
|
|
18
|
+
if (!src || !out) throw new Error('usage: build-analysis-runtime.mjs <src> <out>');
|
|
19
|
+
const VERSION = '1.23.2';
|
|
20
|
+
const PLATFORMS = { 'darwin-x64': ['darwin', 'x64'], 'darwin-arm64': ['darwin', 'arm64'], 'win32-x64': ['win32', 'x64'] };
|
|
21
|
+
|
|
22
|
+
const tmp = mkdtempSync(join(tmpdir(), 'ort-runtime-'));
|
|
23
|
+
const unpack = (tgz, dest) => {
|
|
24
|
+
mkdirSync(dest, { recursive: true });
|
|
25
|
+
const r = spawnSync('tar', ['xzf', join(src, tgz), '-C', dest, '--strip-components=1']);
|
|
26
|
+
if (r.status !== 0) throw new Error(`tar ${tgz}: ${r.stderr}`);
|
|
27
|
+
};
|
|
28
|
+
unpack(`onnxruntime-node-${VERSION}.tgz`, join(tmp, 'node'));
|
|
29
|
+
unpack(`onnxruntime-common-${VERSION}.tgz`, join(tmp, 'common'));
|
|
30
|
+
mkdirSync(out, { recursive: true });
|
|
31
|
+
for (const [name, [os, arch]] of Object.entries(PLATFORMS)) {
|
|
32
|
+
const stage = join(tmp, name, 'onnxruntime-node');
|
|
33
|
+
for (const f of ['package.json', 'dist']) cpSync(join(tmp, 'node', f), join(stage, f), { recursive: true });
|
|
34
|
+
cpSync(join(tmp, 'node', 'bin', 'napi-v6', os, arch), join(stage, 'bin', 'napi-v6', os, arch), { recursive: true });
|
|
35
|
+
cpSync(join(tmp, 'common'), join(stage, 'node_modules', 'onnxruntime-common'), { recursive: true });
|
|
36
|
+
cpSync(join(src, 'LICENSE'), join(stage, 'LICENSE'));
|
|
37
|
+
cpSync(join(src, 'ThirdPartyNotices.txt'), join(stage, 'ThirdPartyNotices.txt'));
|
|
38
|
+
const zip = new AdmZip();
|
|
39
|
+
zip.addLocalFolder(join(tmp, name));
|
|
40
|
+
const file = join(out, `onnxruntime-${VERSION}-${name}.zip`);
|
|
41
|
+
zip.writeZip(file);
|
|
42
|
+
const sha = createHash('sha256').update(readFileSync(file)).digest('hex');
|
|
43
|
+
console.log(JSON.stringify({ platform: name, file: `onnxruntime-${VERSION}-${name}.zip`, size: statSync(file).size, sha256: sha }));
|
|
44
|
+
}
|
|
45
|
+
rmSync(tmp, { recursive: true, force: true });
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
#!/bin/sh
|
|
2
|
+
# Compila ffmpeg.exe + ffprobe.exe para Windows x64 desde Linux (MinGW-w64), con las mismas
|
|
3
|
+
# reglas que en el Mac: GPL sin libx264 ni non-free y sin buscar librerías por su cuenta.
|
|
4
|
+
# Codifica con Media Foundation (h264_mf), el codificador del propio Windows; si el PC tiene
|
|
5
|
+
# NVIDIA/Intel/AMD el motor no los usa con este binario (no se compilan sus SDK).
|
|
6
|
+
# Requiere: mingw-w64, nasm, make, curl (en Ubuntu: apt-get install mingw-w64 nasm make curl).
|
|
7
|
+
# Uso: scripts/build-ffmpeg-windows.sh [dir-de-trabajo] → <dir>/out/{ffmpeg.exe,ffprobe.exe}
|
|
8
|
+
set -eu
|
|
9
|
+
FFMPEG_VERSION=7.1.2
|
|
10
|
+
ZLIB_VERSION=1.3.1
|
|
11
|
+
WORK=${1:-"$HOME/ignix-editor-build/ffmpeg-windows-x64"}
|
|
12
|
+
JOBS=$(nproc 2>/dev/null || sysctl -n hw.ncpu)
|
|
13
|
+
HOST=x86_64-w64-mingw32
|
|
14
|
+
mkdir -p "$WORK" && cd "$WORK"
|
|
15
|
+
PREFIX="$WORK/deps"
|
|
16
|
+
|
|
17
|
+
# zlib estática (PNG la necesita: el motor extrae y compone fotogramas en PNG)
|
|
18
|
+
if [ ! -f "$PREFIX/lib/libz.a" ]; then
|
|
19
|
+
curl -sfL "https://zlib.net/fossils/zlib-$ZLIB_VERSION.tar.gz" -o zlib.tar.gz
|
|
20
|
+
tar xf zlib.tar.gz && cd "zlib-$ZLIB_VERSION"
|
|
21
|
+
make -f win32/Makefile.gcc PREFIX="$HOST-" -j"$JOBS" libz.a >/dev/null
|
|
22
|
+
mkdir -p "$PREFIX/lib" "$PREFIX/include"
|
|
23
|
+
cp libz.a "$PREFIX/lib/" && cp zlib.h zconf.h "$PREFIX/include/"
|
|
24
|
+
cd "$WORK"
|
|
25
|
+
fi
|
|
26
|
+
|
|
27
|
+
[ -d "ffmpeg-$FFMPEG_VERSION" ] || { curl -sfL "https://ffmpeg.org/releases/ffmpeg-$FFMPEG_VERSION.tar.xz" -o ffmpeg.tar.xz && tar xf ffmpeg.tar.xz; }
|
|
28
|
+
cd "ffmpeg-$FFMPEG_VERSION"
|
|
29
|
+
./configure \
|
|
30
|
+
--prefix="$WORK/out-prefix" \
|
|
31
|
+
--target-os=mingw32 --arch=x86_64 --cross-prefix="$HOST-" \
|
|
32
|
+
--enable-gpl --disable-nonfree --disable-version3 \
|
|
33
|
+
--disable-autodetect \
|
|
34
|
+
--disable-debug --disable-doc --disable-ffplay \
|
|
35
|
+
--disable-indevs --enable-indev=lavfi --disable-outdevs \
|
|
36
|
+
--disable-shared --enable-static \
|
|
37
|
+
--enable-w32threads --enable-mediafoundation \
|
|
38
|
+
--enable-zlib --disable-bzlib --disable-iconv \
|
|
39
|
+
--pkg-config=false \
|
|
40
|
+
--extra-cflags="-I$PREFIX/include" --extra-ldflags="-L$PREFIX/lib -static" >/dev/null
|
|
41
|
+
make -j"$JOBS" >/dev/null
|
|
42
|
+
mkdir -p "$WORK/out" && cp ffmpeg.exe ffprobe.exe "$WORK/out/"
|
|
43
|
+
"$HOST-objdump" -p "$WORK/out/ffmpeg.exe" | grep "DLL Name" | sort -u
|
|
44
|
+
echo "LISTO: $WORK/out"
|
|
@@ -0,0 +1,142 @@
|
|
|
1
|
+
// CLIP ViT-B/32 (MIT; el mismo modelo que usaba Creator) en onnxruntime: imágenes y frases en
|
|
2
|
+
// el mismo espacio, para elegir el plano que enseña lo que dice la voz. Coseno imagen↔texto:
|
|
3
|
+
// ≈0,18 sin relación, ≈0,3 cuando la imagen lo muestra.
|
|
4
|
+
import { readFileSync } from 'node:fs';
|
|
5
|
+
import { resizeBicubic, toCHW } from './image.mjs';
|
|
6
|
+
import { ensureModel, loadOrt, session } from './runtime.mjs';
|
|
7
|
+
|
|
8
|
+
const MEAN = [0.48145466, 0.4578275, 0.40821073];
|
|
9
|
+
const STD = [0.26862954, 0.26130258, 0.27577711];
|
|
10
|
+
const SOT = 49406;
|
|
11
|
+
const EOT = 49407;
|
|
12
|
+
const MAX_TOKENS = 77;
|
|
13
|
+
|
|
14
|
+
// ── Tokenizador BPE de CLIP (bytes → unicode, fusiones por rango, «</w>» al final de palabra)
|
|
15
|
+
let tok = null;
|
|
16
|
+
async function tokenizer(log) {
|
|
17
|
+
if (tok) return tok;
|
|
18
|
+
const vocab = JSON.parse(readFileSync(await ensureModel('clip_vocab', log), 'utf8'));
|
|
19
|
+
const lines = readFileSync(await ensureModel('clip_merges', log), 'utf8').split('\n');
|
|
20
|
+
const ranks = new Map();
|
|
21
|
+
lines.slice(1).forEach((l, i) => {
|
|
22
|
+
if (l.trim()) ranks.set(l.trim(), i);
|
|
23
|
+
});
|
|
24
|
+
const byteMap = bytesToUnicode();
|
|
25
|
+
const cache = new Map();
|
|
26
|
+
const bpe = (token) => {
|
|
27
|
+
if (cache.has(token)) return cache.get(token);
|
|
28
|
+
let word = [...token.slice(0, -1), token.slice(-1) + '</w>'];
|
|
29
|
+
if (token.length === 1) word = [token + '</w>'];
|
|
30
|
+
for (;;) {
|
|
31
|
+
let best = null;
|
|
32
|
+
let bestRank = Infinity;
|
|
33
|
+
for (let i = 0; i < word.length - 1; i++) {
|
|
34
|
+
const r = ranks.get(`${word[i]} ${word[i + 1]}`);
|
|
35
|
+
if (r !== undefined && r < bestRank) {
|
|
36
|
+
bestRank = r;
|
|
37
|
+
best = i;
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
if (best === null) break;
|
|
41
|
+
const a = word[best];
|
|
42
|
+
const b = word[best + 1];
|
|
43
|
+
const merged = [];
|
|
44
|
+
for (let i = 0; i < word.length; i++) {
|
|
45
|
+
if (i < word.length - 1 && word[i] === a && word[i + 1] === b) {
|
|
46
|
+
merged.push(a + b);
|
|
47
|
+
i++;
|
|
48
|
+
} else merged.push(word[i]);
|
|
49
|
+
}
|
|
50
|
+
word = merged;
|
|
51
|
+
}
|
|
52
|
+
cache.set(token, word);
|
|
53
|
+
return word;
|
|
54
|
+
};
|
|
55
|
+
const PAT = /<\|startoftext\|>|<\|endoftext\|>|'s|'t|'re|'ve|'m|'ll|'d|[\p{L}]+|[\p{N}]|[^\s\p{L}\p{N}]+/gu;
|
|
56
|
+
tok = (text) => {
|
|
57
|
+
const clean = text.normalize('NFC').replace(/\s+/g, ' ').trim().toLowerCase();
|
|
58
|
+
const ids = [SOT];
|
|
59
|
+
for (const m of clean.matchAll(PAT)) {
|
|
60
|
+
const mapped = [...Buffer.from(m[0], 'utf8')].map((b) => byteMap[b]).join('');
|
|
61
|
+
for (const piece of bpe(mapped)) {
|
|
62
|
+
const id = vocab[piece];
|
|
63
|
+
if (id !== undefined) ids.push(id);
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
return [...ids.slice(0, MAX_TOKENS - 1), EOT];
|
|
67
|
+
};
|
|
68
|
+
return tok;
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
function bytesToUnicode() {
|
|
72
|
+
const bs = [];
|
|
73
|
+
for (let b = 33; b <= 126; b++) bs.push(b);
|
|
74
|
+
for (let b = 161; b <= 172; b++) bs.push(b);
|
|
75
|
+
for (let b = 174; b <= 255; b++) bs.push(b);
|
|
76
|
+
const cs = [...bs];
|
|
77
|
+
let n = 0;
|
|
78
|
+
for (let b = 0; b < 256; b++) {
|
|
79
|
+
if (!bs.includes(b)) {
|
|
80
|
+
bs.push(b);
|
|
81
|
+
cs.push(256 + n);
|
|
82
|
+
n++;
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
const map = {};
|
|
86
|
+
bs.forEach((b, i) => (map[b] = String.fromCodePoint(cs[i])));
|
|
87
|
+
return map;
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
const unit = (v) => {
|
|
91
|
+
let s = 0;
|
|
92
|
+
for (const x of v) s += x * x;
|
|
93
|
+
const n = Math.sqrt(s) || 1;
|
|
94
|
+
return Float32Array.from(v, (x) => x / n);
|
|
95
|
+
};
|
|
96
|
+
|
|
97
|
+
/** Vectores unitarios (512) de cada frase. */
|
|
98
|
+
export async function embedTexts(texts, log) {
|
|
99
|
+
const t = await tokenizer(log);
|
|
100
|
+
const s = await session('clip_text', log);
|
|
101
|
+
const { Tensor } = await loadOrt(log);
|
|
102
|
+
const out = [];
|
|
103
|
+
for (const text of texts) {
|
|
104
|
+
const ids = t(text);
|
|
105
|
+
const res = await s.run({ input_ids: new Tensor('int64', BigInt64Array.from(ids.map(BigInt)), [1, ids.length]) });
|
|
106
|
+
out.push(unit(res.text_embeds.data));
|
|
107
|
+
}
|
|
108
|
+
return out;
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
/** Vector unitario de una imagen (RGB intercalado w×h), preparada como CLIP. */
|
|
112
|
+
export async function embedImage(frame, log) {
|
|
113
|
+
const s = await session('clip_vision', log);
|
|
114
|
+
const { Tensor } = await loadOrt(log);
|
|
115
|
+
// Lado corto a 224 (bicúbico, como PIL) y recorte central de 224×224.
|
|
116
|
+
const k = 224 / Math.min(frame.w, frame.h);
|
|
117
|
+
const W = Math.max(224, Math.round(frame.w * k));
|
|
118
|
+
const H = Math.max(224, Math.round(frame.h * k));
|
|
119
|
+
const big = resizeBicubic(frame.rgb, frame.w, frame.h, W, H);
|
|
120
|
+
const x0 = Math.floor((W - 224) / 2);
|
|
121
|
+
const y0 = Math.floor((H - 224) / 2);
|
|
122
|
+
const crop = new Uint8Array(224 * 224 * 3);
|
|
123
|
+
for (let y = 0; y < 224; y++) crop.set(big.subarray(((y0 + y) * W + x0) * 3, ((y0 + y) * W + x0 + 224) * 3), y * 224 * 3);
|
|
124
|
+
const input = new Tensor('float32', toCHW(crop, 224, 224, { scale: 1 / 255, mean: MEAN, std: STD }), [1, 3, 224, 224]);
|
|
125
|
+
const res = await s.run({ pixel_values: input });
|
|
126
|
+
return unit(res.image_embeds.data);
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
export const dot = (a, b) => {
|
|
130
|
+
let s = 0;
|
|
131
|
+
for (let i = 0; i < a.length; i++) s += a[i] * b[i];
|
|
132
|
+
return s;
|
|
133
|
+
};
|
|
134
|
+
|
|
135
|
+
/** Media de varios vectores, unitaria (un plano = sus fotogramas juntos). */
|
|
136
|
+
export function pool(vecs) {
|
|
137
|
+
const out = new Float32Array(vecs[0].length);
|
|
138
|
+
for (const v of vecs) for (let i = 0; i < v.length; i++) out[i] += v[i];
|
|
139
|
+
return unit(out);
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
export { tokenizer as clipTokenizer };
|
|
@@ -0,0 +1,185 @@
|
|
|
1
|
+
// Detectores sobre onnxruntime, iguales en Mac y en Windows:
|
|
2
|
+
// · personas por el CUERPO (YOLOX-S, Apache-2.0): alguien que mira abajo o de espaldas sigue
|
|
3
|
+
// siendo alguien. El mismo detector que usa el trabajador del recast en la nube, para que los
|
|
4
|
+
// dos lados cuenten las mismas personas.
|
|
5
|
+
// · caras (YuNet, MIT) con sus cinco puntos (ojos, nariz, boca): sirven para encuadrar y para
|
|
6
|
+
// saber si una cara se ve de FRENTE (el parecido de un recast depende de eso).
|
|
7
|
+
// · texto quemado (detector de PP-OCRv3, Apache-2.0): letras sobre la persona salen
|
|
8
|
+
// garabateadas en un recast.
|
|
9
|
+
// Todas las cajas salen normalizadas (0..1) respecto al fotograma.
|
|
10
|
+
import { letterbox, nms, resize, toCHW } from './image.mjs';
|
|
11
|
+
import { session } from './runtime.mjs';
|
|
12
|
+
|
|
13
|
+
const S = 640;
|
|
14
|
+
|
|
15
|
+
// ── YOLOX
|
|
16
|
+
const ANCHORS = (() => {
|
|
17
|
+
const g = [];
|
|
18
|
+
for (const st of [8, 16, 32]) {
|
|
19
|
+
const n = S / st;
|
|
20
|
+
for (let y = 0; y < n; y++) for (let x = 0; x < n; x++) g.push([x, y, st]);
|
|
21
|
+
}
|
|
22
|
+
return g;
|
|
23
|
+
})();
|
|
24
|
+
export const PERSON_MIN_SCORE = 0.4;
|
|
25
|
+
|
|
26
|
+
/** Salida cruda de YOLOX (8400 × 85) → cajas de persona en píxeles del fotograma. */
|
|
27
|
+
export function decodeYolox(out, ratio, width, height, minScore = PERSON_MIN_SCORE) {
|
|
28
|
+
const found = [];
|
|
29
|
+
for (let i = 0; i < ANCHORS.length; i++) {
|
|
30
|
+
const o = i * 85;
|
|
31
|
+
const score = out[o + 4] * out[o + 5]; // un objeto, y ese objeto es una persona
|
|
32
|
+
if (score < minScore) continue;
|
|
33
|
+
const [gx, gy, st] = ANCHORS[i];
|
|
34
|
+
const cx = (out[o] + gx) * st;
|
|
35
|
+
const cy = (out[o + 1] + gy) * st;
|
|
36
|
+
const w = Math.exp(out[o + 2]) * st;
|
|
37
|
+
const h = Math.exp(out[o + 3]) * st;
|
|
38
|
+
const x0 = Math.max(0, (cx - w / 2) / ratio);
|
|
39
|
+
const y0 = Math.max(0, (cy - h / 2) / ratio);
|
|
40
|
+
const x1 = Math.min(width, (cx + w / 2) / ratio);
|
|
41
|
+
const y1 = Math.min(height, (cy + h / 2) / ratio);
|
|
42
|
+
if (x1 > x0 && y1 > y0) found.push({ x: x0, y: y0, w: x1 - x0, h: y1 - y0, score });
|
|
43
|
+
}
|
|
44
|
+
return nms(found, 0.5);
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
export async function people(frame, log) {
|
|
48
|
+
const s = await session('yolox', log);
|
|
49
|
+
const { rgb, r } = letterbox(frame.rgb, frame.w, frame.h, S, 114);
|
|
50
|
+
const input = new (await tensorClass(log))('float32', toCHW(rgb, S, S), [1, 3, S, S]);
|
|
51
|
+
const res = await s.run({ [s.inputNames[0]]: input });
|
|
52
|
+
const out = res[s.outputNames[0]].data;
|
|
53
|
+
return decodeYolox(out, r, frame.w, frame.h).map((b) => norm(b, frame));
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
// ── YuNet
|
|
57
|
+
export const FACE_MIN_SCORE = 0.7;
|
|
58
|
+
|
|
59
|
+
/** Salidas de YuNet → caras {x,y,w,h,score,kps:[[x,y]×5]} en píxeles del fotograma. */
|
|
60
|
+
export function decodeYunet(out, ratio, width, height, minScore = FACE_MIN_SCORE) {
|
|
61
|
+
const found = [];
|
|
62
|
+
for (const st of [8, 16, 32]) {
|
|
63
|
+
const cls = out[`cls_${st}`];
|
|
64
|
+
const obj = out[`obj_${st}`];
|
|
65
|
+
const box = out[`bbox_${st}`];
|
|
66
|
+
const kps = out[`kps_${st}`];
|
|
67
|
+
const cols = S / st;
|
|
68
|
+
for (let i = 0; i < cls.length; i++) {
|
|
69
|
+
const score = Math.sqrt(Math.min(1, Math.max(0, cls[i])) * Math.min(1, Math.max(0, obj[i])));
|
|
70
|
+
if (score < minScore) continue;
|
|
71
|
+
const c = i % cols;
|
|
72
|
+
const r = Math.floor(i / cols);
|
|
73
|
+
const cx = (c + box[i * 4]) * st;
|
|
74
|
+
const cy = (r + box[i * 4 + 1]) * st;
|
|
75
|
+
const w = Math.exp(box[i * 4 + 2]) * st;
|
|
76
|
+
const h = Math.exp(box[i * 4 + 3]) * st;
|
|
77
|
+
const pts = [];
|
|
78
|
+
for (let k = 0; k < 5; k++) pts.push([((kps[i * 10 + 2 * k] + c) * st) / ratio, ((kps[i * 10 + 2 * k + 1] + r) * st) / ratio]);
|
|
79
|
+
const x0 = Math.max(0, (cx - w / 2) / ratio);
|
|
80
|
+
const y0 = Math.max(0, (cy - h / 2) / ratio);
|
|
81
|
+
const x1 = Math.min(width, (cx + w / 2) / ratio);
|
|
82
|
+
const y1 = Math.min(height, (cy + h / 2) / ratio);
|
|
83
|
+
if (x1 > x0 && y1 > y0) found.push({ x: x0, y: y0, w: x1 - x0, h: y1 - y0, score, kps: pts });
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
return nms(found, 0.3);
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
/**
|
|
90
|
+
* ¿Se ve la cara de frente? Con los cinco puntos de YuNet: la nariz cae entre los dos ojos y a
|
|
91
|
+
* la misma distancia de cada uno, y los ojos no están demasiado juntos para el tamaño de la cara
|
|
92
|
+
* (de perfil se juntan). Umbrales medidos con los clips del recast (ADR-0023 E1).
|
|
93
|
+
*/
|
|
94
|
+
export function isFrontal(face, { maxSkew = FRONTAL_MAX_SKEW, minEyes = FRONTAL_MIN_EYES } = {}) {
|
|
95
|
+
const [re, le, nose] = face.kps;
|
|
96
|
+
const eyes = Math.hypot(le[0] - re[0], le[1] - re[1]);
|
|
97
|
+
if (eyes < minEyes * face.w) return false;
|
|
98
|
+
const dl = Math.hypot(nose[0] - le[0], nose[1] - le[1]);
|
|
99
|
+
const dr = Math.hypot(nose[0] - re[0], nose[1] - re[1]);
|
|
100
|
+
return Math.abs(dl - dr) / (dl + dr) <= maxSkew;
|
|
101
|
+
}
|
|
102
|
+
// Medido (2026-10-06) con seis clips del recast: buenos 83–94 % de muestras de frente,
|
|
103
|
+
// malos 14–37 %; con los umbrales de Creator (≥50 %, ≥110 px) separan igual que el Haar de antes.
|
|
104
|
+
export const FRONTAL_MAX_SKEW = 0.2;
|
|
105
|
+
export const FRONTAL_MIN_EYES = 0.3;
|
|
106
|
+
|
|
107
|
+
export async function faces(frame, log) {
|
|
108
|
+
const s = await session('yunet', log);
|
|
109
|
+
// YuNet espera BGR en 0..255 (como lo prepara OpenCV), en un cuadrado de 640.
|
|
110
|
+
const { rgb, r } = letterbox(frame.rgb, frame.w, frame.h, S, 0);
|
|
111
|
+
const input = new (await tensorClass(log))('float32', toCHW(rgb, S, S, { bgr: true }), [1, 3, S, S]);
|
|
112
|
+
const res = await s.run({ [s.inputNames[0]]: input });
|
|
113
|
+
const out = Object.fromEntries(s.outputNames.map((n) => [n, res[n].data]));
|
|
114
|
+
return decodeYunet(out, r, frame.w, frame.h).map((f) => ({
|
|
115
|
+
...norm(f, frame),
|
|
116
|
+
frontal: isFrontal(f),
|
|
117
|
+
kps: f.kps.map(([x, y]) => [x / frame.w, y / frame.h]),
|
|
118
|
+
// Tamaño de la cara en píxeles del ORIGINAL, no del fotograma de trabajo.
|
|
119
|
+
px: f.h * (frame.scale ?? 1),
|
|
120
|
+
}));
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
// ── PP-OCR (detector de texto DB)
|
|
124
|
+
const OCR_MEAN = [123.675, 116.28, 103.53];
|
|
125
|
+
const OCR_STD = [0.229, 0.224, 0.225];
|
|
126
|
+
|
|
127
|
+
/** Mapa de probabilidad del detector DB → cajas de texto en coordenadas del mapa. */
|
|
128
|
+
export function decodeDb(prob, W, H, { bin = 0.3, minScore = 0.5, unclip = 2.0, max = 200 } = {}) {
|
|
129
|
+
const seen = new Uint8Array(W * H);
|
|
130
|
+
const boxes = [];
|
|
131
|
+
const stack = [];
|
|
132
|
+
for (let start = 0; start < W * H && boxes.length < max; start++) {
|
|
133
|
+
if (seen[start] || prob[start] < bin) continue;
|
|
134
|
+
// Componente conexa (4 vecinos) de píxeles sobre el umbral.
|
|
135
|
+
let x0 = W, y0 = H, x1 = 0, y1 = 0, sum = 0, n = 0;
|
|
136
|
+
stack.push(start);
|
|
137
|
+
seen[start] = 1;
|
|
138
|
+
while (stack.length) {
|
|
139
|
+
const p = stack.pop();
|
|
140
|
+
const x = p % W;
|
|
141
|
+
const y = (p - x) / W;
|
|
142
|
+
sum += prob[p];
|
|
143
|
+
n += 1;
|
|
144
|
+
if (x < x0) x0 = x;
|
|
145
|
+
if (x > x1) x1 = x;
|
|
146
|
+
if (y < y0) y0 = y;
|
|
147
|
+
if (y > y1) y1 = y;
|
|
148
|
+
for (const q of [x > 0 ? p - 1 : -1, x < W - 1 ? p + 1 : -1, y > 0 ? p - W : -1, y < H - 1 ? p + W : -1]) {
|
|
149
|
+
if (q >= 0 && !seen[q] && prob[q] >= bin) {
|
|
150
|
+
seen[q] = 1;
|
|
151
|
+
stack.push(q);
|
|
152
|
+
}
|
|
153
|
+
}
|
|
154
|
+
}
|
|
155
|
+
const w = x1 - x0 + 1;
|
|
156
|
+
const h = y1 - y0 + 1;
|
|
157
|
+
if (Math.min(w, h) < 3 || sum / n < minScore) continue;
|
|
158
|
+
// El detector DB «encoge» el texto: se agranda la caja en área·ratio/perímetro.
|
|
159
|
+
const d = (w * h * unclip) / (2 * (w + h));
|
|
160
|
+
boxes.push({ x: Math.max(0, x0 - d), y: Math.max(0, y0 - d), w: Math.min(W, x1 + 1 + d) - Math.max(0, x0 - d), h: Math.min(H, y1 + 1 + d) - Math.max(0, y0 - d), score: sum / n });
|
|
161
|
+
}
|
|
162
|
+
return boxes;
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
export async function text(frame, log) {
|
|
166
|
+
const s = await session('ppocr', log);
|
|
167
|
+
const [W, H] = frame.h > frame.w ? [416, 736] : [736, 416];
|
|
168
|
+
const small = resize(frame.rgb, frame.w, frame.h, W, H);
|
|
169
|
+
// Como lo prepara OpenCV para este modelo: canales en orden BGR, medias y escalas por posición.
|
|
170
|
+
const input = new (await tensorClass(log))('float32', toCHW(small, W, H, { bgr: true, mean: OCR_MEAN, std: OCR_STD.map((v) => v * 255) }), [1, 3, H, W]);
|
|
171
|
+
const res = await s.run({ [s.inputNames[0]]: input });
|
|
172
|
+
const prob = res[s.outputNames[0]].data;
|
|
173
|
+
return decodeDb(prob, W, H)
|
|
174
|
+
.slice(0, 40)
|
|
175
|
+
.map((b) => ({ x: b.x / W, y: b.y / H, w: Math.min(1, b.w / W), h: Math.min(1, b.h / H) }));
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
// ── utilidades
|
|
179
|
+
const norm = (b, frame) => ({ x: b.x / frame.w, y: b.y / frame.h, w: b.w / frame.w, h: b.h / frame.h, score: +b.score.toFixed(3) });
|
|
180
|
+
|
|
181
|
+
let Tensor = null;
|
|
182
|
+
async function tensorClass(log) {
|
|
183
|
+
if (!Tensor) Tensor = (await import('./runtime.mjs').then((m) => m.loadOrt(log))).Tensor;
|
|
184
|
+
return Tensor;
|
|
185
|
+
}
|