fablecut 1.13.0 → 1.15.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CLAUDE.md +802 -16
- package/README.md +71 -7
- package/analyze.js +161 -9
- package/app.js +3905 -258
- package/comp.js +195 -0
- package/delivery.js +779 -0
- package/edit-ops.js +50 -18
- package/export-jobs.js +9 -5
- package/frame-cache.js +49 -0
- package/frame-stats.js +267 -0
- package/index.html +282 -72
- package/library/README.md +2 -0
- package/library/elements/logo-placeholder.svg +12 -0
- package/mcp-server.js +324 -656
- package/media-lib.js +287 -0
- package/media-probe.js +88 -0
- package/motion.js +803 -0
- package/package.json +1 -1
- package/paths.js +33 -3
- package/project-ops.js +1290 -0
- package/proxy.js +180 -0
- package/server.js +517 -15
- package/style.css +534 -20
- package/templates.js +576 -0
package/edit-ops.js
CHANGED
|
@@ -20,10 +20,11 @@
|
|
|
20
20
|
lockedTracks and untargetedTracks) for headless callers. */
|
|
21
21
|
(function (root, factory) {
|
|
22
22
|
const node = typeof module === "object" && module.exports;
|
|
23
|
-
const api = factory(node ? require("./mask") : root.FableCutMask, node ? require("./tracker") : root.FableCutTracker
|
|
23
|
+
const api = factory(node ? require("./mask") : root.FableCutMask, node ? require("./tracker") : root.FableCutTracker,
|
|
24
|
+
node ? require("./motion") : root.FableCutMotion);
|
|
24
25
|
if (node) module.exports = api;
|
|
25
26
|
else root.FableCutEdit = api;
|
|
26
|
-
})(typeof self !== "undefined" ? self : this, function (MaskLib, TrackLib) {
|
|
27
|
+
})(typeof self !== "undefined" ? self : this, function (MaskLib, TrackLib, MotionLib) {
|
|
27
28
|
"use strict";
|
|
28
29
|
|
|
29
30
|
const MIN_DUR = 0.05;
|
|
@@ -41,8 +42,10 @@
|
|
|
41
42
|
/* ── Pure clip helpers ── */
|
|
42
43
|
const clamp = (v, a, b) => Math.min(b, Math.max(a, v));
|
|
43
44
|
const clipEnd = (c) => c.start + c.duration;
|
|
44
|
-
const clipSpeed = (c) => clamp(+(c.props?.speed) || 1, 0.1, 8);
|
|
45
|
+
const clipSpeed = (c) => (c.kind === "comp" ? 1 : clamp(+(c.props?.speed) || 1, 0.1, 8));
|
|
45
46
|
const isMediaClip = (c) => c.kind === "video" || c.kind === "audio";
|
|
47
|
+
/** Clips with a source window (`in` into something with a length): media, and comp clips (into their comp). */
|
|
48
|
+
const hasSourceTime = (c) => isMediaClip(c) || c.kind === "comp";
|
|
46
49
|
const EASE = {
|
|
47
50
|
linear: (u) => u,
|
|
48
51
|
"ease-in": (u) => u * u,
|
|
@@ -65,7 +68,7 @@
|
|
|
65
68
|
return fallback;
|
|
66
69
|
}
|
|
67
70
|
function hasSpeedRamp(c) {
|
|
68
|
-
return Array.isArray(c.keyframes?.speed) && c.keyframes.speed.length > 0;
|
|
71
|
+
return c.kind !== "comp" && Array.isArray(c.keyframes?.speed) && c.keyframes.speed.length > 0; // comps run at 1×
|
|
69
72
|
}
|
|
70
73
|
/** Source time at timeline time t: in + ∫speed (trapezoid, 1/120 s steps). */
|
|
71
74
|
function mediaTimeAt(c, t) {
|
|
@@ -93,10 +96,11 @@
|
|
|
93
96
|
}
|
|
94
97
|
return Object.keys(out).length ? out : undefined;
|
|
95
98
|
}
|
|
96
|
-
/** A clip's head moved by `offset` s: its mask keys and track samples follow, like shiftKF does for keyframes. */
|
|
99
|
+
/** A clip's head moved by `offset` s: its mask keys, shape keys and track samples follow, like shiftKF does for keyframes. */
|
|
97
100
|
function shiftMasks(c, offset) {
|
|
98
101
|
if (!offset || !c.props) return;
|
|
99
|
-
const ms = c.props.masks, ts = c.props.tracks;
|
|
102
|
+
const ms = c.props.masks, ts = c.props.tracks, ss = c.props.shapes;
|
|
103
|
+
if (MotionLib && Array.isArray(ss)) c.props = { ...c.props, shapes: MotionLib.shiftKeys(ss, offset) };
|
|
100
104
|
if (MaskLib && Array.isArray(ms)) {
|
|
101
105
|
const next = MaskLib.shiftKeys(ms, offset);
|
|
102
106
|
if (next !== ms) c.props = { ...c.props, masks: next };
|
|
@@ -107,6 +111,27 @@
|
|
|
107
111
|
if (next.length) c.props.tracks = next; else delete c.props.tracks;
|
|
108
112
|
}
|
|
109
113
|
}
|
|
114
|
+
/** Clips are leaving the document: their children stay where they are. Each child is
|
|
115
|
+
* re-parented to the nearest ancestor that stays (or to nothing), with its static
|
|
116
|
+
* position / rotation / scale rewritten to hold its place at its own start. Keyed
|
|
117
|
+
* channels keep their keys. Call it before the clips are removed. */
|
|
118
|
+
function detachChildren(project, goneIds) {
|
|
119
|
+
if (!MotionLib || !goneIds || !goneIds.size) return 0;
|
|
120
|
+
const byId = (id) => project.clips.find((x) => x.id === id) || null;
|
|
121
|
+
let n = 0;
|
|
122
|
+
for (const k of project.clips) {
|
|
123
|
+
if (!k.parent || goneIds.has(k.id) || !goneIds.has(k.parent)) continue;
|
|
124
|
+
let up = byId(k.parent), guard = 0;
|
|
125
|
+
while (up && goneIds.has(up.id) && guard++ < 64) up = up.parent ? byId(up.parent) : null;
|
|
126
|
+
const next = up && !goneIds.has(up.id) ? up.id : null;
|
|
127
|
+
const v = MotionLib.reparentValues(k, next, byId, k.start || 0);
|
|
128
|
+
k.props = { ...(k.props || {}) };
|
|
129
|
+
for (const [p, val] of Object.entries(v)) if (!(k.keyframes && Array.isArray(k.keyframes[p]) && k.keyframes[p].length)) k.props[p] = val;
|
|
130
|
+
if (next) k.parent = next; else delete k.parent;
|
|
131
|
+
n++;
|
|
132
|
+
}
|
|
133
|
+
return n;
|
|
134
|
+
}
|
|
110
135
|
/** Default stereo pan for an isolated linked stem (L −1, R +1, else center). */
|
|
111
136
|
function defaultPanForChannel(ch) {
|
|
112
137
|
if (ch === 0) return -1;
|
|
@@ -120,9 +145,16 @@
|
|
|
120
145
|
const timeAt = env.mediaTimeAt || mediaTimeAt;
|
|
121
146
|
const removed = (c) => { if (env.onRemove) env.onRemove(c); };
|
|
122
147
|
const getMedia = (id) => P().media.find((m) => m.id === id);
|
|
148
|
+
/** Length of what c's `in` points into (media or comp), 0 when unknown. */
|
|
149
|
+
const sourceLen = (c) => {
|
|
150
|
+
if (c.kind === "comp") { const k = (P().comps || []).find((x) => x.id === c.compId); return k ? +k.duration || 0 : 0; }
|
|
151
|
+
const m = getMedia(c.mediaId);
|
|
152
|
+
return m ? +m.duration || 0 : 0;
|
|
153
|
+
};
|
|
123
154
|
const getClip = (id) => P().clips.find((c) => c.id === id) || null;
|
|
124
155
|
const dropClips = (pred) => {
|
|
125
156
|
const project = P();
|
|
157
|
+
detachChildren(project, new Set(project.clips.filter(pred).map((x) => x.id)));
|
|
126
158
|
const keep = [];
|
|
127
159
|
for (const x of project.clips) { if (pred(x)) removed(x); else keep.push(x); }
|
|
128
160
|
project.clips = keep;
|
|
@@ -286,7 +318,7 @@
|
|
|
286
318
|
const dIn = t0 - start;
|
|
287
319
|
if (dIn > 1e-6) {
|
|
288
320
|
c.start = t0;
|
|
289
|
-
if (
|
|
321
|
+
if (hasSourceTime(c)) c.in += dIn * clipSpeed(c);
|
|
290
322
|
else c.in = 0;
|
|
291
323
|
c.duration -= dIn;
|
|
292
324
|
c.keyframes = shiftKF(c.keyframes, dIn, c.duration);
|
|
@@ -676,10 +708,10 @@
|
|
|
676
708
|
|
|
677
709
|
/** Longest this clip may run before it runs out of source media. */
|
|
678
710
|
function maxClipDur(c) {
|
|
679
|
-
if (!
|
|
680
|
-
const
|
|
681
|
-
if (!
|
|
682
|
-
return Math.max(MIN_DUR, (
|
|
711
|
+
if (!hasSourceTime(c)) return Infinity;
|
|
712
|
+
const len = sourceLen(c);
|
|
713
|
+
if (!(len > 0)) return Infinity;
|
|
714
|
+
return Math.max(MIN_DUR, (len - c.in) / clipSpeed(c));
|
|
683
715
|
}
|
|
684
716
|
/* A linked group shares its timing, but a partner may run at a different
|
|
685
717
|
speed (links are inferred from timing alone), so every source limit is the
|
|
@@ -690,7 +722,7 @@
|
|
|
690
722
|
}
|
|
691
723
|
/** How far c's group can pull its head back before any member reaches source 0. */
|
|
692
724
|
function groupHeadRoom(c) {
|
|
693
|
-
return Math.min(...withLinked([c]).map((x) => (
|
|
725
|
+
return Math.min(...withLinked([c]).map((x) => (hasSourceTime(x) ? x.in / clipSpeed(x) : Infinity)));
|
|
694
726
|
}
|
|
695
727
|
/** The clip butting against c's head ("in") or tail ("out") on its track. */
|
|
696
728
|
function adjacentClip(c, side) {
|
|
@@ -726,7 +758,7 @@
|
|
|
726
758
|
function trimGroupHead(c, d, moveStart) {
|
|
727
759
|
for (const x of withLinked([c])) {
|
|
728
760
|
if (moveStart) x.start = +(x.start + d).toFixed(4);
|
|
729
|
-
if (
|
|
761
|
+
if (hasSourceTime(x)) x.in = +(x.in + d * clipSpeed(x)).toFixed(4);
|
|
730
762
|
x.duration = +(x.duration - d).toFixed(4);
|
|
731
763
|
x.keyframes = shiftKF(x.keyframes, d, x.duration);
|
|
732
764
|
shiftMasks(x, d);
|
|
@@ -796,7 +828,7 @@
|
|
|
796
828
|
* delta > 0 drags the picture right, i.e. earlier source. Position and
|
|
797
829
|
* length stay; clip-local keyframes stay put on the timeline. */
|
|
798
830
|
function slipClip(c, delta) {
|
|
799
|
-
if (!
|
|
831
|
+
if (!hasSourceTime(c)) return "Slip needs a video or audio clip, or a comp clip — stills and titles have no source time";
|
|
800
832
|
if (isGroupLocked(c)) return TRIM_LOCKED;
|
|
801
833
|
const sp = clipSpeed(c);
|
|
802
834
|
const group = withLinked([c]);
|
|
@@ -804,9 +836,9 @@
|
|
|
804
836
|
// consumes its own span of source (speed can differ across the group).
|
|
805
837
|
let maxIn = Infinity;
|
|
806
838
|
for (const x of group) {
|
|
807
|
-
if (!
|
|
808
|
-
const
|
|
809
|
-
if (
|
|
839
|
+
if (!hasSourceTime(x)) continue;
|
|
840
|
+
const len = sourceLen(x);
|
|
841
|
+
if (len > 0) maxIn = Math.min(maxIn, Math.max(0, len - (timeAt(x, clipEnd(x)) - x.in)));
|
|
810
842
|
}
|
|
811
843
|
const newIn = clampD(c.in - delta * sp, 0, maxIn);
|
|
812
844
|
const d = (c.in - newIn) / sp;
|
|
@@ -977,7 +1009,7 @@
|
|
|
977
1009
|
|
|
978
1010
|
return {
|
|
979
1011
|
MIN_DUR, MIN_TRANS_DUR, CROSSFADE_DUR, GAP_EPS, TRIM_LOCKED, DEFAULT_TRACKS,
|
|
980
|
-
clipEnd, clipSpeed, kfChannel, hasSpeedRamp, mediaTimeAt, shiftKF, shiftMasks, defaultPanForChannel,
|
|
1012
|
+
clipEnd, clipSpeed, isMediaClip, hasSourceTime, kfChannel, hasSpeedRamp, mediaTimeAt, shiftKF, shiftMasks, detachChildren, defaultPanForChannel,
|
|
981
1013
|
create, forDoc,
|
|
982
1014
|
};
|
|
983
1015
|
});
|
package/export-jobs.js
CHANGED
|
@@ -49,7 +49,7 @@ function createExportJobs(opts) {
|
|
|
49
49
|
let timer = null;
|
|
50
50
|
const isEnded = (j) => j.status === "done" || j.status === "failed" || j.status === "cancelled";
|
|
51
51
|
const view = (j) => ({
|
|
52
|
-
id: j.id, kind: j.kind, status: j.status, via: j.via, range: j.range, profile: j.profile || null,
|
|
52
|
+
id: j.id, kind: j.kind, status: j.status, via: j.via, range: j.range, profile: j.profile || null, preset: j.delivery ? j.delivery.preset || "custom" : null,
|
|
53
53
|
time: j.time ?? null, progress: j.progress, src: j.src || null, result: j.result || null, error: j.error || null,
|
|
54
54
|
created: j.created, updated: j.updated,
|
|
55
55
|
});
|
|
@@ -104,8 +104,10 @@ function createExportJobs(opts) {
|
|
|
104
104
|
return {
|
|
105
105
|
/** Queue an export. where: auto (open tab, else headless) · tab · headless.
|
|
106
106
|
* kind "scopes" measures one still at `time` instead (fablecut_scopes);
|
|
107
|
-
* kind "track" follows a region of a clip and saves it (fablecut_track).
|
|
108
|
-
|
|
107
|
+
* kind "track" follows a region of a clip and saves it (fablecut_track).
|
|
108
|
+
* A scopes job's `image` flag also returns that still as a JPEG data URL,
|
|
109
|
+
* which is how fablecut_frame shows an agent the composed program picture. */
|
|
110
|
+
request({ kind = "export", range, profile, delivery = null, deliveries = null, time, revision, where = "auto", matte = null, mask = null, track = null, image = false }) {
|
|
109
111
|
if (!["auto", "tab", "headless"].includes(where)) throw Object.assign(new Error("where must be auto, tab or headless"), { code: 400 });
|
|
110
112
|
if (!["export", "scopes", "track"].includes(kind)) throw Object.assign(new Error("kind must be export, scopes or track"), { code: 400 });
|
|
111
113
|
if (time != null && !(Number.isFinite(time) && time >= 0)) throw Object.assign(new Error("time must be seconds ≥ 0"), { code: 400 });
|
|
@@ -117,10 +119,12 @@ function createExportJobs(opts) {
|
|
|
117
119
|
const now = Date.now();
|
|
118
120
|
const j = {
|
|
119
121
|
id: "x_" + Math.random().toString(36).slice(2, 10), kind, status: "pending", progress: 0,
|
|
120
|
-
range: range || null, profile: profile || null,
|
|
122
|
+
range: range || null, profile: profile || null, delivery: kind === "export" && delivery ? delivery : null,
|
|
123
|
+
deliveries: kind === "export" && Array.isArray(deliveries) && deliveries.length ? deliveries : null, time: time ?? null, revision: Number.isFinite(revision) ? revision : null,
|
|
121
124
|
matte: kind === "scopes" && matte ? { clip: String(matte.clip), layer: matte.layer } : null,
|
|
122
125
|
mask: kind === "scopes" && mask ? { clip: String(mask.clip) } : null,
|
|
123
126
|
track: kind === "track" && track ? track : null,
|
|
127
|
+
image: kind === "scopes" && !!image,
|
|
124
128
|
via: where === "headless" || (where === "auto" && !tabs) ? "headless" : "tab",
|
|
125
129
|
created: now, updated: now,
|
|
126
130
|
};
|
|
@@ -132,7 +136,7 @@ function createExportJobs(opts) {
|
|
|
132
136
|
},
|
|
133
137
|
get(id) { const j = jobs.get(id); return j ? view(j) : null; },
|
|
134
138
|
/** What a tab needs to run the job (headless pages fetch it by id). */
|
|
135
|
-
ticket(id) { const j = jobs.get(id); return j ? { id: j.id, kind: j.kind, range: j.range, profile: j.profile, time: j.time, revision: j.revision, matte: j.matte || null, mask: j.mask || null, track: j.track || null, status: j.status } : null; },
|
|
139
|
+
ticket(id) { const j = jobs.get(id); return j ? { id: j.id, kind: j.kind, range: j.range, profile: j.profile, delivery: j.delivery || null, deliveries: j.deliveries || null, time: j.time, revision: j.revision, matte: j.matte || null, mask: j.mask || null, track: j.track || null, image: !!j.image, status: j.status } : null; },
|
|
136
140
|
/** First tab to claim a pending job runs it. */
|
|
137
141
|
claim(id) {
|
|
138
142
|
const j = jobs.get(id);
|
package/frame-cache.js
ADDED
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
/* ═══════════════════════════════════════════════════════════════════════════
|
|
2
|
+
The /api/frame render cache (analysis/frames/) — bounded, oldest-first.
|
|
3
|
+
|
|
4
|
+
Every distinct combination of source + arguments renders its own JPEG, so an
|
|
5
|
+
unbounded cache is a disk-fill primitive: varying w, q or t mints a new file
|
|
6
|
+
each time, and a hostile page could do exactly that through the endpoint.
|
|
7
|
+
(The header gate on ?path= keeps strangers out; this keeps friends honest.)
|
|
8
|
+
Each entry is a .jpg plus its .json sidecar; pruning removes both, oldest
|
|
9
|
+
mtime first, until the directory is back under both limits.
|
|
10
|
+
|
|
11
|
+
Pure filesystem logic, no ffmpeg — unit-tested directly in frame-cache.test.js.
|
|
12
|
+
═══════════════════════════════════════════════════════════════════════════ */
|
|
13
|
+
"use strict";
|
|
14
|
+
const fs = require("fs");
|
|
15
|
+
const path = require("path");
|
|
16
|
+
|
|
17
|
+
/* Generous enough that a real editing session never notices: a few hundred
|
|
18
|
+
grabs at ~100 KB each is tens of megabytes. */
|
|
19
|
+
const DEFAULT_MAX_FILES = 500;
|
|
20
|
+
const DEFAULT_MAX_BYTES = 128 * 1024 * 1024;
|
|
21
|
+
|
|
22
|
+
/** Delete oldest entries until dir holds ≤ maxFiles JPEGs and ≤ maxBytes.
|
|
23
|
+
* Returns {files, bytes, pruned}. Never throws — a prune failure must not
|
|
24
|
+
* fail the request that triggered it. */
|
|
25
|
+
function pruneCache(dir, { maxFiles = DEFAULT_MAX_FILES, maxBytes = DEFAULT_MAX_BYTES } = {}) {
|
|
26
|
+
let names;
|
|
27
|
+
try { names = fs.readdirSync(dir); } catch { return { files: 0, bytes: 0, pruned: 0 }; }
|
|
28
|
+
const entries = [];
|
|
29
|
+
for (const name of names) {
|
|
30
|
+
if (!name.endsWith(".jpg")) continue;
|
|
31
|
+
try {
|
|
32
|
+
const st = fs.statSync(path.join(dir, name));
|
|
33
|
+
if (st.isFile()) entries.push({ name, size: st.size, mtime: st.mtimeMs });
|
|
34
|
+
} catch { /* vanished mid-sweep — someone else pruned it */ }
|
|
35
|
+
}
|
|
36
|
+
entries.sort((a, b) => a.mtime - b.mtime); // oldest first
|
|
37
|
+
let bytes = entries.reduce((s, e) => s + e.size, 0);
|
|
38
|
+
let pruned = 0;
|
|
39
|
+
while ((entries.length > maxFiles || bytes > maxBytes) && entries.length) {
|
|
40
|
+
const victim = entries.shift();
|
|
41
|
+
bytes -= victim.size;
|
|
42
|
+
pruned++;
|
|
43
|
+
try { fs.rmSync(path.join(dir, victim.name)); } catch {}
|
|
44
|
+
try { fs.rmSync(path.join(dir, victim.name.replace(/\.jpg$/, ".json"))); } catch {}
|
|
45
|
+
}
|
|
46
|
+
return { files: entries.length, bytes, pruned };
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
module.exports = { pruneCache, DEFAULT_MAX_FILES, DEFAULT_MAX_BYTES };
|
package/frame-stats.js
ADDED
|
@@ -0,0 +1,267 @@
|
|
|
1
|
+
/* ═══════════════════════════════════════════════════════════════════════════
|
|
2
|
+
Frame statistics — describe a frame in words, for anyone who can't see it.
|
|
3
|
+
|
|
4
|
+
`fablecut_frame` hands a multimodal model an actual image, which is the point.
|
|
5
|
+
But not every MCP client, and not every model behind it, can read one: a
|
|
6
|
+
text-only agent gets the coordinates of the picture and nothing else, so it
|
|
7
|
+
learns nothing about the footage it was asked to judge.
|
|
8
|
+
|
|
9
|
+
This closes that gap with measurements instead of pixels — brightness, the
|
|
10
|
+
dominant colours, and how much a moment differs from the one before it — the
|
|
11
|
+
same numbers `fablecut_scopes` reports for the composed picture, applied to a
|
|
12
|
+
source frame. Deliberately NOT a caption: no zero-dependency way to describe
|
|
13
|
+
what is in the picture honestly. What it gives instead is enough to rank and
|
|
14
|
+
route footage ("bright outdoor, high motion, dominant green") without a
|
|
15
|
+
picture at all.
|
|
16
|
+
|
|
17
|
+
Pure functions over a raw RGB buffer. No ffmpeg, no canvas, no dependencies,
|
|
18
|
+
so the whole thing is testable on its own. ═══════════════════════════════════ */
|
|
19
|
+
"use strict";
|
|
20
|
+
|
|
21
|
+
/* Rec. 709 luma weights — the same coefficients the browser uses, so a number
|
|
22
|
+
here means the same thing as a number from fablecut_scopes. */
|
|
23
|
+
/* Every luma field below is 0–1, rounded to 3 decimals — the same convention
|
|
24
|
+
as color.js scopeStats, so the two readings can sit side by side without a
|
|
25
|
+
conversion nobody remembers to do. */
|
|
26
|
+
const WR = 0.2126, WG = 0.7152, WB = 0.0722;
|
|
27
|
+
const r3 = (v) => Math.round(v * 1000) / 1000;
|
|
28
|
+
|
|
29
|
+
/* Perceptual buckets. A 4-bit-per-channel histogram is 4096 entries: fine
|
|
30
|
+
enough to separate a lime from a green, coarse enough that JPEG noise in a
|
|
31
|
+
gradient lands in one bucket instead of smearing across a hundred. */
|
|
32
|
+
const BITS = 4, SHIFT = 8 - BITS, BUCKETS = 1 << (BITS * 3);
|
|
33
|
+
|
|
34
|
+
/** Rec. 709 luma of one pixel, 0–255. */
|
|
35
|
+
function luma(r, g, b) { return WR * r + WG * g + WB * b; }
|
|
36
|
+
|
|
37
|
+
/* Named colours a model can reason about, matched against the histogram's top
|
|
38
|
+
buckets. These are the useful coarse categories — "sky", "foliage", "skin",
|
|
39
|
+
"night" — rather than 4096 hex codes nobody can picture. Nearest match by
|
|
40
|
+
weighted RGB distance, so a slightly-off bucket still lands sensibly. */
|
|
41
|
+
const NAMED = [
|
|
42
|
+
{ name: "black", rgb: [10, 10, 12] },
|
|
43
|
+
{ name: "deep blue", rgb: [16, 26, 74] },
|
|
44
|
+
{ name: "blue", rgb: [40, 84, 190] },
|
|
45
|
+
{ name: "sky blue", rgb: [128, 186, 235] },
|
|
46
|
+
{ name: "cyan", rgb: [56, 190, 200] },
|
|
47
|
+
{ name: "green", rgb: [56, 142, 62] },
|
|
48
|
+
{ name: "lime", rgb: [140, 205, 60] },
|
|
49
|
+
{ name: "olive", rgb: [120, 118, 52] },
|
|
50
|
+
{ name: "yellow", rgb: [226, 200, 62] },
|
|
51
|
+
{ name: "orange", rgb: [222, 122, 44] },
|
|
52
|
+
{ name: "red", rgb: [196, 52, 44] },
|
|
53
|
+
{ name: "dark red", rgb: [104, 26, 26] },
|
|
54
|
+
{ name: "magenta", rgb: [176, 58, 168] },
|
|
55
|
+
{ name: "skin", rgb: [214, 168, 132] },
|
|
56
|
+
{ name: "grey", rgb: [128, 128, 130] },
|
|
57
|
+
{ name: "light grey", rgb: [206, 206, 208] },
|
|
58
|
+
{ name: "white", rgb: [242, 242, 244] },
|
|
59
|
+
];
|
|
60
|
+
function nameColor(r, g, b) {
|
|
61
|
+
let best = NAMED[0], bestD = Infinity;
|
|
62
|
+
for (const c of NAMED) {
|
|
63
|
+
/* Plain Euclidean RGB distance. Luma weighting looks principled and is
|
|
64
|
+
wrong here: Rec.709 gives blue a weight of 0.07, so a bucket that differs
|
|
65
|
+
from "blue" mostly in blue reads as "deep blue" instead. For naming a
|
|
66
|
+
colour, hue is the thing being asked about. */
|
|
67
|
+
const d = (r - c.rgb[0]) ** 2 + (g - c.rgb[1]) ** 2 + (b - c.rgb[2]) ** 2;
|
|
68
|
+
if (d < bestD) { bestD = d; best = c; }
|
|
69
|
+
}
|
|
70
|
+
return best.name;
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
/** Bucket a pixel into the 4-bit-per-channel histogram. */
|
|
74
|
+
const bucketOf = (r, g, b) => ((r >> SHIFT) << (BITS * 2)) | ((g >> SHIFT) << BITS) | (b >> SHIFT);
|
|
75
|
+
function bucketRgb(i) {
|
|
76
|
+
return [((i >> (BITS * 2)) & 0xf) << SHIFT, ((i >> BITS) & 0xf) << SHIFT, (i & 0xf) << SHIFT];
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
/**
|
|
80
|
+
* Describe one frame.
|
|
81
|
+
*
|
|
82
|
+
* @param rgb raw RGB24 bytes (w*h*3)
|
|
83
|
+
* @param w,h its dimensions
|
|
84
|
+
* @param opts {stride} — bytes per row, when rgb is a window into a bigger
|
|
85
|
+
* buffer (the contact sheet slices cells out of one grid image)
|
|
86
|
+
* @returns {{luma:object, brightness:string, colours:Array, saturation:number,
|
|
87
|
+
* meanRgb:number[], texture:number, samples:number}}
|
|
88
|
+
*/
|
|
89
|
+
function describe(rgb, w, h, { stride = w * 3, offset = 0 } = {}) {
|
|
90
|
+
const n = w * h;
|
|
91
|
+
const hist = new Uint32Array(BUCKETS);
|
|
92
|
+
const lums = new Float64Array(n);
|
|
93
|
+
let sr = 0, sg = 0, sb = 0, k = 0;
|
|
94
|
+
for (let y = 0; y < h; y++) {
|
|
95
|
+
let i = offset + y * stride;
|
|
96
|
+
for (let x = 0; x < w; x++, i += 3, k++) {
|
|
97
|
+
const r = rgb[i], g = rgb[i + 1], b = rgb[i + 2];
|
|
98
|
+
sr += r; sg += g; sb += b;
|
|
99
|
+
lums[k] = luma(r, g, b);
|
|
100
|
+
hist[bucketOf(r, g, b)]++;
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
// Percentiles need a sort; a frame is at most a few hundred thousand samples,
|
|
104
|
+
// and this runs once per call, not per frame of a render.
|
|
105
|
+
const sorted = Float64Array.prototype.slice.call(lums).sort();
|
|
106
|
+
const q = (p) => sorted[Math.min(n - 1, Math.max(0, Math.round(p * (n - 1))))];
|
|
107
|
+
let sum = 0;
|
|
108
|
+
for (let i = 0; i < n; i++) sum += lums[i];
|
|
109
|
+
const mean = sum / n / 255;
|
|
110
|
+
const meanRgb = [sr / n, sg / n, sb / n];
|
|
111
|
+
|
|
112
|
+
// Saturation 0–1, from mean chroma against mean luma (the cheap version —
|
|
113
|
+
// no per-pixel HSV pass).
|
|
114
|
+
const chroma = (Math.max(...meanRgb) - Math.min(...meanRgb)) / 255;
|
|
115
|
+
const lum01 = mean;
|
|
116
|
+
const saturation = lum01 < 0.005 ? 0 : Math.min(1, chroma / (lum01 * 2));
|
|
117
|
+
|
|
118
|
+
/* "Texture": the mean absolute luma step between horizontally adjacent
|
|
119
|
+
samples, as a percentage of full range. Smooth sky or a locked-off tripod
|
|
120
|
+
shot scores near 0; foliage, crowds or a busy frame score clearly higher.
|
|
121
|
+
Reported on the same 0–100 scale as the analyzer's per-shot `energy`, so
|
|
122
|
+
the two can be compared directly. It measures detail *within* the frame,
|
|
123
|
+
not change over time — motionScore() is the change-over-time one. */
|
|
124
|
+
let diff = 0, dN = 0;
|
|
125
|
+
for (let y = 0; y < h; y++) {
|
|
126
|
+
// Start one pixel in: the step reads the pixel before the cursor, and on
|
|
127
|
+
// the first row that would be rgb[-3] — undefined, and silently NaN.
|
|
128
|
+
let i = offset + y * stride + 3;
|
|
129
|
+
for (let x = 1; x < w; x++, i += 3) {
|
|
130
|
+
diff += Math.abs(luma(rgb[i], rgb[i + 1], rgb[i + 2]) - luma(rgb[i - 3], rgb[i - 2], rgb[i - 1]));
|
|
131
|
+
dN++;
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
// A single-pixel-wide sample has no horizontal neighbours; report no texture
|
|
135
|
+
// rather than NaN.
|
|
136
|
+
const detail = dN ? Math.round(diff / dN / 255 * 1000) / 10 : 0;
|
|
137
|
+
|
|
138
|
+
// Top colour buckets, merged when they are neighbours so a sky reads as one
|
|
139
|
+
// entry rather than four.
|
|
140
|
+
const top = [];
|
|
141
|
+
for (let i = 0; i < BUCKETS; i++) {
|
|
142
|
+
if (hist[i]) top.push([i, hist[i]]);
|
|
143
|
+
}
|
|
144
|
+
top.sort((a, b) => b[1] - a[1]);
|
|
145
|
+
const colours = [];
|
|
146
|
+
for (const [idx, count] of top) {
|
|
147
|
+
const [r, g, b] = bucketRgb(idx);
|
|
148
|
+
const near = colours.find((c) =>
|
|
149
|
+
(c.rgb[0] - r) ** 2 + (c.rgb[1] - g) ** 2 + (c.rgb[2] - b) ** 2 < 900);
|
|
150
|
+
if (near) {
|
|
151
|
+
// Weighted mean, so the merged entry points at the colour actually seen.
|
|
152
|
+
const tot = near.count + count;
|
|
153
|
+
near.rgb = near.rgb.map((v, j) => Math.round((v * near.count + [r, g, b][j] * count) / tot));
|
|
154
|
+
near.count = tot;
|
|
155
|
+
near.pct = tot;
|
|
156
|
+
} else {
|
|
157
|
+
colours.push({ rgb: [r, g, b], count, pct: count });
|
|
158
|
+
}
|
|
159
|
+
}
|
|
160
|
+
colours.sort((a, b) => b.count - a.count);
|
|
161
|
+
const shown = colours.slice(0, 3);
|
|
162
|
+
for (const c of shown) {
|
|
163
|
+
c.pct = Math.round(c.pct / n * 100);
|
|
164
|
+
c.hex = "#" + c.rgb.map((v) => v.toString(16).padStart(2, "0")).join("");
|
|
165
|
+
c.name = nameColor(c.rgb[0], c.rgb[1], c.rgb[2]);
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
const l = mean;
|
|
169
|
+
return {
|
|
170
|
+
luma: {
|
|
171
|
+
min: r3(q(0) / 255), p1: r3(q(0.01) / 255), median: r3(q(0.5) / 255),
|
|
172
|
+
mean: r3(l), p99: r3(q(0.99) / 255), max: r3(q(1) / 255),
|
|
173
|
+
},
|
|
174
|
+
brightness: l < 0.12 ? "very dark" : l < 0.35 ? "dark" : l < 0.65 ? "mid" : l < 0.85 ? "bright" : "very bright",
|
|
175
|
+
colours: shown,
|
|
176
|
+
meanRgb: meanRgb.map((v) => Math.round(v)),
|
|
177
|
+
saturation: Math.round(saturation * 100) / 100,
|
|
178
|
+
texture: detail,
|
|
179
|
+
samples: n,
|
|
180
|
+
};
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
/**
|
|
184
|
+
* Mean absolute per-channel difference between two equally sized regions,
|
|
185
|
+
* 0–1. High means the picture changed a lot between them — a cut, or fast
|
|
186
|
+
* motion. This is the "is this a different shot" signal a contact sheet's cells
|
|
187
|
+
* can be ranked by.
|
|
188
|
+
*/
|
|
189
|
+
function motionScore(rgbA, rgbB, w, h) {
|
|
190
|
+
const n = w * h * 3;
|
|
191
|
+
if (!n || !rgbA || !rgbB || rgbA.length < n || rgbB.length < n) return 0;
|
|
192
|
+
let d = 0;
|
|
193
|
+
for (let i = 0; i < n; i++) d += Math.abs(rgbA[i] - rgbB[i]);
|
|
194
|
+
return Math.round(d / n / 255 * 1000) / 1000;
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
/**
|
|
198
|
+
* Describe a tiled contact sheet cell by cell. `rgb` is the whole grid image;
|
|
199
|
+
* cells are read left→right then top→bottom, so cell i is the frame at times[i].
|
|
200
|
+
*
|
|
201
|
+
* Each cell reports its own brightness/colours, plus motion against the previous
|
|
202
|
+
* populated cell — which is what lets a text-only agent find the cut points and
|
|
203
|
+
* pick the high-motion moments without seeing anything.
|
|
204
|
+
*
|
|
205
|
+
* @param rgb raw RGB24 of the whole grid
|
|
206
|
+
* @param w,h grid dimensions
|
|
207
|
+
* @param cols,rows the tile layout
|
|
208
|
+
* @param times one timestamp per cell, in cell order
|
|
209
|
+
*/
|
|
210
|
+
function describeGrid(rgb, w, h, cols, rows, times = []) {
|
|
211
|
+
// tile lays cells out edge to edge, so the cell size is the grid divided by
|
|
212
|
+
// the layout. Guard the rounding: a width that doesn't divide evenly would
|
|
213
|
+
// drift a pixel per row and smear the last column.
|
|
214
|
+
const cw = Math.floor(w / cols), ch = Math.floor(h / rows);
|
|
215
|
+
const cells = [];
|
|
216
|
+
let prev = null, prevIdx = -1;
|
|
217
|
+
for (let r = 0; r < rows; r++) {
|
|
218
|
+
for (let c = 0; c < cols; c++) {
|
|
219
|
+
const index = r * cols + c;
|
|
220
|
+
const time = times[index];
|
|
221
|
+
// Fewer timestamps than cells means tile padded the last row with black —
|
|
222
|
+
// say so rather than describing a padding cell as a black frame.
|
|
223
|
+
if (time == null) { cells.push({ index, padded: true }); continue; }
|
|
224
|
+
const ox = c * cw, oy = r * ch;
|
|
225
|
+
const view = new Uint8Array(cw * ch * 3);
|
|
226
|
+
for (let y = 0; y < ch; y++) {
|
|
227
|
+
const from = ((oy + y) * w + ox) * 3;
|
|
228
|
+
view.set(rgb.subarray(from, from + cw * 3), y * cw * 3);
|
|
229
|
+
}
|
|
230
|
+
const stats = describe(view, cw, ch);
|
|
231
|
+
const cell = { index, time, ...stats };
|
|
232
|
+
// Compare against the previous *real* cell, skipping padding, so the last
|
|
233
|
+
// filled cell of a short row is still measured against its neighbour.
|
|
234
|
+
cell.motion = prev ? motionScore(view, prev, cw, ch) : 0;
|
|
235
|
+
cells.push(cell);
|
|
236
|
+
prev = view; prevIdx = index;
|
|
237
|
+
}
|
|
238
|
+
}
|
|
239
|
+
return { cols, rows, cellWidth: cw, cellHeight: ch, cells };
|
|
240
|
+
}
|
|
241
|
+
|
|
242
|
+
/** One line per thing worth saying, for a single frame. These lines are the whole
|
|
243
|
+
* point of the text fallback: read by a model that never sees the pixels.
|
|
244
|
+
* Keep them short — this is what an agent gets instead of the image. */
|
|
245
|
+
function frameLines(label, stats) {
|
|
246
|
+
const L = stats.luma;
|
|
247
|
+
const cols = stats.colours.map((c) => `${c.name} ${c.hex} ${c.pct}%`).join(", ");
|
|
248
|
+
return [
|
|
249
|
+
`${label}: ${stats.brightness} (luma mean ${L.mean}, median ${L.median}, range ${L.min}–${L.max})`,
|
|
250
|
+
` dominant: ${cols || "flat"}`,
|
|
251
|
+
` texture ${stats.texture}/100 · saturation ${stats.saturation} · mean rgb [${stats.meanRgb.join(", ")}]`,
|
|
252
|
+
];
|
|
253
|
+
}
|
|
254
|
+
|
|
255
|
+
/** One line per cell, for a contact sheet. `motion` is the change since the
|
|
256
|
+
* previous cell, so it spikes at a cut *and* during fast movement — it ranks
|
|
257
|
+
* the most active moments, it does not prove there was a cut. */
|
|
258
|
+
function gridLines(grid) {
|
|
259
|
+
const lines = grid.cells.map((c) => {
|
|
260
|
+
if (c.padded) return ` cell ${c.index + 1}: (padding — not a frame in this clip)`;
|
|
261
|
+
return ` cell ${c.index + 1} @ ${c.time}s: ${c.brightness}, ${c.colours.map((x) => `${x.name} ${x.pct}%`).join(" / ") || "flat"}`
|
|
262
|
+
+ ` · texture ${c.texture}/100 · change ${c.motion}`;
|
|
263
|
+
});
|
|
264
|
+
return [`Contact sheet ${grid.cols}x${grid.rows}, ${grid.cells.filter((c) => !c.padded).length} frames:`, ...lines];
|
|
265
|
+
}
|
|
266
|
+
|
|
267
|
+
module.exports = { describe, describeGrid, motionScore, luma, nameColor, frameLines, gridLines };
|