@framefields/node-kenburns 2.0.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.turbo/turbo-build.log +20 -0
- package/CHANGELOG.md +23 -0
- package/LICENCE +202 -0
- package/dist/index.d.mts +256 -0
- package/dist/index.d.mts.map +1 -0
- package/dist/index.mjs +122 -0
- package/dist/index.mjs.map +1 -0
- package/dist/renderer.d.mts +7 -0
- package/dist/renderer.d.mts.map +1 -0
- package/dist/renderer.mjs +3 -0
- package/dist/renderers-BoufaXfn.mjs +336 -0
- package/dist/renderers-BoufaXfn.mjs.map +1 -0
- package/package.json +46 -0
- package/src/index.ts +2 -0
- package/src/renderers/index.ts +6 -0
- package/src/renderers/webgpu-renderer.ts +500 -0
- package/src/shared/config.ts +107 -0
- package/src/shared/index.ts +2 -0
- package/src/shared/processor-logic.ts +108 -0
- package/tsconfig.json +13 -0
- package/tsdown.config.ts +13 -0
|
@@ -0,0 +1,500 @@
|
|
|
1
|
+
import type { WebGPUNodeRenderer } from "@framefields/node-sdk";
|
|
2
|
+
import type { KenBurnsKeyframe } from "../shared/config.js";
|
|
3
|
+
|
|
4
|
+
const easeIn = (t: number): number => t * t;
|
|
5
|
+
const easeOut = (t: number): number => t * (2 - t);
|
|
6
|
+
const easeInOut = (t: number): number =>
|
|
7
|
+
t < 0.5 ? 2 * t * t : -1 + (4 - 2 * t) * t;
|
|
8
|
+
const springEase = (t: number): number => {
|
|
9
|
+
const c4 = (2 * Math.PI) / 3;
|
|
10
|
+
return t === 0
|
|
11
|
+
? 0
|
|
12
|
+
: t === 1
|
|
13
|
+
? 1
|
|
14
|
+
: 2 ** (-10 * t) * Math.sin((t * 10 - 0.75) * c4) + 1;
|
|
15
|
+
};
|
|
16
|
+
|
|
17
|
+
const applyEasing = (t: number, easing?: string): number => {
|
|
18
|
+
switch (easing) {
|
|
19
|
+
case "ease-in":
|
|
20
|
+
return easeIn(t);
|
|
21
|
+
case "ease-out":
|
|
22
|
+
return easeOut(t);
|
|
23
|
+
case "ease-in-out":
|
|
24
|
+
return easeInOut(t);
|
|
25
|
+
case "spring":
|
|
26
|
+
return springEase(t);
|
|
27
|
+
default:
|
|
28
|
+
return t;
|
|
29
|
+
}
|
|
30
|
+
};
|
|
31
|
+
|
|
32
|
+
const catmullRom = (
|
|
33
|
+
p0: number,
|
|
34
|
+
p1: number,
|
|
35
|
+
p2: number,
|
|
36
|
+
p3: number,
|
|
37
|
+
t: number,
|
|
38
|
+
): number => {
|
|
39
|
+
const t2 = t * t;
|
|
40
|
+
const t3 = t2 * t;
|
|
41
|
+
return (
|
|
42
|
+
0.5 *
|
|
43
|
+
(2 * p1 +
|
|
44
|
+
(-p0 + p2) * t +
|
|
45
|
+
(2 * p0 - 5 * p1 + 4 * p2 - p3) * t2 +
|
|
46
|
+
(-p0 + 3 * p1 - 3 * p2 + p3) * t3)
|
|
47
|
+
);
|
|
48
|
+
};
|
|
49
|
+
|
|
50
|
+
interface InterpolatedState {
|
|
51
|
+
x: number;
|
|
52
|
+
y: number;
|
|
53
|
+
scale: number;
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
function interpolateKenBurns(
|
|
57
|
+
keyframes: KenBurnsKeyframe[],
|
|
58
|
+
timeMs: number,
|
|
59
|
+
movementStyle = "spline",
|
|
60
|
+
): InterpolatedState {
|
|
61
|
+
if (keyframes.length === 0) return { x: 50, y: 50, scale: 1 };
|
|
62
|
+
if (keyframes.length === 1) {
|
|
63
|
+
const kf = keyframes[0];
|
|
64
|
+
return { x: kf.x ?? 50, y: kf.y ?? 50, scale: kf.scale ?? 1 };
|
|
65
|
+
}
|
|
66
|
+
let elapsed = 0;
|
|
67
|
+
for (let i = 0; i < keyframes.length; i++) {
|
|
68
|
+
const kf = keyframes[i];
|
|
69
|
+
const hold = kf.holdMs ?? 0;
|
|
70
|
+
const dur = kf.durationMs ?? 0;
|
|
71
|
+
if (timeMs <= elapsed + hold)
|
|
72
|
+
return { x: kf.x ?? 50, y: kf.y ?? 50, scale: kf.scale ?? 1 };
|
|
73
|
+
elapsed += hold;
|
|
74
|
+
if (i < keyframes.length - 1 && dur > 0) {
|
|
75
|
+
if (timeMs <= elapsed + dur) {
|
|
76
|
+
const rawT = (timeMs - elapsed) / dur;
|
|
77
|
+
const t = applyEasing(Math.max(0, Math.min(1, rawT)), kf.easing);
|
|
78
|
+
const next = keyframes[i + 1];
|
|
79
|
+
if (movementStyle === "spline") {
|
|
80
|
+
const prev = keyframes[i - 1] ?? kf;
|
|
81
|
+
const afterNext = keyframes[i + 2] ?? next;
|
|
82
|
+
return {
|
|
83
|
+
x: catmullRom(
|
|
84
|
+
prev.x ?? 50,
|
|
85
|
+
kf.x ?? 50,
|
|
86
|
+
next.x ?? 50,
|
|
87
|
+
afterNext.x ?? 50,
|
|
88
|
+
t,
|
|
89
|
+
),
|
|
90
|
+
y: catmullRom(
|
|
91
|
+
prev.y ?? 50,
|
|
92
|
+
kf.y ?? 50,
|
|
93
|
+
next.y ?? 50,
|
|
94
|
+
afterNext.y ?? 50,
|
|
95
|
+
t,
|
|
96
|
+
),
|
|
97
|
+
scale: catmullRom(
|
|
98
|
+
prev.scale ?? 1,
|
|
99
|
+
kf.scale ?? 1,
|
|
100
|
+
next.scale ?? 1,
|
|
101
|
+
afterNext.scale ?? 1,
|
|
102
|
+
t,
|
|
103
|
+
),
|
|
104
|
+
};
|
|
105
|
+
}
|
|
106
|
+
return {
|
|
107
|
+
x: (kf.x ?? 50) + ((next.x ?? 50) - (kf.x ?? 50)) * t,
|
|
108
|
+
y: (kf.y ?? 50) + ((next.y ?? 50) - (kf.y ?? 50)) * t,
|
|
109
|
+
scale: (kf.scale ?? 1) + ((next.scale ?? 1) - (kf.scale ?? 1)) * t,
|
|
110
|
+
};
|
|
111
|
+
}
|
|
112
|
+
elapsed += dur;
|
|
113
|
+
}
|
|
114
|
+
}
|
|
115
|
+
const last = keyframes[keyframes.length - 1];
|
|
116
|
+
return { x: last.x ?? 50, y: last.y ?? 50, scale: last.scale ?? 1 };
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
function calculateSrcRect(
|
|
120
|
+
state: InterpolatedState,
|
|
121
|
+
sourceWidth: number,
|
|
122
|
+
sourceHeight: number,
|
|
123
|
+
targetAspectRatio: string,
|
|
124
|
+
) {
|
|
125
|
+
let baseBoxW_pct = 100;
|
|
126
|
+
let baseBoxH_pct = 100;
|
|
127
|
+
|
|
128
|
+
if (targetAspectRatio !== "input") {
|
|
129
|
+
const parts = targetAspectRatio.split(":");
|
|
130
|
+
const tW = Number(parts[0]);
|
|
131
|
+
const tH = Number(parts[1]);
|
|
132
|
+
if (!Number.isNaN(tW) && !Number.isNaN(tH) && tW > 0 && tH > 0) {
|
|
133
|
+
const desiredAR = tW / tH;
|
|
134
|
+
const sourceAR = sourceWidth / sourceHeight;
|
|
135
|
+
if (sourceAR > desiredAR) {
|
|
136
|
+
baseBoxH_pct = 100;
|
|
137
|
+
baseBoxW_pct = (desiredAR / sourceAR) * 100;
|
|
138
|
+
} else {
|
|
139
|
+
baseBoxW_pct = 100;
|
|
140
|
+
baseBoxH_pct = (sourceAR / desiredAR) * 100;
|
|
141
|
+
}
|
|
142
|
+
}
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
const baseBoxPxW = (baseBoxW_pct / 100) * sourceWidth;
|
|
146
|
+
const baseBoxPxH = (baseBoxH_pct / 100) * sourceHeight;
|
|
147
|
+
const clampedScale = Math.max(state.scale, 1);
|
|
148
|
+
const currentBoxPxW = baseBoxPxW / clampedScale;
|
|
149
|
+
const currentBoxPxH = baseBoxPxH / clampedScale;
|
|
150
|
+
|
|
151
|
+
const rawFocalX = (state.x / 100) * sourceWidth;
|
|
152
|
+
const rawFocalY = (state.y / 100) * sourceHeight;
|
|
153
|
+
const halfBoxW = currentBoxPxW / 2;
|
|
154
|
+
const halfBoxH = currentBoxPxH / 2;
|
|
155
|
+
const focalX = Math.max(
|
|
156
|
+
halfBoxW,
|
|
157
|
+
Math.min(sourceWidth - halfBoxW, rawFocalX),
|
|
158
|
+
);
|
|
159
|
+
const focalY = Math.max(
|
|
160
|
+
halfBoxH,
|
|
161
|
+
Math.min(sourceHeight - halfBoxH, rawFocalY),
|
|
162
|
+
);
|
|
163
|
+
|
|
164
|
+
return {
|
|
165
|
+
x: focalX - halfBoxW,
|
|
166
|
+
y: focalY - halfBoxH,
|
|
167
|
+
width: currentBoxPxW,
|
|
168
|
+
height: currentBoxPxH,
|
|
169
|
+
};
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
const motionBlurWgsl = `
|
|
173
|
+
struct MotionBlurUniforms {
|
|
174
|
+
prev_origin : vec2<f32>,
|
|
175
|
+
prev_size : vec2<f32>,
|
|
176
|
+
curr_origin : vec2<f32>,
|
|
177
|
+
curr_size : vec2<f32>,
|
|
178
|
+
samples : f32,
|
|
179
|
+
blurSize : f32,
|
|
180
|
+
padding : vec2<f32>,
|
|
181
|
+
};
|
|
182
|
+
|
|
183
|
+
@group(0) @binding(0) var<uniform> u : MotionBlurUniforms;
|
|
184
|
+
@group(1) @binding(0) var tex : texture_2d<f32>;
|
|
185
|
+
@group(1) @binding(1) var samp : sampler;
|
|
186
|
+
|
|
187
|
+
struct VSOut {
|
|
188
|
+
@builtin(position) pos : vec4<f32>,
|
|
189
|
+
@location(0) uv : vec2<f32>,
|
|
190
|
+
};
|
|
191
|
+
|
|
192
|
+
@vertex fn vs(@builtin(vertex_index) vi: u32) -> VSOut {
|
|
193
|
+
var pos = array<vec2<f32>, 4>(
|
|
194
|
+
vec2<f32>(-1.0, 1.0),
|
|
195
|
+
vec2<f32>(1.0, 1.0),
|
|
196
|
+
vec2<f32>(-1.0, -1.0),
|
|
197
|
+
vec2<f32>(1.0, -1.0)
|
|
198
|
+
);
|
|
199
|
+
var uv = array<vec2<f32>, 4>(
|
|
200
|
+
vec2<f32>(0.0, 0.0),
|
|
201
|
+
vec2<f32>(1.0, 0.0),
|
|
202
|
+
vec2<f32>(0.0, 1.0),
|
|
203
|
+
vec2<f32>(1.0, 1.0)
|
|
204
|
+
);
|
|
205
|
+
return VSOut(vec4<f32>(pos[vi], 0.0, 1.0), uv[vi]);
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
@fragment fn fs(in : VSOut) -> @location(0) vec4<f32> {
|
|
209
|
+
let uv = in.uv;
|
|
210
|
+
let n = i32(u.samples);
|
|
211
|
+
|
|
212
|
+
if (n <= 1) {
|
|
213
|
+
let p_curr = u.curr_origin + uv * u.curr_size;
|
|
214
|
+
return textureSampleLevel(tex, samp, p_curr, 0.0);
|
|
215
|
+
}
|
|
216
|
+
|
|
217
|
+
var color = vec4<f32>(0.0);
|
|
218
|
+
for (var i = 0; i < n; i = i + 1) {
|
|
219
|
+
// Use blurSize to control the shutter interval width.
|
|
220
|
+
// t=1.0 is current frame, t=0.0 is previous frame.
|
|
221
|
+
let t = 1.0 + (f32(i) / f32(n - 1) - 0.5) * u.blurSize;
|
|
222
|
+
|
|
223
|
+
let origin = mix(u.prev_origin, u.curr_origin, t);
|
|
224
|
+
let size = mix(u.prev_size, u.curr_size, t);
|
|
225
|
+
let p_sample = origin + uv * size;
|
|
226
|
+
|
|
227
|
+
color += textureSampleLevel(tex, samp, p_sample, 0.0);
|
|
228
|
+
}
|
|
229
|
+
return color / f32(n);
|
|
230
|
+
}
|
|
231
|
+
`;
|
|
232
|
+
|
|
233
|
+
interface DeviceKenBurnsResources {
|
|
234
|
+
motionBlurPipeline: GPURenderPipeline;
|
|
235
|
+
motionBlurUniformLayout: GPUBindGroupLayout;
|
|
236
|
+
singleTextureLayout: GPUBindGroupLayout;
|
|
237
|
+
}
|
|
238
|
+
|
|
239
|
+
const deviceResourceCache = new WeakMap<GPUDevice, DeviceKenBurnsResources>();
|
|
240
|
+
const motionBlurData = new Float32Array(12);
|
|
241
|
+
|
|
242
|
+
function getMotionBlurResources(device: GPUDevice, format: GPUTextureFormat) {
|
|
243
|
+
let res = deviceResourceCache.get(device);
|
|
244
|
+
if (!res) {
|
|
245
|
+
const motionBlurUniformLayout = device.createBindGroupLayout({
|
|
246
|
+
entries: [
|
|
247
|
+
{
|
|
248
|
+
binding: 0,
|
|
249
|
+
visibility: GPUShaderStage.FRAGMENT,
|
|
250
|
+
buffer: { type: "uniform" },
|
|
251
|
+
},
|
|
252
|
+
],
|
|
253
|
+
});
|
|
254
|
+
const singleTextureLayout = device.createBindGroupLayout({
|
|
255
|
+
entries: [
|
|
256
|
+
{
|
|
257
|
+
binding: 0,
|
|
258
|
+
visibility: GPUShaderStage.FRAGMENT,
|
|
259
|
+
texture: { sampleType: "float" },
|
|
260
|
+
},
|
|
261
|
+
{
|
|
262
|
+
binding: 1,
|
|
263
|
+
visibility: GPUShaderStage.FRAGMENT,
|
|
264
|
+
sampler: { type: "filtering" },
|
|
265
|
+
},
|
|
266
|
+
],
|
|
267
|
+
});
|
|
268
|
+
const module = device.createShaderModule({
|
|
269
|
+
label: "kenburns_motionblur.wgsl",
|
|
270
|
+
code: motionBlurWgsl,
|
|
271
|
+
});
|
|
272
|
+
const motionBlurPipeline = device.createRenderPipeline({
|
|
273
|
+
label: "MotionBlurPipeline",
|
|
274
|
+
layout: device.createPipelineLayout({
|
|
275
|
+
bindGroupLayouts: [motionBlurUniformLayout, singleTextureLayout],
|
|
276
|
+
}),
|
|
277
|
+
vertex: { module, entryPoint: "vs" },
|
|
278
|
+
fragment: { module, entryPoint: "fs", targets: [{ format }] },
|
|
279
|
+
primitive: { topology: "triangle-strip" },
|
|
280
|
+
});
|
|
281
|
+
res = { motionBlurPipeline, motionBlurUniformLayout, singleTextureLayout };
|
|
282
|
+
deviceResourceCache.set(device, res);
|
|
283
|
+
}
|
|
284
|
+
return res;
|
|
285
|
+
}
|
|
286
|
+
|
|
287
|
+
export const KenBurnsWebGPURenderer: WebGPUNodeRenderer = async (args) => {
|
|
288
|
+
const {
|
|
289
|
+
ctx,
|
|
290
|
+
encoder,
|
|
291
|
+
pass,
|
|
292
|
+
targetView,
|
|
293
|
+
targetTexture,
|
|
294
|
+
targetWidth,
|
|
295
|
+
targetHeight,
|
|
296
|
+
props,
|
|
297
|
+
drawChild,
|
|
298
|
+
} = args;
|
|
299
|
+
|
|
300
|
+
const {
|
|
301
|
+
virtualMedia,
|
|
302
|
+
containerWidth,
|
|
303
|
+
containerHeight,
|
|
304
|
+
frame,
|
|
305
|
+
fps,
|
|
306
|
+
trimStartOverride,
|
|
307
|
+
opacity = 1,
|
|
308
|
+
inheritedSeekOffset = 0,
|
|
309
|
+
} = props;
|
|
310
|
+
|
|
311
|
+
const op = virtualMedia.operation as any;
|
|
312
|
+
if (op?.op !== "KenBurns") return;
|
|
313
|
+
|
|
314
|
+
const childMedia = virtualMedia.children?.[0];
|
|
315
|
+
if (!childMedia) return;
|
|
316
|
+
|
|
317
|
+
// ── Source dimensions ────────────────────────────────────────────────────
|
|
318
|
+
const sourceWidth =
|
|
319
|
+
op.originalWidth ??
|
|
320
|
+
childMedia.metadata?.width ??
|
|
321
|
+
virtualMedia.metadata?.width ??
|
|
322
|
+
containerWidth ??
|
|
323
|
+
targetWidth;
|
|
324
|
+
const sourceHeight =
|
|
325
|
+
op.originalHeight ??
|
|
326
|
+
childMedia.metadata?.height ??
|
|
327
|
+
virtualMedia.metadata?.height ??
|
|
328
|
+
containerHeight ??
|
|
329
|
+
targetHeight;
|
|
330
|
+
|
|
331
|
+
const keyframes = op.keyframes ?? [];
|
|
332
|
+
const movementStyle = op.movementStyle ?? "spline";
|
|
333
|
+
const targetAspectRatio = op.aspectRatio || "input";
|
|
334
|
+
|
|
335
|
+
// ── Interpolate Ken Burns state ──────────────────────────────────────────
|
|
336
|
+
const offsetMs = ((trimStartOverride ?? 0) + inheritedSeekOffset) * 1000;
|
|
337
|
+
const fpsVal = Math.max(1, fps ?? 30);
|
|
338
|
+
const frameDurationMs = 1000 / fpsVal;
|
|
339
|
+
const timeMs = offsetMs + ((frame ?? 0) / fpsVal) * 1000;
|
|
340
|
+
const prevTimeMs = timeMs - frameDurationMs;
|
|
341
|
+
|
|
342
|
+
const state = interpolateKenBurns(keyframes, timeMs, movementStyle);
|
|
343
|
+
const prevState = interpolateKenBurns(keyframes, prevTimeMs, movementStyle);
|
|
344
|
+
|
|
345
|
+
const srcRect = calculateSrcRect(
|
|
346
|
+
state,
|
|
347
|
+
sourceWidth,
|
|
348
|
+
sourceHeight,
|
|
349
|
+
targetAspectRatio,
|
|
350
|
+
);
|
|
351
|
+
const prevSrcRect = calculateSrcRect(
|
|
352
|
+
prevState,
|
|
353
|
+
sourceWidth,
|
|
354
|
+
sourceHeight,
|
|
355
|
+
targetAspectRatio,
|
|
356
|
+
);
|
|
357
|
+
const dstWidth = containerWidth ?? targetWidth;
|
|
358
|
+
const dstHeight = containerHeight ?? targetHeight;
|
|
359
|
+
const dstRect = { x: 0, y: 0, width: dstWidth, height: dstHeight };
|
|
360
|
+
|
|
361
|
+
// ── 5. Render child into a temporary offscreen texture ───────────────────
|
|
362
|
+
//
|
|
363
|
+
// ROOT CAUSE FIX: drawChild receives `matrix: new DOMMatrix()` which resets
|
|
364
|
+
// the renderer's transform stack internally, so pushTransform() is silently
|
|
365
|
+
// ignored by the child. We therefore render the child to a temp texture at
|
|
366
|
+
// its native size (no Ken Burns transform applied), then composite the
|
|
367
|
+
// cropped region ourselves using drawTextureRegion — which does not depend
|
|
368
|
+
// on the transform stack at all.
|
|
369
|
+
//
|
|
370
|
+
const tempTex = ctx.renderer.getTemporaryTexture(sourceWidth, sourceHeight, [
|
|
371
|
+
...(props.excludeTextures || []),
|
|
372
|
+
targetTexture,
|
|
373
|
+
]);
|
|
374
|
+
const tempView = tempTex.createView();
|
|
375
|
+
|
|
376
|
+
// End the current pass before starting child rendering.
|
|
377
|
+
pass.end();
|
|
378
|
+
|
|
379
|
+
// Clear the temporary texture to avoid garbage from previous frames
|
|
380
|
+
const clearPass = ctx.renderer.beginFrame(
|
|
381
|
+
encoder,
|
|
382
|
+
tempView,
|
|
383
|
+
{ r: 0, g: 0, b: 0, a: 0 },
|
|
384
|
+
sourceWidth,
|
|
385
|
+
sourceHeight,
|
|
386
|
+
"clear",
|
|
387
|
+
);
|
|
388
|
+
clearPass.end();
|
|
389
|
+
|
|
390
|
+
ctx.renderer.pushScissor({
|
|
391
|
+
x: 0,
|
|
392
|
+
y: 0,
|
|
393
|
+
width: sourceWidth,
|
|
394
|
+
height: sourceHeight,
|
|
395
|
+
});
|
|
396
|
+
ctx.renderer.pushIdentity();
|
|
397
|
+
|
|
398
|
+
await drawChild(
|
|
399
|
+
childMedia,
|
|
400
|
+
{
|
|
401
|
+
...props,
|
|
402
|
+
containerWidth: sourceWidth,
|
|
403
|
+
containerHeight: sourceHeight,
|
|
404
|
+
},
|
|
405
|
+
tempView,
|
|
406
|
+
tempTex,
|
|
407
|
+
sourceWidth,
|
|
408
|
+
sourceHeight,
|
|
409
|
+
);
|
|
410
|
+
|
|
411
|
+
ctx.renderer.popTransform();
|
|
412
|
+
ctx.renderer.popScissor();
|
|
413
|
+
|
|
414
|
+
const motionBlurSize = op.motionBlurSize ?? 1.5;
|
|
415
|
+
let blurTex: GPUTexture | null = null;
|
|
416
|
+
|
|
417
|
+
if (motionBlurSize > 0) {
|
|
418
|
+
const {
|
|
419
|
+
motionBlurPipeline: pipeline,
|
|
420
|
+
motionBlurUniformLayout: uLayout,
|
|
421
|
+
singleTextureLayout: tLayout,
|
|
422
|
+
} = getMotionBlurResources(ctx.device, ctx.renderer.format);
|
|
423
|
+
const sampler = ctx.renderer.samplerCache.getSampler(ctx.device);
|
|
424
|
+
|
|
425
|
+
motionBlurData[0] = prevSrcRect.x / sourceWidth;
|
|
426
|
+
motionBlurData[1] = prevSrcRect.y / sourceHeight;
|
|
427
|
+
motionBlurData[2] = prevSrcRect.width / sourceWidth;
|
|
428
|
+
motionBlurData[3] = prevSrcRect.height / sourceHeight;
|
|
429
|
+
motionBlurData[4] = srcRect.x / sourceWidth;
|
|
430
|
+
motionBlurData[5] = srcRect.y / sourceHeight;
|
|
431
|
+
motionBlurData[6] = srcRect.width / sourceWidth;
|
|
432
|
+
motionBlurData[7] = srcRect.height / sourceHeight;
|
|
433
|
+
motionBlurData[8] = 12; // samples
|
|
434
|
+
motionBlurData[9] = motionBlurSize;
|
|
435
|
+
|
|
436
|
+
const uBuffer = ctx.renderer.getTemporaryBuffer(motionBlurData);
|
|
437
|
+
|
|
438
|
+
// 1. Create a temporary texture for the blurred result at layer size
|
|
439
|
+
blurTex = ctx.renderer.getTemporaryTexture(dstWidth, dstHeight, [
|
|
440
|
+
targetTexture,
|
|
441
|
+
tempTex,
|
|
442
|
+
...(props.excludeTextures || []),
|
|
443
|
+
]);
|
|
444
|
+
const blurView = blurTex.createView();
|
|
445
|
+
|
|
446
|
+
// 2. Render blurred result into blurTex
|
|
447
|
+
const blurPass = encoder.beginRenderPass({
|
|
448
|
+
colorAttachments: [
|
|
449
|
+
{
|
|
450
|
+
view: blurView,
|
|
451
|
+
loadOp: "clear",
|
|
452
|
+
storeOp: "store",
|
|
453
|
+
clearValue: { r: 0, g: 0, b: 0, a: 0 },
|
|
454
|
+
},
|
|
455
|
+
],
|
|
456
|
+
});
|
|
457
|
+
|
|
458
|
+
blurPass.setPipeline(pipeline!);
|
|
459
|
+
blurPass.setBindGroup(
|
|
460
|
+
0,
|
|
461
|
+
ctx.device.createBindGroup({
|
|
462
|
+
layout: uLayout!,
|
|
463
|
+
entries: [{ binding: 0, resource: { buffer: uBuffer } }],
|
|
464
|
+
}),
|
|
465
|
+
);
|
|
466
|
+
blurPass.setBindGroup(
|
|
467
|
+
1,
|
|
468
|
+
ctx.renderer.bindGroupCache.getBindGroup(
|
|
469
|
+
ctx.device,
|
|
470
|
+
tLayout!,
|
|
471
|
+
tempTex,
|
|
472
|
+
sampler,
|
|
473
|
+
),
|
|
474
|
+
);
|
|
475
|
+
blurPass.draw(4);
|
|
476
|
+
blurPass.end();
|
|
477
|
+
}
|
|
478
|
+
|
|
479
|
+
// ── 6. Composite the cropped source onto the output ──────────────────────
|
|
480
|
+
// Begin a new pass on the real target, preserving any existing content.
|
|
481
|
+
const outPass = ctx.renderer.beginFrame(
|
|
482
|
+
encoder,
|
|
483
|
+
targetView,
|
|
484
|
+
{ r: 0, g: 0, b: 0, a: 0 },
|
|
485
|
+
targetWidth,
|
|
486
|
+
targetHeight,
|
|
487
|
+
"load",
|
|
488
|
+
);
|
|
489
|
+
|
|
490
|
+
if (blurTex) {
|
|
491
|
+
// 3. Draw the blurred result to the target (respecting parent transform)
|
|
492
|
+
ctx.renderer.drawTexture(outPass, blurTex, dstRect, { opacity });
|
|
493
|
+
} else {
|
|
494
|
+
ctx.renderer.drawTextureRegion(outPass, tempTex, srcRect, dstRect, {
|
|
495
|
+
opacity,
|
|
496
|
+
});
|
|
497
|
+
}
|
|
498
|
+
|
|
499
|
+
args.pass = outPass;
|
|
500
|
+
};
|
|
@@ -0,0 +1,107 @@
|
|
|
1
|
+
import {
|
|
2
|
+
createOutputItemSchema,
|
|
3
|
+
MediaMetadataSchema,
|
|
4
|
+
SingleOutputGenericSchema,
|
|
5
|
+
VirtualMediaDataSchema,
|
|
6
|
+
} from "@framefields/core";
|
|
7
|
+
import { z } from "zod";
|
|
8
|
+
|
|
9
|
+
export const KEN_BURNS_EASING_OPTIONS = [
|
|
10
|
+
"linear",
|
|
11
|
+
"ease-in",
|
|
12
|
+
"ease-out",
|
|
13
|
+
"ease-in-out",
|
|
14
|
+
"spring",
|
|
15
|
+
] as const;
|
|
16
|
+
|
|
17
|
+
export type KenBurnsEasing = (typeof KEN_BURNS_EASING_OPTIONS)[number];
|
|
18
|
+
|
|
19
|
+
export const KenBurnsKeyframeSchema = z.object({
|
|
20
|
+
id: z.string().min(1).optional(),
|
|
21
|
+
durationMs: z.number().int().min(0).max(20000),
|
|
22
|
+
holdMs: z.number().int().min(0).max(20000).optional().default(1000),
|
|
23
|
+
scale: z.number().min(1).max(20),
|
|
24
|
+
x: z.number().min(0).max(100),
|
|
25
|
+
y: z.number().min(0).max(100),
|
|
26
|
+
easing: z.enum(KEN_BURNS_EASING_OPTIONS).default("ease-in-out").optional(),
|
|
27
|
+
});
|
|
28
|
+
|
|
29
|
+
export type KenBurnsKeyframe = z.infer<typeof KenBurnsKeyframeSchema>;
|
|
30
|
+
|
|
31
|
+
export const KEN_BURNS_ASPECT_RATIOS = [
|
|
32
|
+
"16:9",
|
|
33
|
+
"9:16",
|
|
34
|
+
"21:9",
|
|
35
|
+
"9:21",
|
|
36
|
+
"1:1",
|
|
37
|
+
"4:3",
|
|
38
|
+
"3:2",
|
|
39
|
+
"2:3",
|
|
40
|
+
"4:5",
|
|
41
|
+
"5:4",
|
|
42
|
+
] as const;
|
|
43
|
+
|
|
44
|
+
export type KenBurnsAspectRatio = (typeof KEN_BURNS_ASPECT_RATIOS)[number];
|
|
45
|
+
|
|
46
|
+
export const KenBurnsConfigBaseSchema = z
|
|
47
|
+
.object({
|
|
48
|
+
keyframes: z
|
|
49
|
+
.array(KenBurnsKeyframeSchema)
|
|
50
|
+
.default([])
|
|
51
|
+
.superRefine((kfs, ctx) => {
|
|
52
|
+
if (kfs.length >= 2) {
|
|
53
|
+
for (let i = 0; i < kfs.length - 1; i++) {
|
|
54
|
+
if (kfs[i].durationMs <= 0) {
|
|
55
|
+
ctx.addIssue({
|
|
56
|
+
code: z.ZodIssueCode.custom,
|
|
57
|
+
path: [i, "durationMs"],
|
|
58
|
+
message:
|
|
59
|
+
"Intermediate keyframes must have a duration greater than 0",
|
|
60
|
+
});
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
}),
|
|
65
|
+
motionBlurSize: z.number().min(0).max(10).optional().default(1.5),
|
|
66
|
+
movementStyle: z.enum(["spline", "direct"]).default("spline"),
|
|
67
|
+
aspectRatio: z.enum(["input", ...KEN_BURNS_ASPECT_RATIOS]).default("input"),
|
|
68
|
+
})
|
|
69
|
+
.strict();
|
|
70
|
+
|
|
71
|
+
const transformLastKeyframe = <T extends { keyframes: KenBurnsKeyframe[] }>(
|
|
72
|
+
data: T,
|
|
73
|
+
): T => {
|
|
74
|
+
if (data.keyframes.length === 0) return data;
|
|
75
|
+
const keyframes = [...data.keyframes];
|
|
76
|
+
const lastIdx = keyframes.length - 1;
|
|
77
|
+
keyframes[lastIdx] = {
|
|
78
|
+
...keyframes[lastIdx],
|
|
79
|
+
durationMs: 0,
|
|
80
|
+
};
|
|
81
|
+
return {
|
|
82
|
+
...data,
|
|
83
|
+
keyframes,
|
|
84
|
+
};
|
|
85
|
+
};
|
|
86
|
+
|
|
87
|
+
export const KenBurnsConfigSchema = KenBurnsConfigBaseSchema.transform(
|
|
88
|
+
transformLastKeyframe,
|
|
89
|
+
);
|
|
90
|
+
|
|
91
|
+
export type KenBurnsConfig = z.infer<typeof KenBurnsConfigSchema>;
|
|
92
|
+
|
|
93
|
+
export const KenBurnsOperationSchema = KenBurnsConfigBaseSchema.extend({
|
|
94
|
+
op: z.literal("KenBurns"),
|
|
95
|
+
metadata: MediaMetadataSchema.optional(),
|
|
96
|
+
originalWidth: z.number().int().positive().optional(),
|
|
97
|
+
originalHeight: z.number().int().positive().optional(),
|
|
98
|
+
}).transform(transformLastKeyframe);
|
|
99
|
+
|
|
100
|
+
export type KenBurnsOperation = z.infer<typeof KenBurnsOperationSchema>;
|
|
101
|
+
|
|
102
|
+
// Ken Burns always outputs a Video because it applies animation over time
|
|
103
|
+
export const KenBurnsResultSchema = SingleOutputGenericSchema(
|
|
104
|
+
createOutputItemSchema(z.literal("Video"), VirtualMediaDataSchema),
|
|
105
|
+
);
|
|
106
|
+
|
|
107
|
+
export type KenBurnsResult = z.infer<typeof KenBurnsResultSchema>;
|
|
@@ -0,0 +1,108 @@
|
|
|
1
|
+
import type {
|
|
2
|
+
DataType,
|
|
3
|
+
MediaMetadata,
|
|
4
|
+
MediaOperation,
|
|
5
|
+
VirtualMediaData,
|
|
6
|
+
} from "@framefields/core";
|
|
7
|
+
import type { KenBurnsConfig } from "./config.js";
|
|
8
|
+
|
|
9
|
+
/**
|
|
10
|
+
* Calculates the output metadata for a Ken Burns operation based on current metadata and config.
|
|
11
|
+
* Ensures consistent behavior between browser and server processors.
|
|
12
|
+
*/
|
|
13
|
+
export function calculateKenBurnsMetadata(
|
|
14
|
+
currentMeta: MediaMetadata,
|
|
15
|
+
config: KenBurnsConfig,
|
|
16
|
+
inputType: DataType = "Image",
|
|
17
|
+
): MediaMetadata {
|
|
18
|
+
let outWidth = currentMeta.width;
|
|
19
|
+
let outHeight = currentMeta.height;
|
|
20
|
+
if (!outWidth || !outHeight) {
|
|
21
|
+
throw new Error("Dimensions are missing from input media.");
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
// Crop bounding box to fit the new aspect ratio
|
|
25
|
+
if (config.aspectRatio && config.aspectRatio !== "input") {
|
|
26
|
+
const [wRatio, hRatio] = config.aspectRatio.split(":").map(Number);
|
|
27
|
+
const targetAr = wRatio / hRatio;
|
|
28
|
+
const sourceAr = outWidth / outHeight;
|
|
29
|
+
|
|
30
|
+
if (sourceAr > targetAr) {
|
|
31
|
+
// Source is wider, bind output to height
|
|
32
|
+
outWidth = Math.round(outHeight * targetAr);
|
|
33
|
+
} else {
|
|
34
|
+
// Source is taller, bind output to width
|
|
35
|
+
outHeight = Math.round(outWidth / targetAr);
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
// Calculate total chain duration (minimum 100ms)
|
|
40
|
+
let totalDuration = currentMeta.durationMs ?? 3000;
|
|
41
|
+
if (config.keyframes && config.keyframes.length > 0) {
|
|
42
|
+
const keyframesDuration = config.keyframes.reduce(
|
|
43
|
+
(acc, kf) => acc + kf.durationMs + (kf.holdMs || 0),
|
|
44
|
+
0,
|
|
45
|
+
);
|
|
46
|
+
|
|
47
|
+
if (inputType === "Image" || inputType === "SVG") {
|
|
48
|
+
totalDuration = Math.max(100, keyframesDuration);
|
|
49
|
+
} else {
|
|
50
|
+
// For videos/audio, we cap the animation duration to the available content duration
|
|
51
|
+
totalDuration = Math.min(totalDuration, Math.max(100, keyframesDuration));
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
let fps = currentMeta.fps;
|
|
56
|
+
if (inputType === "Image" || inputType === "SVG" || inputType === "Lottie") {
|
|
57
|
+
fps = 60;
|
|
58
|
+
} else if (!fps) {
|
|
59
|
+
fps = 24; // fallback if missing
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
return {
|
|
63
|
+
...currentMeta,
|
|
64
|
+
width: Math.round(outWidth),
|
|
65
|
+
height: Math.round(outHeight),
|
|
66
|
+
durationMs: totalDuration,
|
|
67
|
+
fps,
|
|
68
|
+
};
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
/**
|
|
72
|
+
* Environment-specific functions needed for Ken Burns processing.
|
|
73
|
+
*/
|
|
74
|
+
export interface KenBurnsProcessorEnv {
|
|
75
|
+
getActiveMediaMetadata: (media: VirtualMediaData) => MediaMetadata | null;
|
|
76
|
+
appendOperation: (
|
|
77
|
+
media: VirtualMediaData,
|
|
78
|
+
operation: MediaOperation,
|
|
79
|
+
) => VirtualMediaData;
|
|
80
|
+
getMediaType: (media: VirtualMediaData) => DataType;
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
/**
|
|
84
|
+
* Performs the core Ken Burns processing logic, abstracting away environment-specific function calls.
|
|
85
|
+
*/
|
|
86
|
+
export function performKenBurnsProcessing(
|
|
87
|
+
inputMedia: VirtualMediaData,
|
|
88
|
+
config: KenBurnsConfig,
|
|
89
|
+
env: KenBurnsProcessorEnv,
|
|
90
|
+
): VirtualMediaData {
|
|
91
|
+
const currentMeta = env.getActiveMediaMetadata(inputMedia);
|
|
92
|
+
if (!currentMeta) {
|
|
93
|
+
throw new Error("Unable to read media metadata");
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
const mediaType = env.getMediaType(inputMedia);
|
|
97
|
+
const newMeta = calculateKenBurnsMetadata(currentMeta, config, mediaType);
|
|
98
|
+
|
|
99
|
+
// Append the Ken Burns operation to the media tree with calculated metadata
|
|
100
|
+
return env.appendOperation(inputMedia, {
|
|
101
|
+
op: "KenBurns",
|
|
102
|
+
...config,
|
|
103
|
+
originalWidth: currentMeta.width,
|
|
104
|
+
originalHeight: currentMeta.height,
|
|
105
|
+
metadata: newMeta,
|
|
106
|
+
dataType: "Video",
|
|
107
|
+
});
|
|
108
|
+
}
|