@framefields/node-compositor 0.0.0-stage → 2.0.3

package/package.json

@@ -1,6 +1,54 @@
 {
   "name": "@framefields/node-compositor",
-  "version": "0.0.0-stage",
-  "stub": true,
-  "description": "Temporary package placeholder for staged publishing"
+  "version": "2.0.3",
+  "private": false,
+  "framefields": {
+    "type": "Compositor",
+    "kind": "utility"
+  },
+  "type": "module",
+  "license": "Apache-2.0",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.mts",
+      "development": "./src/index.ts",
+      "import": "./dist/index.mjs",
+      "default": "./dist/index.mjs"
+    },
+    "./renderer": {
+      "types": "./dist/renderer.d.mts",
+      "development": "./src/renderers/index.ts",
+      "import": "./dist/renderer.mjs",
+      "default": "./dist/renderer.mjs"
+    }
+  },
+  "dependencies": {
+    "gsap": "^3.15.0",
+    "zod": "^4.3.6",
+    "@framefields/core": "2.0.3",
+    "@framefields/node-sdk": "2.0.3",
+    "@framefields/client-utils": "2.0.3",
+    "@framefields/tensor-webgpu": "2.0.3",
+    "@framefields/renderers": "2.0.3",
+    "@framefields/webgpu-renderers": "2.0.3",
+    "@framefields/compositions": "2.0.3"
+  },
+  "devDependencies": {
+    "@types/node": "^24.1.0",
+    "tsdown": "^0.19.0",
+    "typescript": "^5.8.3",
+    "vitest": "^4.1.5",
+    "webgpu": "^0.4.0",
+    "@framefields/tsconfig": "2.0.3"
+  },
+  "publishConfig": {
+    "access": "public"
+  },
+  "scripts": {
+    "build": "tsdown",
+    "dev": "tsdown --watch",
+    "clean": "rm -rf dist .turbo",
+    "test": "vitest run --passWithNoTests",
+    "typecheck": "tsc --noEmit"
+  }
 }

package/src/index.ts

@@ -0,0 +1,2 @@
+export * from "./shared/index.js";
+export * from "./renderers/index.js";

package/src/renderers/audio-processor.ts

@@ -0,0 +1,123 @@
+import type { LayerAnimation } from "@framefields/compositions/program";
+import type { VirtualMediaData } from "@framefields/core";
+import type { AudioProcessor } from "@framefields/node-sdk";
+import { audioRegistry } from "@framefields/node-sdk";
+import { compileLayerTimeline } from "../shared/compiler.js";
+
+interface CompositorOperation {
+	op: "Compositor" | "CompositorLayer";
+	id?: string;
+	inputHandleId?: string;
+	volume?: number;
+	muted?: boolean;
+	animation?: LayerAnimation;
+}
+
+export const compositorLayerAudioProcessor: AudioProcessor = async (
+	channels: Float32Array[],
+	sampleRate: number,
+	virtualMedia: VirtualMediaData,
+	ctx,
+) => {
+	const op = virtualMedia.operation as unknown as CompositorOperation;
+	if (!op) return;
+
+	const numChannels = channels.length;
+	if (numChannels === 0) return;
+	const numSamples = channels[0].length;
+	if (numSamples === 0) return;
+
+	// Master gain (L1): the audio extractor dispatches this processor for the
+	// root Compositor op too (registered under the node type "Compositor").
+	// The root's channels ARE the fully-mixed composition, so scale them
+	// once by the program-level `volume` — the per-layer path below handles
+	// each CompositorLayer op's own volume/mute animation.
+	if (op.op === "Compositor") {
+		const master = typeof op.volume === "number" ? op.volume : 1;
+		if (master === 1) return;
+		for (let c = 0; c < numChannels; c++) {
+			const ch = channels[c];
+			for (let i = 0; i < ch.length; i++) ch[i] *= master;
+		}
+		return;
+	}
+
+	const volume = typeof op.volume === "number" ? op.volume : 1;
+	const muted = !!op.muted;
+	const animation = op.animation;
+	const fps = ctx?.fps ?? 24;
+
+	// Check if there are active volume or muted animation tracks with keyframes
+	const tracks = animation?.tracks || [];
+	const hasVolumeKeys = tracks.some(
+		(t) => t.prop === "volume" && t.keyframes && t.keyframes.length > 0,
+	);
+	const hasMutedKeys = tracks.some(
+		(t) => t.prop === "muted" && t.keyframes && t.keyframes.length > 0,
+	);
+
+	if (!hasVolumeKeys && !hasMutedKeys) {
+		// If no animated volume/muted tracks exist, the static volume and mute state
+		// have already been applied by the child clip or audio extractor.
+		return;
+	}
+
+	// Compile or retrieve the GSAP timeline for this layer
+	const { tl, stub } = compileLayerTimeline(
+		op.id || op.inputHandleId || "layer",
+		animation,
+		volume,
+		muted,
+		fps,
+	);
+
+	const elapsedMs = ctx?.elapsedMs ?? 0;
+	const startTimeSec = elapsedMs / 1000;
+
+	// Process in sub-chunks of 128 samples to be fast and smooth
+	const subChunkSize = 128;
+
+	// Seek to the starting time to establish baseline values
+	tl.seek(startTimeSec);
+	let lastVol = stub.volume;
+	let lastMuted = stub.muted;
+
+	const baseGain = muted ? 0 : volume;
+
+	for (let i = 0; i < numSamples; i += subChunkSize) {
+		const nextI = Math.min(numSamples, i + subChunkSize);
+		const tNext = startTimeSec + (nextI - i) / sampleRate;
+		tl.seek(tNext);
+		const nextVol = stub.volume;
+		const nextMuted = stub.muted;
+
+		for (let j = i; j < nextI; j++) {
+			const p = (j - i) / (nextI - i);
+			// L3: clamp — keyframes are schema-bounded to 0–1, but a lerp
+			// between in-range values can overshoot with custom eases; the
+			// relativeGain compensation below must never amplify past the
+			// channel's own scale.
+			const currentVol = Math.min(
+				1,
+				Math.max(0, lastVol + (nextVol - lastVol) * p),
+			);
+			// For muted, if it changes mid-sub-chunk, toggle it at the halfway point
+			const currentMuted = j - i < (nextI - i) / 2 ? lastMuted : nextMuted;
+
+			const finalGain = currentMuted ? 0 : currentVol;
+			const relativeGain = baseGain === 0 ? 0 : finalGain / baseGain;
+
+			if (relativeGain !== 1) {
+				for (let c = 0; c < numChannels; c++) {
+					channels[c][j] *= relativeGain;
+				}
+			}
+		}
+
+		lastVol = nextVol;
+		lastMuted = nextMuted;
+	}
+};
+
+// Register immediately on load
+audioRegistry.register("CompositorLayer", compositorLayerAudioProcessor);

package/src/renderers/container-bg.test.ts

@@ -0,0 +1,22 @@
+import { describe, expect, it } from "vitest";
+import { shouldPaintContainerBackground } from "./container-bg.js";
+
+describe("shouldPaintContainerBackground", () => {
+	it("paints every container kind when a background is set", () => {
+		for (const kind of ["flex", "block", "box"]) {
+			expect(shouldPaintContainerBackground(kind, "#3a2f1e")).toBe(true);
+		}
+	});
+
+	it("never paints when there is no background", () => {
+		expect(shouldPaintContainerBackground("flex", undefined)).toBe(false);
+		expect(shouldPaintContainerBackground("box", null)).toBe(false);
+		expect(shouldPaintContainerBackground("box", "")).toBe(false);
+	});
+
+	it("does not paint non-container kinds", () => {
+		for (const kind of ["text", "media", undefined]) {
+			expect(shouldPaintContainerBackground(kind, "#ffffff")).toBe(false);
+		}
+	});
+});

package/src/renderers/container-bg.ts

@@ -0,0 +1,15 @@
+/**
+ * Which container kinds paint their own background.
+ *
+ * flex/block/box all accept an optional `background` in the program schema.
+ * The renderer must draw the rounded background pass for ALL of them (the old
+ * code only painted `box`, so flex/block backgrounds were silently dropped and
+ * their "transparent-optimized" fast path skipped the fill entirely).
+ */
+export function shouldPaintContainerBackground(
+	kind: string | undefined,
+	background?: string | null,
+): boolean {
+	if (!background) return false;
+	return kind === "flex" || kind === "block" || kind === "box";
+}

package/src/renderers/index.ts

@@ -0,0 +1,9 @@
+import { defineRenderer } from "@framefields/node-sdk/renderer";
+import { CompositorWebGPURenderer } from "./webgpu-renderer.js";
+import "./audio-processor.js";
+
+export default defineRenderer({
+	WebGPURenderer: CompositorWebGPURenderer,
+});
+
+export * from "./transform3d.js";

package/src/renderers/transform.test.ts

@@ -0,0 +1,120 @@
+import { describe, expect, it } from "vitest";
+import {
+	buildLayerMatrix,
+	type Matrix2D,
+	multiply,
+	rotate,
+	scale,
+	transformPoint,
+	translate,
+} from "./transform.js";
+
+const closeTo = (actual: number, expected: number, eps = 1e-9) =>
+	expect(Math.abs(actual - expected)).toBeLessThan(eps);
+
+const expectPoint = (
+	a: { x: number; y: number },
+	e: { x: number; y: number },
+) => {
+	closeTo(a.x, e.x);
+	closeTo(a.y, e.y);
+};
+
+describe("transform (pure affine)", () => {
+	it("composes translations in apply-order (m1 ∘ m2)", () => {
+		const m = multiply(translate(3, 4), translate(5, 6));
+		expectPoint(transformPoint(m, 0, 0), { x: 8, y: 10 });
+	});
+
+	it("rotate is clockwise in screen space (y-down)", () => {
+		expectPoint(transformPoint(rotate(90), 1, 0), { x: 0, y: 1 });
+		expectPoint(transformPoint(rotate(90), 0, 1), { x: -1, y: 0 });
+	});
+
+	it("scale multiplies and can mirror through negative x", () => {
+		expectPoint(transformPoint(scale(2), 3, 4), { x: 6, y: 8 });
+	});
+});
+
+describe("buildLayerMatrix — anchor-aware pivot (review H2)", () => {
+	const box = { x: 10, y: 20, width: 200, height: 100 };
+
+	it("is a plain placement when scale=1 and rotation=0 (any anchor)", () => {
+		for (const anchorX of [0, 0.5, 1]) {
+			for (const anchorY of [0, 0.5, 1]) {
+				const m = buildLayerMatrix({
+					...box,
+					rotation: 0,
+					scale: 1,
+					anchorX,
+					anchorY,
+				});
+				expectPoint(transformPoint(m, 0, 0), { x: 10, y: 20 });
+				expectPoint(transformPoint(m, 200, 100), { x: 210, y: 120 });
+			}
+		}
+	});
+
+	it("keeps the default center anchor (0.5/0.5) invariant under scale", () => {
+		const m = buildLayerMatrix({ ...box, rotation: 0, scale: 2 });
+		// center of the box
+		expectPoint(transformPoint(m, 100, 50), { x: 110, y: 70 });
+		// top-left scales AWAY from the center (the H2 fix)
+		expectPoint(transformPoint(m, 0, 0), { x: -90, y: -30 });
+		// top-right scales away symmetrically: x + w·s − ax·w = 10 + 400 − 100
+		expectPoint(transformPoint(m, 200, 0), { x: 310, y: -30 });
+	});
+
+	it("keeps the center anchor invariant under rotation", () => {
+		const m = buildLayerMatrix({ ...box, rotation: 45, scale: 1 });
+		expectPoint(transformPoint(m, 100, 50), { x: 110, y: 70 });
+	});
+
+	it("keeps ANY anchor point invariant under the full transform (scale+rotation)", () => {
+		for (const anchorX of [0, 0.25, 0.5, 0.75, 1]) {
+			for (const anchorY of [0, 0.25, 0.5, 0.75, 1]) {
+				const anchorXAbs = anchorX * box.width;
+				const anchorYAbs = anchorY * box.height;
+				const m = buildLayerMatrix({
+					...box,
+					rotation: 33,
+					scale: 1.7,
+					anchorX,
+					anchorY,
+				});
+				expectPoint(transformPoint(m, anchorXAbs, anchorYAbs), {
+					x: box.x + anchorXAbs,
+					y: box.y + anchorYAbs,
+				});
+			}
+		}
+	});
+
+	it("anchor (0,0) reproduces the legacy top-left pivot exactly", () => {
+		const m = buildLayerMatrix({
+			...box,
+			rotation: 90,
+			scale: 2,
+			anchorX: 0,
+			anchorY: 0,
+		});
+		// top-left corner stays put; the rest scales/rotates around it
+		expectPoint(transformPoint(m, 0, 0), { x: 10, y: 20 });
+		// right-middle: (200,50) → rotate 90° clockwise around (0,0): (x,y)→(-50·? …)
+		// y' = x = 200 → scaled 2 → (0 - 50·2, 20 + 200·2) = (-100, 420)
+		expectPoint(transformPoint(m, 200, 50), {
+			x: 10 - 50 * 2,
+			y: 20 + 200 * 2,
+		});
+	});
+
+	it("does not corrupt the matrix type contract used by the renderer (DOMMatrix mapping)", () => {
+		const m: Matrix2D = buildLayerMatrix({ ...box, rotation: 0, scale: 1 });
+		// DOMMatrix([a,b,c,d,e,f]) maps (x,y) → (a·x + c·y + e, b·x + d·y + f)
+		const mapped = {
+			x: m.a * 100 + m.c * 50 + m.e,
+			y: m.b * 100 + m.d * 50 + m.f,
+		};
+		expectPoint(mapped, { x: 110, y: 70 });
+	});
+});

package/src/renderers/transform.ts

@@ -0,0 +1,101 @@
+/**
+ * Minimal pure 2D affine matrix (same 6-number layout as DOMMatrix:
+ * [a, b, c, d, e, f] with x' = a·x + c·y + e, y' = b·x + d·y + f).
+ *
+ * DOM-free on purpose — WebGPU node renderers build DOMMatrix chains before
+ * pushing transforms, but DOMMatrix is not available in Node (unit tests /
+ * headless layout), so the anchor math that decides WHERE a layer pivots is
+ * implemented here as pure functions and only materialized into a DOMMatrix
+ * at the render boundary via `toDOMMatrix`.
+ */
+export interface Matrix2D {
+	a: number;
+	b: number;
+	c: number;
+	d: number;
+	e: number;
+	f: number;
+}
+
+export const IDENTITY: Matrix2D = { a: 1, b: 0, c: 0, d: 1, e: 0, f: 0 };
+
+export function translate(tx: number, ty: number): Matrix2D {
+	return { a: 1, b: 0, c: 0, d: 1, e: tx, f: ty };
+}
+
+/** Rotation in degrees (screen convention: positive = clockwise, y-down). */
+export function rotate(degrees: number): Matrix2D {
+	const rad = (degrees * Math.PI) / 180;
+	const cos = Math.cos(rad);
+	const sin = Math.sin(rad);
+	return { a: cos, b: sin, c: -sin, d: cos, e: 0, f: 0 };
+}
+
+export function scale(sx: number, sy: number = sx): Matrix2D {
+	return { a: sx, b: 0, c: 0, d: sy, e: 0, f: 0 };
+}
+
+/** m1 ∘ m2 (m2 applied first — matches DOMMatrix.multiply ordering). */
+export function multiply(m1: Matrix2D, m2: Matrix2D): Matrix2D {
+	return {
+		a: m1.a * m2.a + m1.c * m2.b,
+		b: m1.b * m2.a + m1.d * m2.b,
+		c: m1.a * m2.c + m1.c * m2.d,
+		d: m1.b * m2.c + m1.d * m2.d,
+		e: m1.a * m2.e + m1.c * m2.f + m1.e,
+		f: m1.b * m2.e + m1.d * m2.f + m1.f,
+	};
+}
+
+export function transformPoint(
+	m: Matrix2D,
+	x: number,
+	y: number,
+): { x: number; y: number } {
+	return { x: m.a * x + m.c * y + m.e, y: m.b * x + m.d * y + m.f };
+}
+
+/** Materialize into a DOMMatrix for the renderer's transform stack. */
+export function toDOMMatrix(m: Matrix2D): DOMMatrix {
+	return new DOMMatrix([m.a, m.b, m.c, m.d, m.e, m.f]);
+}
+
+/**
+ * Layer transform matrix with anchor-aware pivot semantics.
+ *
+ * The composition node box is drawn at (0, 0, width, height) in local space
+ * and positioned at `x`/`y`. `anchorX`/`anchorY` (0–1 fractions of the box)
+ * select the pivot point that scale/rotation are applied AROUND:
+ *
+ *   position → pivot → rotate → scale → un-pivot
+ *
+ * With the default center anchor (0.5, 0.5) a `scale: 2` animation grows the
+ * node symmetrically from its middle instead of from its top-left corner.
+ * The pivot point itself is invariant under rotation and scale.
+ */
+export function buildLayerMatrix(params: {
+	x: number;
+	y: number;
+	width: number;
+	height: number;
+	rotation: number;
+	scale: number;
+	anchorX?: number;
+	anchorY?: number;
+}): Matrix2D {
+	const ax = params.anchorX ?? 0.5;
+	const ay = params.anchorY ?? 0.5;
+	const px = ax * params.width;
+	const py = ay * params.height;
+
+	return multiply(
+		translate(params.x, params.y),
+		multiply(
+			translate(px, py),
+			multiply(
+				rotate(params.rotation),
+				multiply(scale(params.scale), translate(-px, -py)),
+			),
+		),
+	);
+}