@elah/core 0.1.0

package/dist/renderer/gpu/layers/VideoLayer.js

@@ -0,0 +1,194 @@
+import { ShaderProgram } from '../ShaderProgram';
+import { QUAD_FRAG_SRC } from '../shaders/quad.frag';
+import { QUAD_VERT_SRC } from '../shaders/quad.vert';
+import { VideoTexture } from '../VideoTexture';
+import { createVideoFrameProvider, } from '../../../media/video';
+import { buildDrawTransformMatrix } from './drawRect';
+export { buildTransformMatrixFromRect, resolveTransformRect, resolveDrawRect, buildDrawTransformMatrix, } from './drawRect';
+export function buildVideoTransformMatrix(item, stageWidth, stageHeight, contentWidth, contentHeight) {
+    return buildDrawTransformMatrix(item.transform, stageWidth, stageHeight, contentWidth, contentHeight);
+}
+export class VideoLayer {
+    constructor(pool, providerFactoryOrDeps) {
+        this._program = null;
+        this._vao = null;
+        this._gl = null;
+        this._providers = new Map();
+        this._textures = new Map();
+        this._srcByItemId = new Map();
+        this._contentSizeByItemId = new Map();
+        this._lastLoggedSourceFrameByItemId = new Map();
+        this._holdoverTexture = null;
+        this._pool = pool;
+        if (typeof providerFactoryOrDeps === 'function') {
+            this._providerFactory = providerFactoryOrDeps;
+            this._deps = undefined;
+        }
+        else {
+            this._deps = providerFactoryOrDeps;
+            this._providerFactory = (src) => createVideoFrameProvider(src, this._deps);
+        }
+    }
+    acquire(item, ctx) {
+        const { gl } = ctx;
+        this._gl = gl;
+        this._ensurePipeline(gl);
+        this._textures.set(item.id, new VideoTexture(this._pool));
+        this._srcByItemId.set(item.id, item.src);
+        let entry = this._providers.get(item.id);
+        if (!entry) {
+            const provider = this._deps
+                ? createVideoFrameProvider(item.src, { ...this._deps, fps: ctx.fps })
+                : this._providerFactory(item.src);
+            entry = { provider, refCount: 0 };
+            this._providers.set(item.id, entry);
+        }
+        entry.refCount++;
+        entry.provider.markActive();
+    }
+    release(itemId) {
+        const src = this._srcByItemId.get(itemId);
+        if (!src)
+            return;
+        const texture = this._textures.get(itemId);
+        if (texture?.hasContent) {
+            this._holdoverTexture?.dispose();
+            this._holdoverTexture = texture;
+        }
+        else {
+            texture?.dispose();
+        }
+        this._textures.delete(itemId);
+        this._srcByItemId.delete(itemId);
+        this._contentSizeByItemId.delete(itemId);
+        this._lastLoggedSourceFrameByItemId.delete(itemId);
+        const entry = this._providers.get(itemId);
+        if (!entry)
+            return;
+        entry.refCount--;
+        if (entry.refCount <= 0) {
+            entry.refCount = 0;
+            entry.provider.markIdle();
+        }
+    }
+    draw(item, ctx) {
+        this._gl = ctx.gl;
+        const texture = this._textures.get(item.id);
+        const entry = this._providers.get(item.id);
+        if (!texture || !entry)
+            return;
+        const { provider } = entry;
+        provider.setPlayhead(item.sourceFrame);
+        const frame = provider.getCurrent(item.sourceFrame);
+        if (frame !== null) {
+            const uploaded = texture.upload(ctx.gl, frame);
+            if (uploaded) {
+                const width = 'displayWidth' in frame ? frame.displayWidth : frame.width;
+                const height = 'displayHeight' in frame ? frame.displayHeight : frame.height;
+                this._contentSizeByItemId.set(item.id, { width, height });
+                this._holdoverTexture?.dispose();
+                this._holdoverTexture = null;
+            }
+        }
+        if (!this._program || !this._vao)
+            return;
+        const contentSize = this._contentSizeByItemId.get(item.id);
+        const opacity = item.opacity ?? 1;
+        this._program.use(ctx.gl);
+        ctx.gl.bindVertexArray(this._vao);
+        let unit = texture.bind(ctx.gl, 0);
+        if (unit < 0 && this._holdoverTexture !== null) {
+            unit = this._holdoverTexture.bind(ctx.gl, 0);
+        }
+        if (unit < 0)
+            return;
+        this._program.setUniform1i(ctx.gl, 'uTexture', unit);
+        this._program.setUniform1f(ctx.gl, 'uOpacity', opacity);
+        this._program.setUniformMatrix3fv(ctx.gl, 'uTransform', false, buildVideoTransformMatrix(item, ctx.stage.width, ctx.stage.height, contentSize?.width, contentSize?.height));
+        ctx.gl.drawArrays(ctx.gl.TRIANGLE_STRIP, 0, 4);
+        ctx.gl.bindTexture(ctx.gl.TEXTURE_2D, null);
+        ctx.gl.bindVertexArray(null);
+    }
+    dispose() {
+        for (const texture of this._textures.values()) {
+            texture.dispose();
+        }
+        this._textures.clear();
+        this._srcByItemId.clear();
+        this._contentSizeByItemId.clear();
+        this._holdoverTexture?.dispose();
+        this._holdoverTexture = null;
+        for (const entry of this._providers.values()) {
+            entry.provider.dispose();
+        }
+        this._providers.clear();
+        if (this._gl) {
+            if (this._vao)
+                this._gl.deleteVertexArray(this._vao);
+            if (this._program)
+                this._program.dispose(this._gl);
+        }
+        this._program = null;
+        this._vao = null;
+        this._gl = null;
+    }
+    notifyContextLost() {
+        for (const texture of this._textures.values()) {
+            texture.handleContextLost();
+        }
+        this._holdoverTexture?.handleContextLost();
+        this._holdoverTexture = null;
+        this._program = null;
+        this._vao = null;
+        this._gl = null;
+    }
+    disposeGL(gl) {
+        for (const texture of this._textures.values()) {
+            texture.dispose();
+        }
+        if (this._vao)
+            gl.deleteVertexArray(this._vao);
+        if (this._program)
+            this._program.dispose(gl);
+        this.dispose();
+    }
+    getProviderForItemId(itemId) {
+        return this._providers.get(itemId)?.provider;
+    }
+    getProviderForSrc(src) {
+        for (const [itemId, itemSrc] of this._srcByItemId) {
+            if (itemSrc === src)
+                return this._providers.get(itemId)?.provider;
+        }
+        return undefined;
+    }
+    getProviderRefCount(itemId) {
+        return this._providers.get(itemId)?.refCount ?? 0;
+    }
+    getTextureCount() {
+        return this._textures.size;
+    }
+    getProviderCount() {
+        return this._providers.size;
+    }
+    getDecoderStates() {
+        const out = {};
+        for (const [src, entry] of this._providers) {
+            const provider = entry.provider;
+            if (typeof provider.decoderState === 'string') {
+                out[src] = provider.decoderState;
+            }
+        }
+        return out;
+    }
+    _ensurePipeline(gl) {
+        if (this._program && this._vao)
+            return;
+        this._program = ShaderProgram.create(gl, QUAD_VERT_SRC, QUAD_FRAG_SRC);
+        const vao = gl.createVertexArray();
+        if (!vao) {
+            throw new Error('VideoLayer: gl.createVertexArray() returned null');
+        }
+        this._vao = vao;
+    }
+}

package/dist/renderer/gpu/layers/drawRect.d.ts

@@ -0,0 +1,13 @@
+import type { Transform } from '../../../types';
+export interface DrawRect {
+    x: number;
+    y: number;
+    width: number;
+    height: number;
+    rotation: number;
+}
+export declare function buildTransformMatrixFromRect(rect: DrawRect, stageWidth: number, stageHeight: number): Float32Array;
+export declare function resolveTransformRect(transform: Transform, stageWidth: number, stageHeight: number, contentWidth: number, contentHeight: number): DrawRect;
+export declare function resolveDrawRect(transform: Transform | undefined, stageWidth: number, stageHeight: number, contentWidth?: number, contentHeight?: number): DrawRect;
+export declare function transformFromContainRect(contentWidth: number, contentHeight: number, stageWidth: number, stageHeight: number): Transform;
+export declare function buildDrawTransformMatrix(transform: Transform | undefined, stageWidth: number, stageHeight: number, contentWidth?: number, contentHeight?: number): Float32Array;

package/dist/renderer/gpu/layers/drawRect.js

@@ -0,0 +1,55 @@
+import { computeContainRect } from './objectFit';
+export function buildTransformMatrixFromRect(rect, stageWidth, stageHeight) {
+    const xs = rect.x / stageWidth;
+    const ys = rect.y / stageHeight;
+    const ws = rect.width / stageWidth;
+    const hs = rect.height / stageHeight;
+    const rotation = rect.rotation;
+    const sc = Math.cos(rotation);
+    const ss = Math.sin(rotation);
+    const tx = 2 * (xs + ws / 2 - 0.5 * sc * ws + 0.5 * ss * hs) - 1;
+    const ty = 2 * (ys + hs / 2 - 0.5 * ss * ws - 0.5 * sc * hs) - 1;
+    return new Float32Array([
+        2 * sc * ws, 2 * ss * ws, 0,
+        -2 * ss * hs, 2 * sc * hs, 0,
+        tx, ty, 1,
+    ]);
+}
+export function resolveTransformRect(transform, stageWidth, stageHeight, contentWidth, contentHeight) {
+    const width = contentWidth * transform.scale;
+    const height = contentHeight * transform.scale;
+    const anchorPxX = transform.anchor.x * width;
+    const anchorPxY = transform.anchor.y * height;
+    const x = transform.x * stageWidth - anchorPxX;
+    const y = transform.y * stageHeight - anchorPxY;
+    return {
+        x,
+        y,
+        width,
+        height,
+        rotation: transform.rotation,
+    };
+}
+export function resolveDrawRect(transform, stageWidth, stageHeight, contentWidth, contentHeight) {
+    if (!transform) {
+        if (contentWidth && contentHeight) {
+            const fit = computeContainRect(contentWidth, contentHeight, stageWidth, stageHeight);
+            return { x: fit.x, y: fit.y, width: fit.width, height: fit.height, rotation: 0 };
+        }
+        return { x: 0, y: 0, width: stageWidth, height: stageHeight, rotation: 0 };
+    }
+    return resolveTransformRect(transform, stageWidth, stageHeight, contentWidth ?? stageWidth, contentHeight ?? stageHeight);
+}
+export function transformFromContainRect(contentWidth, contentHeight, stageWidth, stageHeight) {
+    const fit = computeContainRect(contentWidth, contentHeight, stageWidth, stageHeight);
+    return {
+        x: stageWidth > 0 ? (fit.x + fit.width / 2) / stageWidth : 0.5,
+        y: stageHeight > 0 ? (fit.y + fit.height / 2) / stageHeight : 0.5,
+        scale: contentWidth > 0 ? fit.width / contentWidth : 1,
+        rotation: 0,
+        anchor: { x: 0.5, y: 0.5 },
+    };
+}
+export function buildDrawTransformMatrix(transform, stageWidth, stageHeight, contentWidth, contentHeight) {
+    return buildTransformMatrixFromRect(resolveDrawRect(transform, stageWidth, stageHeight, contentWidth, contentHeight), stageWidth, stageHeight);
+}

package/dist/renderer/gpu/layers/objectFit.d.ts

@@ -0,0 +1,7 @@
+export interface FitRect {
+    x: number;
+    y: number;
+    width: number;
+    height: number;
+}
+export declare function computeContainRect(contentWidth: number, contentHeight: number, stageWidth: number, stageHeight: number): FitRect;

package/dist/renderer/gpu/layers/objectFit.js

@@ -0,0 +1,26 @@
+export function computeContainRect(contentWidth, contentHeight, stageWidth, stageHeight) {
+    if (contentWidth <= 0 ||
+        contentHeight <= 0 ||
+        stageWidth <= 0 ||
+        stageHeight <= 0) {
+        return { x: 0, y: 0, width: Math.max(stageWidth, 0), height: Math.max(stageHeight, 0) };
+    }
+    const stageAspect = stageWidth / stageHeight;
+    const contentAspect = contentWidth / contentHeight;
+    let width;
+    let height;
+    if (stageAspect > contentAspect) {
+        height = stageHeight;
+        width = height * contentAspect;
+    }
+    else {
+        width = stageWidth;
+        height = width / contentAspect;
+    }
+    return {
+        x: (stageWidth - width) / 2,
+        y: (stageHeight - height) / 2,
+        width,
+        height,
+    };
+}

package/dist/renderer/gpu/layers/textLayout.d.ts

@@ -0,0 +1,47 @@
+import type { ActiveTextClip } from '../../../resolver/scene';
+export declare const DEFAULT_FONT_SIZE = 48;
+export declare const DEFAULT_COLOR = "#ffffff";
+export declare const DEFAULT_FONT_FAMILY = "sans-serif";
+export declare const DEFAULT_FONT_WEIGHT: 'normal' | 'bold';
+export declare const DEFAULT_TEXT_ALIGN: 'left' | 'center' | 'right';
+export declare const SIDE_MARGIN = 0.05;
+export declare const LINE_HEIGHT = 1.2;
+export type TextMeasurer = Pick<CanvasRenderingContext2D, 'font' | 'measureText'>;
+export interface ResolvedTextStyle {
+    fontSize: number;
+    color: string;
+    fontFamily: string;
+    fontWeight: 'normal' | 'bold';
+    textAlign: 'left' | 'center' | 'right';
+}
+export interface StageRect {
+    x: number;
+    y: number;
+    width: number;
+    height: number;
+}
+export interface TextLayout {
+    style: ResolvedTextStyle;
+    font: string;
+    lines: string[];
+    lineAdvance: number;
+    anchorX: number;
+    firstLineY: number;
+    box: StageRect;
+    center: {
+        x: number;
+        y: number;
+    };
+}
+export declare function resolveTextStyle(item: {
+    fontSize?: number;
+    color?: string;
+    fontFamily?: string;
+    fontWeight?: 'normal' | 'bold';
+    textAlign?: 'left' | 'center' | 'right';
+}): ResolvedTextStyle;
+export declare function wrapText(measure: TextMeasurer, text: string, maxWidth: number): string[];
+export declare function computeTextLayout(measure: TextMeasurer, item: Pick<ActiveTextClip, 'content' | 'transform'> & Partial<ResolvedTextStyle>, stage: {
+    width: number;
+    height: number;
+}): TextLayout;

package/dist/renderer/gpu/layers/textLayout.js

@@ -0,0 +1,82 @@
+export const DEFAULT_FONT_SIZE = 48;
+export const DEFAULT_COLOR = '#ffffff';
+export const DEFAULT_FONT_FAMILY = 'sans-serif';
+export const DEFAULT_FONT_WEIGHT = 'normal';
+export const DEFAULT_TEXT_ALIGN = 'center';
+export const SIDE_MARGIN = 0.05;
+export const LINE_HEIGHT = 1.2;
+export function resolveTextStyle(item) {
+    return {
+        fontSize: item.fontSize ?? DEFAULT_FONT_SIZE,
+        color: item.color ?? DEFAULT_COLOR,
+        fontFamily: item.fontFamily ?? DEFAULT_FONT_FAMILY,
+        fontWeight: item.fontWeight ?? DEFAULT_FONT_WEIGHT,
+        textAlign: item.textAlign ?? DEFAULT_TEXT_ALIGN,
+    };
+}
+export function wrapText(measure, text, maxWidth) {
+    const out = [];
+    for (const paragraph of text.split('\n')) {
+        const words = paragraph.split(' ');
+        let line = '';
+        for (const word of words) {
+            const candidate = line ? `${line} ${word}` : word;
+            if (line && measure.measureText(candidate).width > maxWidth) {
+                out.push(line);
+                line = word;
+            }
+            else {
+                line = candidate;
+            }
+        }
+        out.push(line);
+    }
+    return out;
+}
+export function computeTextLayout(measure, item, stage) {
+    const style = resolveTextStyle(item);
+    const scale = item.transform?.scale ?? 1;
+    if (scale !== 1)
+        style.fontSize *= scale;
+    const font = `${style.fontWeight} ${style.fontSize}px ${style.fontFamily}`;
+    measure.font = font;
+    const maxWidth = stage.width * (1 - 2 * SIDE_MARGIN);
+    const lines = wrapText(measure, item.content, maxWidth);
+    const lineAdvance = style.fontSize * LINE_HEIGHT;
+    const blockHeight = lines.length * lineAdvance;
+    let maxLineWidth = 0;
+    for (const line of lines) {
+        const w = measure.measureText(line).width;
+        if (w > maxLineWidth)
+            maxLineWidth = w;
+    }
+    const center = {
+        x: item.transform ? item.transform.x : 0.5,
+        y: item.transform ? item.transform.y : 0.5,
+    };
+    const centerX = center.x * stage.width;
+    const centerY = center.y * stage.height;
+    const boxX = centerX - maxLineWidth / 2;
+    const boxY = centerY - blockHeight / 2;
+    let anchorX;
+    if (style.textAlign === 'left') {
+        anchorX = boxX;
+    }
+    else if (style.textAlign === 'right') {
+        anchorX = boxX + maxLineWidth;
+    }
+    else {
+        anchorX = centerX;
+    }
+    const firstLineY = boxY + lineAdvance / 2;
+    return {
+        style,
+        font,
+        lines,
+        lineAdvance,
+        anchorX,
+        firstLineY,
+        box: { x: boxX, y: boxY, width: maxLineWidth, height: blockHeight },
+        center,
+    };
+}

package/dist/renderer/gpu/layers/types.d.ts

@@ -0,0 +1,18 @@
+export interface LayerContext {
+    gl: WebGL2RenderingContext;
+    stage: {
+        width: number;
+        height: number;
+    };
+    viewport: {
+        width: number;
+        height: number;
+    };
+    fps: number;
+}
+export interface Layer<TItem = unknown> {
+    acquire(item: TItem, ctx: LayerContext): void;
+    release(itemId: string): void;
+    draw(item: TItem, ctx: LayerContext): void;
+    dispose(): void;
+}

package/dist/renderer/gpu/layers/types.js

@@ -0,0 +1 @@
+export {};

package/dist/renderer/gpu/shaders/quad.frag.d.ts

@@ -0,0 +1 @@
+export declare const QUAD_FRAG_SRC = "#version 300 es\nprecision mediump float;\n\nuniform sampler2D uTexture;\nuniform float uOpacity;\n\nin vec2 vTexCoord;\nout vec4 fragColor;\n\nvoid main() {\n  vec4 texel = texture(uTexture, vTexCoord);\n  fragColor = vec4(texel.rgb * uOpacity, texel.a * uOpacity);\n}\n";

package/dist/renderer/gpu/shaders/quad.frag.js

@@ -0,0 +1,14 @@
+export const QUAD_FRAG_SRC = `#version 300 es
+precision mediump float;
+
+uniform sampler2D uTexture;
+uniform float uOpacity;
+
+in vec2 vTexCoord;
+out vec4 fragColor;
+
+void main() {
+  vec4 texel = texture(uTexture, vTexCoord);
+  fragColor = vec4(texel.rgb * uOpacity, texel.a * uOpacity);
+}
+`;

package/dist/renderer/gpu/shaders/quad.vert.d.ts

@@ -0,0 +1 @@
+export declare const QUAD_VERT_SRC = "#version 300 es\nprecision highp float;\n\n// Positions for a unit quad in normalized stage space (0..1, origin top-left).\n// Laid out for TRIANGLE_STRIP: TL, BL, TR, BR.\nconst vec2 POSITIONS[4] = vec2[4](\n  vec2(0.0, 0.0),\n  vec2(0.0, 1.0),\n  vec2(1.0, 0.0),\n  vec2(1.0, 1.0)\n);\n\n// 3\u00D73 column-major transform: maps normalized stage coords \u2192 clip space.\n// Built by the CPU from the clip's Transform + stage/viewport geometry.\nuniform mat3 uTransform;\n\nout vec2 vTexCoord;\n\nvoid main() {\n  vec2 pos = POSITIONS[gl_VertexID];\n  vTexCoord = vec2(pos.x, 1.0 - pos.y); // flip Y for GL texture convention\n\n  // Homogeneous multiply, then convert to clip space: stage Y is top-down,\n  // GL NDC is bottom-up, so we negate the Y component produced by the matrix.\n  vec3 ndc = uTransform * vec3(pos, 1.0);\n  gl_Position = vec4(ndc.x, -ndc.y, 0.0, 1.0);\n}\n";

package/dist/renderer/gpu/shaders/quad.vert.js

@@ -0,0 +1,28 @@
+export const QUAD_VERT_SRC = `#version 300 es
+precision highp float;
+
+// Positions for a unit quad in normalized stage space (0..1, origin top-left).
+// Laid out for TRIANGLE_STRIP: TL, BL, TR, BR.
+const vec2 POSITIONS[4] = vec2[4](
+  vec2(0.0, 0.0),
+  vec2(0.0, 1.0),
+  vec2(1.0, 0.0),
+  vec2(1.0, 1.0)
+);
+
+// 3×3 column-major transform: maps normalized stage coords → clip space.
+// Built by the CPU from the clip's Transform + stage/viewport geometry.
+uniform mat3 uTransform;
+
+out vec2 vTexCoord;
+
+void main() {
+  vec2 pos = POSITIONS[gl_VertexID];
+  vTexCoord = vec2(pos.x, 1.0 - pos.y); // flip Y for GL texture convention
+
+  // Homogeneous multiply, then convert to clip space: stage Y is top-down,
+  // GL NDC is bottom-up, so we negate the Y component produced by the matrix.
+  vec3 ndc = uTransform * vec3(pos, 1.0);
+  gl_Position = vec4(ndc.x, -ndc.y, 0.0, 1.0);
+}
+`;

package/dist/renderer/gpu/types.d.ts

@@ -0,0 +1,21 @@
+import type { VideoFrameProviderFactory } from './layers/VideoLayer';
+import type { DemuxerFactory } from '../../media/video/demuxer/MediabunnyDemuxer';
+import type { VideoDecoderFactory } from '../../media/video/VideoDecoderManager';
+export interface Viewport {
+    width: number;
+    height: number;
+}
+export interface RendererOptions {
+    maxTextures?: number;
+    clearColor?: [number, number, number, number];
+    providerFactory?: VideoFrameProviderFactory;
+    demuxerFactory?: DemuxerFactory;
+    decoderFactory?: VideoDecoderFactory;
+    maxOutstandingDecodes?: number;
+    preserveDrawingBuffer?: boolean;
+    probeLayer?: boolean;
+}
+export interface SceneDiff<TItem> {
+    entering: TItem[];
+    leaving: string[];
+}

package/dist/renderer/gpu/types.js

@@ -0,0 +1 @@
+export {};

package/dist/renderer/gpu/viewport.d.ts

@@ -0,0 +1,7 @@
+export interface ViewportRect {
+    x: number;
+    y: number;
+    width: number;
+    height: number;
+}
+export declare function computeContainViewport(canvasWidth: number, canvasHeight: number, stageWidth: number, stageHeight: number): ViewportRect;

package/dist/renderer/gpu/viewport.js

@@ -0,0 +1,33 @@
+export function computeContainViewport(canvasWidth, canvasHeight, stageWidth, stageHeight) {
+    if (canvasWidth <= 0 ||
+        canvasHeight <= 0 ||
+        stageWidth <= 0 ||
+        stageHeight <= 0) {
+        return {
+            x: 0,
+            y: 0,
+            width: Math.max(canvasWidth, 0),
+            height: Math.max(canvasHeight, 0),
+        };
+    }
+    const canvasAspect = canvasWidth / canvasHeight;
+    const stageAspect = stageWidth / stageHeight;
+    let width;
+    let height;
+    if (canvasAspect > stageAspect) {
+        height = canvasHeight;
+        width = height * stageAspect;
+    }
+    else {
+        width = canvasWidth;
+        height = width / stageAspect;
+    }
+    const w = Math.round(width);
+    const h = Math.round(height);
+    return {
+        x: Math.round((canvasWidth - w) / 2),
+        y: Math.round((canvasHeight - h) / 2),
+        width: w,
+        height: h,
+    };
+}

package/dist/renderer/types.d.ts

@@ -0,0 +1,7 @@
+import type { Scene } from '../resolver/scene';
+export interface Renderer {
+    mount(container: HTMLElement): void;
+    resize(cssWidth: number, cssHeight: number, dpr?: number): void;
+    render(scene: Scene): void;
+    dispose(): void;
+}

package/dist/renderer/types.js

@@ -0,0 +1 @@
+export {};

package/dist/resolver/resolveTimeline.d.ts

@@ -0,0 +1,3 @@
+import type { Project } from '../types';
+import type { Scene } from './scene';
+export declare function resolveTimeline(frame: number, project: Project): Scene;