@agrhub/engine-pixi 0.0.0-stage → 1.4.0

package/dist/clips/transition-clip.d.ts

@@ -0,0 +1,45 @@
+import { BaseClip } from './base-clip';
+import { IClip } from './iclip';
+import { TransitionKey } from '../transition/glsl/gl-transition';
+export declare class Transition extends BaseClip {
+    readonly type = "Transition";
+    ready: IClip["ready"];
+    private _meta;
+    get meta(): {
+        duration: number;
+        width: number;
+        height: number;
+    };
+    /**
+     * Unique identifier for this clip instance
+     */
+    id: string;
+    /**
+     * The transition key
+     */
+    transitionKey: TransitionKey;
+    /**
+     * ID of the clip from which the transition starts
+     */
+    fromClipId: string | null;
+    /**
+     * ID of the clip to which the transition ends
+     */
+    toClipId: string | null;
+    /**
+     * Custom shader parameter values (uniform name -> value)
+     */
+    values: Record<string, any>;
+    constructor(transitionKey: TransitionKey);
+    clone(): Promise<this>;
+    tick(_time: number): Promise<{
+        video: ImageBitmap | undefined;
+        state: "success";
+    }>;
+    split(_time: number): Promise<[this, this]>;
+    toJSON(_main?: boolean): any;
+    /**
+     * Create a Transition instance from a JSON object
+     */
+    static fromObject(json: any): Promise<Transition>;
+}

package/dist/clips/video-clip.d.ts

@@ -0,0 +1,216 @@
+import { MP4Sample } from 'wrapbox';
+import { file } from 'opfs-tools';
+import { BaseClip } from './base-clip';
+import { IClip, IPlaybackCapable } from './iclip';
+import { VideoJSON } from '../json-serialization';
+type OPFSToolFile = ReturnType<typeof file>;
+type MPClipCloneArgs = Awaited<ReturnType<typeof mp4FileToSamples>> & {
+    localFile: OPFSToolFile;
+};
+interface MP4DecoderConf {
+    video: VideoDecoderConfig | null;
+    audio: AudioDecoderConfig | null;
+}
+export interface IMP4ClipOpts {
+    audio?: boolean | {
+        volume: number;
+    };
+    /**
+     * Unsafe, may be deprecated at any time
+     */
+    __unsafe_hardwareAcceleration__?: HardwarePreference;
+}
+type ExtMP4Sample = Omit<MP4Sample, "data"> & {
+    is_idr: boolean;
+    deleted?: boolean;
+    data: null | Uint8Array;
+};
+/**
+ * Video clip, parses MP4 files, uses {@link Video.tick} to decode image frames at specified time on demand
+ *
+ * Can be used to implement video frame extraction, thumbnail generation, video editing and other functions
+ *
+ * @example
+ * // Load video clip asynchronously
+ * const videoClip = await Video.fromUrl('clip.mp4', {
+ *   x: 0,
+ *   y: 0,
+ *   width: 1920,
+ *   height: 1080,
+ * });
+ *
+ * // Set timeline position
+ * videoClip.set({
+ *   display: {
+ *     from: 150, // frames
+ *     to: 450, // frames (10 seconds at 30fps)
+ *   },
+ * });
+ *
+ */
+export declare class Video extends BaseClip implements IPlaybackCapable {
+    readonly type = "Video";
+    private insId;
+    private logger;
+    ready: IClip["ready"];
+    private _meta;
+    get meta(): {
+        duration: number;
+        width: number;
+        height: number;
+        audioSampleRate: number;
+        audioChanCount: number;
+    };
+    private localFile;
+    /** Store binary data of video header (box: ftyp, moov) */
+    private headerBoxPos;
+    /**
+     * Provide binary data of video header (box: ftyp, moov)
+     * Use any mp4 demuxer to parse and get detailed video information
+     * Unit tests include sample code using mp4box.js
+     */
+    getFileHeaderBinData(): Promise<ArrayBuffer>;
+    /** Store video transform and rotation info, currently only restores rotation */
+    private parsedMatrix;
+    private vfRotater;
+    private videoSamples;
+    private audioSamples;
+    private videoFrameFinder;
+    private audioFrameFinder;
+    protected onEffectiveVolumeChange(): void;
+    private decoderConf;
+    private opts;
+    /**
+     * Whether to include audio track (hybrid JSON structure)
+     */
+    audio: boolean;
+    /**
+     * Unique identifier for this clip instance
+     */
+    id: string;
+    /**
+     * Array of effects to be applied to this clip
+     * Each effect specifies key, startTime, duration, and optional targets
+     */
+    effects: Array<{
+        id: string;
+        key: string;
+        startTime: number;
+        duration: number;
+    }>;
+    getVisibleHandles(): Array<"tl" | "tr" | "bl" | "br" | "ml" | "mr" | "mt" | "mb" | "rot">;
+    /**
+     * Load a video clip from a URL
+     * @param url Video URL
+     * @param opts Position and size options
+     * @returns Promise that resolves to a video clip
+     *
+     * @example
+     * const videoClip = await Video.fromUrl('clip.mp4', {
+     *   x: 0,
+     *   y: 0,
+     *   width: 1920,
+     *   height: 1080,
+     * });
+     */
+    static fromUrl(url: string, opts?: {
+        x?: number;
+        y?: number;
+        width?: number;
+        height?: number;
+    }): Promise<Video>;
+    constructor(source: OPFSToolFile | ReadableStream<Uint8Array> | MPClipCloneArgs, opts?: IMP4ClipOpts, src?: string);
+    /**
+     * Intercept data returned by {@link Video.tick} method for secondary processing of image and audio data
+     * @param time Time when tick was called
+     * @param tickRet Data returned by tick
+     *
+     *    */
+    tickInterceptor: <T extends Awaited<ReturnType<Video["tick"]>>>(time: number, tickRet: T) => Promise<T>;
+    /**
+     * Get image frame and audio data at specified time
+     * @param time Time in microseconds
+     */
+    tick(time: number): Promise<{
+        video?: VideoFrame;
+        audio: Float32Array[];
+        state: "success" | "done";
+    }>;
+    addEffect(effect: {
+        id: string;
+        key: string;
+        startTime: number;
+        duration: number;
+    }): void;
+    editEffect(effectId: string, newEffectData: Partial<{
+        key: string;
+        startTime: number;
+        duration: number;
+    }>): void;
+    removeEffect(effectId: string): void;
+    /**
+     * Split is handled by core state, not implemented here
+     */
+    split(_time: number): Promise<[this, this]>;
+    clone(): Promise<this>;
+    destroy(): void;
+    toJSON(main?: boolean): VideoJSON;
+    /**
+     * Create a Video instance from a JSON object (fabric.js pattern)
+     * @param json The JSON object representing the clip
+     * @returns Promise that resolves to a Video instance
+     */
+    static fromObject(json: VideoJSON): Promise<Video>;
+    /**
+     * Create HTMLVideoElement for playback
+     */
+    createPlaybackElement(): Promise<{
+        element: HTMLVideoElement;
+        objectUrl?: string;
+    }>;
+    play(element: HTMLVideoElement | HTMLAudioElement, timeSeconds: number): Promise<void>;
+    pause(element: HTMLVideoElement | HTMLAudioElement): void;
+    seek(element: HTMLVideoElement | HTMLAudioElement, timeSeconds: number): Promise<void>;
+    syncPlayback(element: HTMLVideoElement | HTMLAudioElement, isPlaying: boolean, timeSeconds: number): void;
+    cleanupPlayback(element: HTMLVideoElement | HTMLAudioElement, objectUrl?: string): void;
+    /**
+     * Scale clip to fit within the scene dimensions while maintaining aspect ratio
+     * Similar to fabric.js scaleToFit
+     * @param sceneWidth Scene width
+     * @param sceneHeight Scene height
+     */
+    scaleToFit(sceneWidth: number, sceneHeight: number): Promise<void>;
+    /**
+     * Scale clip to fill the scene dimensions while maintaining aspect ratio
+     * May crop parts of the clip. Similar to fabric.js scaleToFill
+     * @param sceneWidth Scene width
+     * @param sceneHeight Scene height
+     */
+    scaleToFill(sceneWidth: number, sceneHeight: number): Promise<void>;
+    /**
+     * Center the clip within the scene dimensions
+     * Similar to fabric.js center
+     * @param sceneWidth Scene width
+     * @param sceneHeight Scene height
+     */
+    centerInScene(sceneWidth: number, sceneHeight: number): void;
+}
+declare function mp4FileToSamples(otFile: OPFSToolFile, opts?: IMP4ClipOpts): Promise<{
+    videoSamples: ExtMP4Sample[];
+    audioSamples: ExtMP4Sample[];
+    decoderConf: MP4DecoderConf;
+    headerBoxPos: {
+        start: number;
+        size: number;
+    }[];
+    parsedMatrix: {
+        perspective: number;
+        rotationRad: number;
+        rotationDeg: number;
+        scaleX: number;
+        scaleY: number;
+        translateX: number;
+        translateY: number;
+    };
+}>;
+export {};

package/dist/compositor/audio-mixer.d.ts

@@ -0,0 +1,8 @@
+/**
+ * Buffer input data and convert to AudioData with fixed frame count.
+ * Matches the WebCodecs audio configuration.
+ *
+ * @param framesPerChunk Number of audio frames per AudioData instance
+ * @param fps The video fps (used for calculating buffer limits)
+ */
+export declare function createAudioTrackBuf(framesPerChunk: number, fps?: number): (timestamp: number, trackAudios: Float32Array[][]) => AudioData[];

package/dist/compositor/av-encoder.d.ts

@@ -0,0 +1,17 @@
+import { recodemux } from 'wrapbox';
+/**
+ * Resolve the WebGL/WebGL2 context already held by an OffscreenCanvas.
+ * Returns null if no WebGL context is attached (e.g. canvas not yet used).
+ */
+export declare function getCanvasGl(canvas: OffscreenCanvas): WebGLRenderingContext | WebGL2RenderingContext | null;
+/**
+ * Creates an AV encoder function that queues and encodes both mixed AudioData and VideoFrames.
+ */
+export declare function createAVEncoder(opts: {
+    muxer: ReturnType<typeof recodemux>;
+    canvas: OffscreenCanvas;
+    outputAudio?: boolean;
+    hasVideoTrack: boolean;
+    timeSlice: number;
+    fps: number;
+}): (timestamp: number, audios: Float32Array[][], hasVideo: boolean) => Promise<void>;

package/dist/compositor/clip-style-renderer.d.ts

@@ -0,0 +1,23 @@
+import { Container, Sprite, Texture, Graphics } from 'pixi.js';
+import { IClip } from '../clips';
+export interface IPooledClipObjects {
+    root: Container;
+    mainSprite: Sprite;
+    mirrors: Sprite[];
+    maskGfx: Graphics | null;
+    strokeGfx: Graphics | null;
+    shadowGfx: Graphics | null;
+}
+/**
+ * Applies layout transforms, mirrors, filters (blur, adjustments, chroma, hsl), and decoration styles
+ * (border radius, strokes, drop shadows) to a set of PixiJS containers and sprites representing a clip.
+ */
+export declare function applyClipStylesAndFilters(opts: {
+    clip: IClip;
+    tex: Texture;
+    rootContainer: Container;
+    tempSprite: Sprite;
+    pooled: IPooledClipObjects;
+    scaleX: number;
+    scaleY: number;
+}): void;

package/dist/compositor/compositor-renderer.d.ts

@@ -0,0 +1,29 @@
+import { Application } from 'pixi.js';
+import { IClip } from '../clips';
+export interface ISpritesRenderOpts {
+    pixiApp: Application | null;
+    backgroundColor: string;
+    sprites: Array<IClip & {
+        main: boolean;
+        expired: boolean;
+    }>;
+    aborter: {
+        aborted: boolean;
+    };
+    scaleX: number;
+    scaleY: number;
+}
+export interface ISpritesRenderResult {
+    render: (timestamp: number) => Promise<{
+        audios: Float32Array[][];
+        mainSprDone: boolean;
+        hasVideo: boolean;
+    }>;
+    cleanup: () => void;
+}
+/**
+ * Creates the renderer pipeline for compositor frames.
+ * Delegates actual transition handling, asset transformations, styles/filters,
+ * and global effects pipeline execution to separate managers.
+ */
+export declare function createSpritesRender(opts: ISpritesRenderOpts): ISpritesRenderResult;

package/dist/compositor/effect-manager.d.ts

@@ -0,0 +1,30 @@
+import { Application, Container, Sprite } from 'pixi.js';
+import { IClip } from '../clips';
+import { PixiSpriteRenderer } from '../sprite/pixi-sprite-renderer';
+export declare class EffectManager {
+    private pixiApp;
+    private effectCache;
+    private effectInputTexture;
+    private scaleX;
+    private scaleY;
+    constructor(pixiApp: Application, hasVideoTrack: boolean, scaleX?: number, scaleY?: number);
+    /**
+     * Processes all active global and adjustment layer effects at the given timestamp.
+     */
+    processEffects(opts: {
+        timestamp: number;
+        sprites: Array<IClip & {
+            expired?: boolean;
+        }>;
+        spriteById: Map<string, IClip>;
+        spriteIndexById: Map<string, number>;
+        spriteRenderers: Map<IClip & {
+            expired?: boolean;
+        }, PixiSpriteRenderer>;
+        transitionSprites: Map<string, Sprite>;
+        clipsEffectContainer: Container;
+        clipsNormalContainer: Container;
+        postProcessContainer: Container;
+    }): Promise<void>;
+    cleanup(): void;
+}

package/dist/compositor/scheduler.d.ts

@@ -0,0 +1,8 @@
+export declare function yieldToScheduler(): Promise<void>;
+/** Max frames allowed in the VideoEncoder queue before back-pressure kicks in. */
+export declare const ENCODER_QUEUE_LIMIT = 20;
+/**
+ * Wait until the encoder queue drains below the limit.
+ * Prevents OOM from too many queued VideoFrames.
+ */
+export declare function waitEncoderQueue(getQueueSize: () => number): Promise<void>;

package/dist/compositor/transition-manager.d.ts

@@ -0,0 +1,45 @@
+import { Application, Container, Sprite } from 'pixi.js';
+import { IClip, Transition } from '../clips';
+export declare class TransitionManager {
+    private pixiApp;
+    private scaleX;
+    private scaleY;
+    private transitionRenderers;
+    private transitionSprites;
+    private transFromTexture;
+    private transToTexture;
+    private transBgGraphics;
+    private transClipPool;
+    constructor(pixiApp: Application, scaleX: number, scaleY: number);
+    /**
+     * Reset the visibility of all transition sprites at the start of a frame.
+     */
+    resetFrame(): void;
+    /**
+     * Checks if a transition sprite has been created for a given ID.
+     */
+    getTransitionSprite(id: string): Sprite | undefined;
+    /**
+     * Retrieve pool container/sprite elements for a given clip to avoid allocations.
+     */
+    private getTransClipObjects;
+    private getClipFrameAtTimestamp;
+    private renderClipToTransitionTexture;
+    /**
+     * Render transition between two clips at a given timeline timestamp.
+     * Modifies target claimed set and updates internal transition sprite.
+     */
+    renderTransition(opts: {
+        sprite: Transition & {
+            expired?: boolean;
+        };
+        timestamp: number;
+        fromClip: IClip;
+        toClip: IClip;
+        getFrameCached: (clip: IClip, timestamp: number) => Promise<any>;
+        clipsNormalContainer: Container;
+        clipsClaimedByTransition: Set<string>;
+        onVideoState: () => void;
+    }): Promise<boolean>;
+    cleanup(): void;
+}

package/dist/compositor.d.ts

@@ -0,0 +1,157 @@
+import { IClip } from './clips';
+import { default as EventEmitter } from './event-emitter';
+import { ProjectJSON } from './json-serialization';
+export interface ICompositorOpts {
+    width?: number;
+    height?: number;
+    bitrate?: number;
+    fps?: number;
+    backgroundColor?: string;
+    format?: string;
+    videoCodec?: string;
+    /**
+     * If false, exclude audio track from the output video
+     */
+    audio?: false;
+    audioCodec?: string;
+    audioSampleRate?: number;
+    /**
+     * Write metadata tags to the output video
+     */
+    metaDataTags?: Record<string, string>;
+    /**
+     * Unsafe, may be deprecated at any time
+     */
+    __unsafe_hardwareAcceleration__?: HardwarePreference;
+}
+/**
+ * Video compositor that can add multiple {@link OffscreenSprite} instances,
+ * @example
+ * const spr1 = new OffscreenSprite(
+ *   new Video((await fetch('<mp4 url>')).body),
+ * );
+ * const spr2 = new OffscreenSprite(
+ *   await Audio.fromUrl('<audio url>'),
+ * );
+ * const com = new Compositor({ width: 1280, height: 720, });
+ 
+ * await com.addSprite(spr1);
+ * await com.addSprite(spr2);
+ 
+ * com.output(); // => ReadableStream
+ *
+ */
+export declare class Compositor extends EventEmitter<{
+    "export:progress": number;
+    error: Error;
+}> {
+    /**
+     * Check compatibility with the current environment
+     * @param args.videoCodec Specify video codec, default avc1.42E032
+     * @param args.width Specify video width, default 1920
+     * @param args.height Specify video height, default 1080
+     * @param args.bitrate Specify video bitrate, default 5e6
+     */
+    static isSupported(args?: {
+        videoCodec?: string;
+        width?: number;
+        height?: number;
+        bitrate?: number;
+    }): Promise<boolean>;
+    private logger;
+    private destroyed;
+    private sprites;
+    private canvas;
+    private pixiApp;
+    private stopOutput;
+    private opts;
+    /** Encodable audio chosen for output (undefined = not resolved yet, null = audio must be skipped) */
+    private resolvedAudio;
+    private explicitOpts;
+    private hasVideoTrack;
+    private jsonDimensions;
+    /**
+     * Create a compositor instance based on configuration
+     * @param opts ICompositorOpts
+     */
+    constructor(opts?: ICompositorOpts);
+    initPixiApp(): Promise<void>;
+    /**
+     * Settles which audio codec/sample rate output() will use, falling back when the requested one
+     * isn't encodable here (e.g. no AAC encoder in headless Chromium). Called by initPixiApp and
+     * loadFromJSON, which callers await before output().
+     */
+    private resolveAudioEncoding;
+    /**
+     * Add a clip for video composition. Video duration defaults to the maximum duration value from all clips
+     * @param clip Clip (extends BaseSprite)
+     * @param opts.main If main is true, the video duration uses this clip's duration value
+     */
+    addSprite(clip: IClip, opts?: {
+        main?: boolean;
+    }): Promise<void>;
+    private initMuxer;
+    /**
+     * Output video file binary stream
+     * @param opts.maxTime Maximum duration allowed for output video, content exceeding this will be ignored
+     */
+    output(opts?: {
+        maxTime?: number;
+    }): ReadableStream<Uint8Array>;
+    /**
+     * Destroy instance and release resources
+     */
+    destroy(): void;
+    private runEncoding;
+    /**
+     * Export current compositor state to JSON
+     */
+    exportToJSON(): ProjectJSON;
+    /**
+     * Load clips from JSON
+     * @param json The JSON project data
+     */
+    loadFromJSON(json: ProjectJSON): Promise<void>;
+    /**
+     * Renders the frame at the given time and returns it as a base64-encoded PNG.
+     *
+     * Initialises the internal Pixi application on demand (if not already
+     * initialised), performs a single-frame render using the same sprite
+     * pipeline as {@link output}, and returns the result without modifying
+     * the compositor's encoding state.
+     *
+     * @param timeMs Time in milliseconds
+     * @param options.transparent If true, the renderer background is made
+     *   transparent so the resulting PNG has a transparent background.
+     * @returns Base64 data-URL string (e.g. "data:image/png;base64,...")
+     *
+     * @example
+     * const compositor = new Compositor({ width: 1280, height: 720 });
+     * await compositor.addSprite(videoClip);
+     * const frame = await compositor.renderFrame(2000); // frame at 2 s
+     * const transparent = await compositor.renderFrame(2000, { transparent: true });
+     */
+    renderFrame(timeMs: number, options?: {
+        transparent?: boolean;
+    }): Promise<string>;
+    /**
+     * Alias for {@link renderFrame} with an optional timestamp.
+     * Captures a single frame as a base64-encoded PNG.
+     *
+     * @param timeMs Optional time in milliseconds, or an options object. If
+     *   omitted, the first frame at time 0 is captured.
+     * @param options.transparent If true, the renderer background is made
+     *   transparent so the resulting PNG has a transparent background.
+     * @returns Base64 data-URL string (e.g. "data:image/png;base64,...")
+     *
+     * @example
+     * const compositor = new Compositor({ width: 1280, height: 720 });
+     * await compositor.addSprite(videoClip);
+     * const firstFrame = await compositor.snapshot();              // time 0
+     * const frameAt2s  = await compositor.snapshot(2000);          // explicit timestamp
+     * const transparent = await compositor.snapshot({ transparent: true }); // time 0, transparent
+     */
+    snapshot(timeMs?: number | {
+        transparent?: boolean;
+    }): Promise<string>;
+}