@twick/visualizer 0.0.1 → 0.14.2

package/src/components/track.tsx

@@ -0,0 +1,237 @@
+/**
+ * Track component that handles the creation and management of different types of tracks
+ * including video, audio, captions, scenes, and elements.
+ */
+
+import { Layout, Rect, View2D, Audio } from "@twick/2d";
+import { VisualizerTrack } from "../helpers/types";
+import { all, Color, createRef, ThreadGenerator, waitFor } from "@twick/core";
+import {
+  CAPTION_STYLE,
+  DEFAULT_CAPTION_COLORS,
+  DEFAULT_CAPTION_FONT,
+} from "../helpers/constants";
+import { logger } from "../helpers/log.utils";
+import elementController from "../controllers/element.controller";
+import { hexToRGB } from "../helpers/utils";
+
+/**
+ * Creates a video track with specified configuration
+ * @param {Object} params - Parameters for video track creation
+ * @param {View2D} params.view - The 2D view to render the video in
+ * @param {VideoTrack} params.track - Video track configuration
+ * @returns {Generator} Generator function for video track animation
+ */
+export function* makeVideoTrack({
+  view,
+  track,
+}: {
+  view: View2D;
+  track: VisualizerTrack;
+}) {
+  const frameRef = createRef<any>();
+  let prevTime = 0;
+  view.add(<Layout size={"100%"} ref={frameRef} layout />);
+  for (const element of track.elements || []) {
+    yield* waitFor(element?.s - prevTime);
+    yield* elementController.get("video")?.create({
+      containerRef: frameRef,
+      element,
+      view,
+    });
+    prevTime = element.e;
+  }
+  yield frameRef().remove();
+}
+
+/**
+ * Creates an audio track with specified configuration
+ * @param {Object} params - Parameters for audio track creation
+ * @param {View2D} params.view - The 2D view to render the audio in
+ * @param {AudioTrack} params.track - Audio track configuration
+ * @returns {Generator} Generator function for audio track animation
+ */
+export function* makeAudioTrack({
+  view,
+  track,
+}: {
+  view: View2D;
+  track: VisualizerTrack;
+}) {
+  let prevTime = 0;
+  for (const audioElement of track.elements) {
+    const audioRef = createRef<any>();
+    yield* waitFor(audioElement?.s - prevTime);
+    prevTime = audioElement?.e;
+    logger(`Adding audio element ${audioElement.id}`);
+    view.add(
+      <Audio ref={audioRef} key={audioElement.id} {...audioElement.props} />
+    );
+    yield* waitFor(Math.max(0, audioElement.e - audioElement.s));
+    yield audioRef().playing(false);
+    yield audioRef().remove();
+  }
+}
+
+/**
+ * Creates a caption track with specified configuration
+ * @param {Object} params - Parameters for caption track creation
+ * @param {View2D} params.view - The 2D view to render the caption in
+ * @param {CaptionTrack} params.track - Caption track configuration
+ * @returns {Generator} Generator function for caption track animation
+ */
+export function* makeCaptionTrack({
+  view,
+  track,
+}: {
+  view: View2D;
+  track: VisualizerTrack;
+}) {
+  let prevTime = 0;
+  const captionTrackRef = createRef<any>();
+  view.add(<Layout size={"100%"} ref={captionTrackRef} />);
+
+  const tProps = track?.props;
+
+  const trackDefaultProps =
+    (CAPTION_STYLE[tProps?.capStyle ?? ""] || {}).word || {};
+
+  for (const element of track.elements) {
+    const eProps = element.props;
+    const rectStyle =
+      (CAPTION_STYLE[eProps?.capStyle ?? tProps?.capStyle ?? ""] || {}).rect ||
+      {};
+    // Cast alignItems/justifyContent as any to satisfy RectProps
+    const mappedRectStyle = {
+      ...rectStyle,
+      justifyContent: rectStyle.justifyContent as any,
+      alignItems: rectStyle.alignItems as any,
+    };
+    const phraseProps = {
+      ...trackDefaultProps,
+      colors: {
+        text:
+          eProps?.colors?.text ??
+          tProps?.colors?.text ??
+          DEFAULT_CAPTION_COLORS.text,
+        background:
+          eProps?.colors?.background ??
+          tProps?.colors?.background ??
+          DEFAULT_CAPTION_COLORS.background,
+      },
+      font: {
+        family:
+          eProps?.font?.family ??
+          tProps?.font?.family ??
+          DEFAULT_CAPTION_FONT.family,
+        size:
+          eProps?.font?.size ?? tProps?.font?.size ?? DEFAULT_CAPTION_FONT.size,
+        weight:
+          eProps?.font?.weight ??
+          tProps?.font?.weight ??
+          DEFAULT_CAPTION_FONT.weight,
+      },
+      fill:
+        eProps?.colors?.text ??
+        tProps?.colors?.text ??
+        DEFAULT_CAPTION_COLORS.text,
+      bgColor:
+        eProps?.colors?.background ??
+        tProps?.colors?.background ??
+        DEFAULT_CAPTION_COLORS.background,
+      bgOpacity: eProps?.bgOpacity ?? tProps?.bgOpacity ?? 1,
+      ...(tProps?.captionProps || {}),
+    };
+
+    yield* waitFor(element?.s - prevTime);
+    const phraseRef = createRef<any>();
+    captionTrackRef().add(
+      <Rect
+        ref={phraseRef}
+        key={element.id}
+        {...mappedRectStyle}
+        x={eProps?.x ?? tProps?.x}
+        y={eProps?.y ?? tProps?.y}
+        layout
+      />
+    );
+    if (tProps?.capStyle === "word_by_word_with_bg") {
+      const _color = new Color({...hexToRGB(phraseProps.bgColor), a: phraseProps?.bgOpacity ?? 1});
+      phraseRef().fill(_color);
+    }
+    yield* elementController.get("caption")?.create({
+      containerRef: phraseRef,
+      caption: {
+        ...element,
+        t: element.t ?? "",
+        capStyle: eProps?.capStyle ?? tProps?.capStyle,
+        props: phraseProps,
+      },
+      view,
+    });
+    prevTime = element.e;
+    yield phraseRef().remove();
+  }
+}
+
+/**
+ * Creates a scene track with specified configuration
+ * @param {Object} params - Parameters for scene track creation
+ * @param {View2D} params.view - The 2D view to render the scene in
+ * @param {SceneTrack} params.track - Scene track configuration
+ * @returns {Generator} Generator function for scene track animation
+ */
+export function* makeSceneTrack({
+  view,
+  track,
+}: {
+  view: View2D;
+  track: VisualizerTrack;
+}) {
+  const frameRef = createRef<any>();
+  view.add(<Layout size={"100%"} ref={frameRef} layout />);
+  for (const sceneElement of track.elements || []) {
+    yield* elementController.get("scene")?.create({
+      containerRef: frameRef,
+      element: sceneElement,
+      view,
+    });
+  }
+  yield frameRef().remove();
+}
+
+/**
+ * Creates an element track with specified configuration
+ * @param {Object} params - Parameters for element track creation
+ * @param {View2D} params.view - The 2D view to render the element in
+ * @param {ElementTrack} params.track - Element track configuration
+ * @returns {Generator} Generator function for element track animation
+ */
+export function* makeElementTrack({
+  view,
+  track,
+}: {
+  view: View2D;
+  track: VisualizerTrack;
+}) {
+  const elementTrackRef = createRef<any>();
+  view.add(<Layout size={"100%"} ref={elementTrackRef} />);
+
+  const sequence: ThreadGenerator[] = [];
+  try {
+    for (const element of track.elements) {
+      sequence.push(
+        elementController.get(element.type)?.create({
+          containerRef: elementTrackRef,
+          element,
+          view,
+        })
+      );
+    }
+  } catch (error) {
+    logger("Error creating element track", error);
+  }
+  
+  yield* all(...sequence);
+  yield elementTrackRef().remove();
+}

package/src/controllers/animation.controller.ts

@@ -0,0 +1,39 @@
+import { BlurAnimation } from "../animations/blur";
+import { BreatheAnimation } from "../animations/breathe";
+import { FadeAnimation } from "../animations/fade";
+import { PhotoRiseAnimation } from "../animations/photo-rise";
+import { PhotoZoomAnimation } from "../animations/photo-zoom";
+import { RiseAnimation } from "../animations/rise";
+import { SuccessionAnimation } from "../animations/succession";
+import { Animation } from "../helpers/types";
+
+export class AnimationController {
+    private animations: Map<string, Animation> = new Map();
+  
+    register(animation: Animation) {
+      this.animations.set(animation.name, animation);
+    }
+  
+    get(name: string): Animation | undefined {
+      return this.animations.get(name);
+    }
+  
+    list(): string[] {
+      return Array.from(this.animations.keys());
+    }
+  }
+
+  export const registerAnimations = () => {
+    animationController.register(FadeAnimation);
+    animationController.register(RiseAnimation);
+    animationController.register(BreatheAnimation);
+    animationController.register(SuccessionAnimation);
+    animationController.register(BlurAnimation);
+    animationController.register(PhotoZoomAnimation);
+    animationController.register(PhotoRiseAnimation);
+  }
+
+  const animationController = new AnimationController();  
+  registerAnimations();
+
+  export default animationController;

package/src/controllers/element.controller.ts

@@ -0,0 +1,43 @@
+import { AudioElement } from "../elements/audio.element";
+import { CaptionElement } from "../elements/caption.element";
+import { CircleElement } from "../elements/circle.element";
+import { IconElement } from "../elements/icon.element";
+import { ImageElement } from "../elements/image.element";
+import { RectElement } from "../elements/rect.element";
+import { SceneElement } from "../elements/scene.element";
+import { TextElement } from "../elements/text.element";
+import { VideoElement } from "../elements/video.element";
+import { Element } from "../helpers/types";
+
+export class ElementController {
+    private elements: Map<string, Element> = new Map();
+  
+    register(element: Element) {
+      this.elements.set(element.name, element);
+    }
+  
+    get(name: string): Element | undefined {
+      return this.elements.get(name);
+    }
+  
+    list(): string[] {
+      return Array.from(this.elements.keys());
+    }
+  }
+
+  export const registerElements = () => {
+    elementController.register(VideoElement);
+    elementController.register(CaptionElement);
+    elementController.register(SceneElement);
+    elementController.register(ImageElement);
+    elementController.register(TextElement);
+    elementController.register(AudioElement);
+    elementController.register(CircleElement);
+    elementController.register(IconElement);
+    elementController.register(RectElement);
+  }
+
+  const elementController = new ElementController();  
+  registerElements();
+
+  export default elementController;

package/src/controllers/frame-effect.controller.tsx

@@ -0,0 +1,30 @@
+
+import { CircleFrameEffect } from "../frame-effects/circle.frame";
+import { RectFrameEffect } from "../frame-effects/rect.frame";
+import {  FrameEffectPlugin } from "../helpers/types";
+
+export class FrameEffectController {
+    private frameEffects: Map<string, FrameEffectPlugin> = new Map();
+  
+    register(frameEffect: FrameEffectPlugin) {
+      this.frameEffects.set(frameEffect.name, frameEffect);
+    }
+  
+    get(name: string): FrameEffectPlugin | undefined {
+      return this.frameEffects.get(name);
+    }
+  
+    list(): string[] {
+      return Array.from(this.frameEffects.keys());
+    }
+  }
+
+  export const registerFrameEffects = () => {
+    frameEffectController.register(CircleFrameEffect);
+    frameEffectController.register(RectFrameEffect);
+  }
+
+  const frameEffectController = new FrameEffectController();  
+  registerFrameEffects();
+
+  export default frameEffectController;

package/src/controllers/text-effect.controller.ts

@@ -0,0 +1,33 @@
+import { ElasticEffect } from "../text-effects/elastic";
+import { EraseEffect } from "../text-effects/erase";  
+import { StreamWordEffect } from "../text-effects/stream-word";
+import { TypewriterEffect } from "../text-effects/typewriter";
+import { TextEffect } from "../helpers/types";
+
+export class TextEffectController {
+    private effects: Map<string, TextEffect> = new Map();
+  
+    register(effect: TextEffect) {
+      this.effects.set(effect.name, effect);
+    }
+  
+    get(name: string): TextEffect | undefined {
+      return this.effects.get(name);
+    }
+  
+    list(): string[] {
+      return Array.from(this.effects.keys());
+    }
+  }
+
+  export const registerTextEffects = () => {
+    textEffectController.register(TypewriterEffect);
+    textEffectController.register(StreamWordEffect);
+    textEffectController.register(EraseEffect);
+    textEffectController.register(ElasticEffect);
+  }
+
+  const textEffectController = new TextEffectController();  
+  registerTextEffects();
+
+  export default textEffectController;

package/src/elements/audio.element.tsx

@@ -0,0 +1,17 @@
+import { ElementParams } from "../helpers/types";
+import { createRef, waitFor } from "@twick/core";
+import { Audio } from "@twick/2d";
+
+export const AudioElement = {
+  name: "audio",
+  *create({ containerRef, element, view }: ElementParams) {
+    const elementRef = createRef<any>();
+    yield* waitFor(element?.s);
+    yield containerRef().add(
+      <Audio ref={elementRef} key={element.id} play={true} {...element.props} />
+    );
+    yield* waitFor(Math.max(0, element.e - element.s));
+    yield elementRef().play(false);
+    yield elementRef().remove();
+  },
+};

package/src/elements/caption.element.tsx

@@ -0,0 +1,87 @@
+import { CaptionProps, ElementParams } from "../helpers/types";
+import { Color, createRef, Reference, waitFor } from "@twick/core";
+import { Rect, Txt } from "@twick/2d";
+import { splitPhraseTiming } from "../helpers/caption.utils";
+import { TRANSPARENT_COLOR } from "../helpers/constants";
+import { hexToRGB } from "../helpers/utils";
+
+export const CaptionElement = {
+  name: "caption",
+  *create({ containerRef, caption }: ElementParams) {
+    const words = splitPhraseTiming(caption);
+    let phraseStart = 0;
+    if (words?.length) {
+      phraseStart = words[0].s;
+    }
+    let wordsState: {
+      refs: Array<{ bgRef?: Reference<any>; textRef: Reference<any> }>;
+      props: CaptionProps[];
+      idx: number;
+      prevTime: number;
+    } = {
+      refs: [],
+      props: [],
+      idx: 0,
+      prevTime: phraseStart,
+    };
+    for (const word of words) {
+      wordsState.props.push(caption.props);
+      const textRef = createRef<Txt>();
+      const captionProps = caption.props;
+      containerRef().add(
+        <Txt ref={textRef} {...captionProps} text={word.t} opacity={0} />
+      );
+      if (caption.capStyle == "highlight_bg") {
+        const bgContainerRef = createRef();
+        const childTextRef = createRef();
+        const _color = new Color({...hexToRGB(captionProps.colors.background), a: captionProps?.bgOpacity ?? 1});
+        containerRef().add(
+          <Rect
+            ref={bgContainerRef}
+            fill={_color}
+            width={textRef().width() + (captionProps.bgOffsetWidth ?? 30)}
+            height={textRef().height() + (captionProps.bgOffsetHeight ?? 10)}
+            margin={captionProps.bgMargin ?? [0, -5]}
+            radius={captionProps.bgRadius ?? 10}
+            padding={captionProps.bgPadding ?? [0, 15]}
+            opacity={0}
+            alignItems={"center"}
+            justifyContent={"center"}
+            layout
+          >
+            <Txt ref={childTextRef} {...captionProps} text={word.t} />
+          </Rect>
+        );
+        textRef().remove();
+        wordsState.refs.push({
+          bgRef: bgContainerRef,
+          textRef: childTextRef,
+        });
+      } else {
+        wordsState.refs.push({
+          textRef: textRef,
+        });
+      }
+      wordsState.prevTime = word.e;
+      wordsState.idx = wordsState.idx + 1;
+    }
+
+    wordsState.prevTime = phraseStart;
+    wordsState.idx = 0;
+
+    for (const word of words) {
+      if (caption.capStyle == "highlight_bg") {
+        yield* wordsState.refs[wordsState.idx]?.bgRef?.().opacity(1, 0);
+        yield* waitFor(Math.max(0, word.e - word.s));
+        yield* wordsState.refs[wordsState.idx]
+          ?.bgRef?.()
+          .fill(TRANSPARENT_COLOR, 0);
+      } else {
+        yield* wordsState.refs[wordsState.idx]?.textRef?.().opacity(1, 0);
+        yield* waitFor(Math.max(0, word.e - word.s));
+      }
+      wordsState.prevTime = word.e;
+      wordsState.idx = wordsState.idx + 1;
+    }
+  },
+};

package/src/elements/circle.element.tsx

@@ -0,0 +1,20 @@
+import { ElementParams } from "../helpers/types";
+import { all, createRef, waitFor } from "@twick/core";
+import { Circle } from "@twick/2d";
+import { addAnimation } from "../helpers/element.utils";
+
+export const CircleElement = {
+  name: "circle",
+  *create({ containerRef, element, view }: ElementParams) {
+    const elementRef = createRef<any>();
+    yield* waitFor(element?.s);
+    yield containerRef().add(
+      <Circle ref={elementRef} key={element.id} {...element.props} />
+    );
+    yield* all(
+      addAnimation({ elementRef: elementRef, element: element, view }),
+      waitFor(Math.max(0, element.e - element.s))
+    );
+    yield elementRef().remove();
+  },
+};

package/src/elements/icon.element.tsx

@@ -0,0 +1,20 @@
+import { ElementParams } from "../helpers/types";
+import { all, createRef, waitFor } from "@twick/core";
+import { Icon } from "@twick/2d";
+import { addAnimation } from "../helpers/element.utils";
+
+export const IconElement = {
+  name: "icon",
+  *create({ containerRef, element, view }: ElementParams) {
+    const elementRef = createRef<any>();
+    yield* waitFor(element?.s);
+    yield containerRef().add(
+      <Icon ref={elementRef} key={element.id} {...element.props} />
+    );
+    yield* all(
+      addAnimation({ elementRef: elementRef, element: element, view }),
+      waitFor(Math.max(0, element.e - element.s))
+    );
+    yield elementRef().remove();
+  },
+};

package/src/elements/image.element.tsx

@@ -0,0 +1,53 @@
+import { ElementParams } from "../helpers/types";
+import { all, createRef, waitFor } from "@twick/core";
+import { Img, Rect } from "@twick/2d";
+import { addAnimation, addFrameEffect, fitElement } from "../helpers/element.utils";
+import { applyColorFilter } from "../helpers/filters";
+
+export const ImageElement = {
+  name: "image",
+  *create({ containerRef, element, view }: ElementParams) {
+    yield* waitFor(element?.s);
+    const frameContainerRef = createRef<any>();
+    const frameElementRef = createRef<any>();
+
+    yield containerRef().add(
+      <Rect ref={frameContainerRef} key={element.id} {...element.frame}>
+        <Img
+          ref={frameElementRef}
+          key={`child-${element.id}`}
+          {...element.props}
+        />
+      </Rect>
+    );
+    if (frameContainerRef()) {
+      yield fitElement({
+        elementRef: frameElementRef,
+        containerSize: frameContainerRef().size(),
+        elementSize: frameElementRef().size(),
+        objectFit: element.objectFit,
+      });
+
+      if (element?.props?.mediaFilter) {
+        applyColorFilter(frameElementRef, element.props.mediaFilter);
+      }
+
+      yield* all(
+        addAnimation({
+          elementRef: frameElementRef,
+          containerRef: frameContainerRef,
+          element: element,
+          view,
+        }),
+        addFrameEffect({
+          containerRef: frameContainerRef,
+          elementRef: frameElementRef,
+          element,
+        }),
+        waitFor(Math.max(0, element.e - element.s))
+      );
+      yield frameElementRef().remove();
+      yield frameContainerRef().remove();
+    }
+  },
+};

package/src/elements/rect.element.tsx

@@ -0,0 +1,22 @@
+import { ElementParams } from "../helpers/types";
+import { all, createRef, waitFor } from "@twick/core";
+import { Rect } from "@twick/2d";
+import { addAnimation } from "../helpers/element.utils";
+import { logger } from "../helpers/log.utils";
+
+export const RectElement = {
+  name: "rect",
+  *create({ containerRef, element, view }: ElementParams) {
+    const elementRef = createRef<any>();
+    yield* waitFor(element?.s);
+    logger(`RectElement: ${JSON.stringify(element)}`);
+    yield containerRef().add(
+      <Rect ref={elementRef} key={element.id} {...element.props}/>
+    );
+    yield* all(
+      addAnimation({ elementRef: elementRef, element: element, view }),
+      waitFor(Math.max(0, element.e - element.s))
+    );
+    yield elementRef().remove();
+  },
+};

package/src/elements/scene.element.tsx

@@ -0,0 +1,29 @@
+import { ElementParams } from "../helpers/types";
+import { createRef, waitFor } from "@twick/core";
+import { Rect } from "@twick/2d";
+import { DEFAULT_BACKGROUND_COLOR, ELEMENT_TYPES } from "../helpers/constants";
+import { ImageElement } from "./image.element";
+import { VideoElement } from "./video.element";
+import { logger } from "../helpers/log.utils";
+
+export const SceneElement = {
+  name: "scene",
+  *create({ containerRef, element, view }: ElementParams) {
+    yield* waitFor(element?.s);
+    const mediaContainerRef = createRef<any>();
+    logger(`SceneElement: ${JSON.stringify(element)}`);
+    yield containerRef().add(
+      <Rect
+        ref={mediaContainerRef}
+        fill={element.backgroundColor ?? DEFAULT_BACKGROUND_COLOR}
+        size={"100%"}
+      />
+    );
+    if (element.type === ELEMENT_TYPES.IMAGE) {
+      yield* ImageElement.create({ containerRef, element, view });
+    } else if (element.type === ELEMENT_TYPES.VIDEO) {
+      yield* VideoElement.create({ containerRef, element, view });
+    }
+    yield mediaContainerRef().remove();
+  },
+};

package/src/elements/text.element.tsx

@@ -0,0 +1,28 @@
+import { ElementParams } from "../helpers/types";
+import { all, createRef, waitFor } from "@twick/core";
+import { Txt } from "@twick/2d";
+import { addAnimation, addTextEffect } from "../helpers/element.utils";
+
+export const TextElement = {
+    name: "text",
+    *create({ containerRef, element, view }: ElementParams) { 
+    const elementRef = createRef<any>();
+
+    yield* waitFor(element?.s);
+    yield containerRef().add(
+      <Txt
+        ref={elementRef}
+        key={element.id}
+        text={element.t}
+        {...element.props}
+      />
+    );
+    yield* all(
+      addAnimation({ elementRef: elementRef, element: element, view }),
+      addTextEffect({ elementRef: elementRef, element: element }),
+      waitFor(Math.max(0, element.e - element.s))
+    );
+    yield elementRef().remove();
+}
+  }
+