@twick/visualizer 0.0.1

package/.eslintrc.json

@@ -0,0 +1,21 @@
+{
+  "root": true,
+  "extends": [
+    "eslint:recommended",
+    "plugin:@typescript-eslint/recommended"
+  ],
+  "parser": "@typescript-eslint/parser",
+  "plugins": ["@typescript-eslint"],
+  "parserOptions": {
+    "ecmaVersion": "latest",
+    "sourceType": "module"
+  },
+  "env": {
+    "es2022": true,
+    "node": true
+  },
+  "rules": {
+    "@typescript-eslint/no-explicit-any": "warn",
+    "@typescript-eslint/no-unused-vars": ["error", { "argsIgnorePattern": "^_" }]
+  }
+} 

package/README.md

@@ -0,0 +1,13 @@
+# @twick/visualizer
+
+A visualization library built on top of [@revideo/2d](https://github.com/re-video/2d) for creating interactive visualizations.
+
+## Installation
+```
+pnpm install
+```
+
+## Build
+```
+pnpm build
+```

package/package.json

@@ -0,0 +1,31 @@
+{
+  "name": "@twick/visualizer",
+  "version": "0.0.1",
+  "license": "Apache-2.0",
+  "scripts": {
+    "start": "revideo editor --projectFile ./src/live.tsx",
+    "build": "tsc && vite build",
+    "docs": "typedoc --out docs src/index.ts",
+    "clean": "rimraf .turbo node_modules dist"
+  },
+  "dependencies": {
+    "@preact/signals": "^1.2.1",
+    "@revideo/2d": "^0.10.4",
+    "@revideo/core": "^0.10.4",
+    "@revideo/renderer": "^0.10.4",
+    "@revideo/vite-plugin": "^0.10.4",
+    "date-fns": "^4.1.0",
+    "preact": "^10.19.2",
+    "crelt": "^1.0.6",
+    "@twick/media-utils": "0.0.1"
+  },
+  "devDependencies": {
+    "@revideo/cli": "0.10.4",
+    "@revideo/ui": "0.10.4",
+    "typescript": "5.4.2",
+    "typedoc": "^0.25.8",
+    "typedoc-plugin-markdown": "^3.17.1",
+    "vite-plugin-dts": "^3.7.3",
+    "vite": "^5.1.4"
+  }
+}

package/src/components/animation.tsx

@@ -0,0 +1,7 @@
+import { all, Reference } from "@revideo/core";
+
+export function* addAnimations({ elementRef }: { elementRef: Reference<any>;}) {
+    yield elementRef();
+    const animations:any[] = []
+    return yield* all(...animations);
+}

package/src/components/element.tsx

@@ -0,0 +1,344 @@
+/**
+ * Element component that handles the creation and management of visual elements
+ * with various properties like position, size, rotation, and effects.
+ */
+
+import { createRef, waitFor, all, Reference, Color } from "@revideo/core";
+import { Audio, Circle, Icon, Img, Rect, Txt, Video } from "@revideo/2d";
+import { logger } from "../helpers/log.utils";
+import {
+  DEFAULT_BACKGROUND_COLOR,
+  ELEMENT_TYPES,
+  TRANSPARENT_COLOR,
+} from "../helpers/constants";
+import { fitElement } from "../helpers/element.utils";
+import { applyColorFilter } from "../helpers/filters";
+import { addFrameEffect } from "./frame-effects";
+import { Caption, CaptionProps, VisualizerElement } from "../helpers/types";
+import { splitPhraseTiming } from "../helpers/caption.utils";
+import { addAnimations } from "./animation";
+
+
+export function* addMediaElement({
+  containerRef,
+  element,
+  mediaType,
+  waitOnStart = true,
+}: {
+  containerRef: Reference<any>;
+  element: VisualizerElement;
+  mediaType: string;
+  waitOnStart?: boolean;
+}) {
+  if (waitOnStart) {
+    yield* waitFor(element.s);
+  }
+  const frameContainerRef = createRef<any>();
+  const frameElementRef = createRef<any>();
+  if (mediaType === ELEMENT_TYPES.VIDEO) {
+    const frameProps = element.frame || element.props;
+    logger(`Adding video element ${element.id}`);
+    yield containerRef().add(
+      <Rect ref={frameContainerRef} key={element.id} {...frameProps}>
+        <Video
+          ref={frameElementRef}
+          key={`child-${element.id}`}
+          {...element.props}
+        />
+      </Rect>
+    );
+  } else if (mediaType === ELEMENT_TYPES.IMAGE) {
+    logger(`Adding image element ${element.id}`);
+    yield containerRef().add(
+      <Rect ref={frameContainerRef} key={element.id} {...element.frame}>
+        <Img
+          ref={frameElementRef}
+          key={`child-${element.id}`}
+          {...element.props}
+        />
+      </Rect>
+    );
+  }
+  if (frameContainerRef()) {
+    yield fitElement({
+      elementRef: frameElementRef,
+      containerSize: frameContainerRef().size(),
+      elementSize: frameElementRef().size(),
+      objectFit: element.objectFit,
+    });
+
+    if (element?.props?.mediaFilter) {
+      applyColorFilter(frameElementRef, element.props.mediaFilter);
+    }
+
+    yield* all(
+      addAnimations({
+        elementRef: frameElementRef,
+      }),
+      addFrameEffect({
+        containerRef: frameContainerRef,
+        elementRef: frameElementRef,
+        frameElement,
+      }),
+      waitFor(Math.max(0, element.e - element.s))
+    );
+    if (element.type === ELEMENT_TYPES.VIDEO) {
+      yield frameElementRef().playing(false);
+    }
+    yield frameElementRef().remove();
+    yield frameContainerRef().remove();
+  }
+}
+
+export function* addCaptionElement({
+  caption,
+  captionProps,
+  containerRef,
+  capStyle,
+}: {
+  caption: Caption;
+  captionProps: CaptionProps;
+  containerRef: Reference<any>;
+  capStyle: string;
+}) {
+  const words = splitPhraseTiming(caption);
+  let phraseStart = 0;
+  if (words?.length) {
+    phraseStart = words[0].s;
+  }
+  let wordsState: {
+    refs: Array<{ bgRef?: Reference<any>; textRef: Reference<any> }>;
+    props: CaptionProps[];
+    idx: number;
+    prevTime: number;
+  } = {
+    refs: [],
+    props: [],
+    idx: 0,
+    prevTime: phraseStart,
+  };
+
+  for (const word of words) {
+    wordsState.props.push(captionProps);
+    const textRef = createRef<Txt>();
+    containerRef().add(
+      <Txt ref={textRef} {...captionProps} text={word.t} opacity={0} />
+    );
+    if (capStyle == "highlight_bg") {
+      const bgContainerRef = createRef();
+      const childTextRef = createRef();
+      containerRef().add(
+        <Rect
+          ref={bgContainerRef}
+          fill={new Color(captionProps.colors.background).alpha(
+            captionProps?.bgOpacity ?? 1
+          )}
+          width={textRef().width() + (captionProps.bgOffsetWidth ?? 30)}
+          height={textRef().height() + (captionProps.bgOffsetHeight ?? 10)}
+          margin={captionProps.bgMargin ?? [0, -5]}
+          radius={captionProps.bgRadius ?? 10}
+          padding={captionProps.bgPadding ?? [0, 15]}
+          opacity={0}
+          alignItems={"center"}
+          justifyContent={"center"}
+          layout
+        >
+          <Txt ref={childTextRef} {...captionProps} text={word.t} />
+        </Rect>
+      );
+      textRef().remove();
+      wordsState.refs.push({
+        bgRef: bgContainerRef,
+        textRef: childTextRef,
+      });
+    } else {
+      wordsState.refs.push({
+        textRef: textRef,
+      });
+    }
+    wordsState.prevTime = word.e;
+    wordsState.idx = wordsState.idx + 1;
+  }
+
+  wordsState.prevTime = phraseStart;
+  wordsState.idx = 0;
+
+  for (const word of words) {
+    if (capStyle == "highlight_bg") {
+      yield* wordsState.refs[wordsState.idx]?.bgRef?.().opacity(1, 0);
+      yield* waitFor(Math.max(0, word.e - word.s));
+      yield* wordsState.refs[wordsState.idx]
+        ?.bgRef?.()
+        .fill(TRANSPARENT_COLOR, 0);
+    } else {
+      yield* wordsState.refs[wordsState.idx]?.textRef?.().opacity(1, 0);
+      yield* waitFor(Math.max(0, word.e - word.s));
+    }
+    wordsState.prevTime = word.e;
+    wordsState.idx = wordsState.idx + 1;
+  }
+}
+
+export function* makeSceneElements({
+  containerRef,
+  element,
+}: {
+  containerRef: Reference<any>;
+  element: VisualizerElement;
+}) {
+  switch (element.type) {
+    case ELEMENT_TYPES.RECT:
+      yield* addSceneRectElement({ containerRef, element });
+      break;
+    case ELEMENT_TYPES.IMAGE:
+    case ELEMENT_TYPES.VIDEO:
+      const mediaContainerRef = createRef<any>();
+      yield containerRef().add(
+        <Rect
+          ref={mediaContainerRef}
+          fill={element.backgroundColor ?? DEFAULT_BACKGROUND_COLOR}
+          size={"100%"}
+        />
+      );
+      yield* addMediaElement({
+        containerRef: mediaContainerRef,
+        element,
+        mediaType: element.type,
+        waitOnStart: false,
+      });
+      yield mediaContainerRef().remove();
+      break;
+    default:
+      yield* waitFor(Math.max(0, element.e - element.s));
+      break;
+  }
+}
+
+function* addSceneRectElement({
+  containerRef,
+  element,
+}: {
+  containerRef: Reference<any>;
+  element: VisualizerElement;
+}) {
+  const elementRef = createRef<any>();
+  let sequence: any[] = [];
+  yield containerRef().add(
+    <Rect ref={elementRef} key={element.id} {...element.props}></Rect>
+  );
+  for (const childElement of element.elements || []) {
+    sequence.push(
+      makeSceneElements({
+        containerRef: elementRef,
+        element: childElement,
+      })
+    );
+  }
+
+  yield* all(
+    ...sequence,
+    addAnimations({
+      elementRef,
+    }),
+    waitFor(Math.max(0, element.e - element.s))
+  );
+  yield elementRef().remove();
+}
+
+export function* addRectElement({
+  containerRef,
+  element,
+}: {
+  containerRef: Reference<any>;
+  element: VisualizerElement;
+}) {
+  const elementRef = createRef<any>();
+  yield* waitFor(element?.s);
+  yield containerRef().add(
+    <Rect ref={elementRef} key={element.id} {...element.props} />
+  );
+  yield* all(
+    addAnimations({ elementRef: elementRef }),
+    waitFor(Math.max(0, element.e - element.s))
+  );
+  yield elementRef().remove();
+}
+
+export function* addTextElement({
+  containerRef,
+  element,
+}: {
+  containerRef: Reference<any>;
+  element: VisualizerElement;
+}) {
+  const elementRef = createRef<any>();
+  yield* waitFor(element?.s);
+  yield containerRef().add(
+    <Txt
+      ref={elementRef}
+      key={element.id}
+      text={element.t}
+      {...element.props}
+    />
+  );
+  yield* all(
+    addAnimations({ elementRef: elementRef }),
+    waitFor(Math.max(0, element.e - element.s))
+  );
+  yield elementRef().remove();
+}
+
+export function* addCircleElement({
+  containerRef,
+  element,
+}: {
+  containerRef: Reference<any>;
+  element: VisualizerElement;
+}) {
+  const elementRef = createRef<any>();
+  yield* waitFor(element?.s);
+  yield containerRef().add(
+    <Circle ref={elementRef} key={element.id} {...element.props} />
+  );
+  yield* all(
+    addAnimations({ elementRef: elementRef }),
+    waitFor(Math.max(0, element.e - element.s))
+  );
+  yield elementRef().remove();
+}
+
+export function* addIconElement({
+  containerRef,
+  element,
+}: {
+  containerRef: Reference<any>;
+  element: VisualizerElement;
+}) {
+  const elementRef = createRef<any>();
+  yield* waitFor(element?.s);
+  yield containerRef().add(
+    <Icon ref={elementRef} key={element.id} {...element.props} />
+  );
+  yield* all(
+    addAnimations({ elementRef: elementRef }),
+    waitFor(Math.max(0, element.e - element.s))
+  );
+  yield elementRef().remove();
+}
+
+export function* addAudioElement({
+  containerRef,
+  element,
+}: {
+  containerRef: Reference<any>;
+  element: VisualizerElement;
+}) {
+  const elementRef = createRef<any>();
+  yield* waitFor(element?.s);
+  yield containerRef().add(
+    <Audio ref={elementRef} key={element.id} {...element.props} />
+  );
+  yield* waitFor(Math.max(0, element.e - element.s));
+  yield elementRef().playing(false);
+  yield elementRef().remove();
+}

package/src/components/frame-effects.tsx

@@ -0,0 +1,190 @@
+/**
+ * Frame effects component that handles the application of various visual effects
+ * to frames and elements in the scene.
+ */
+
+import { all, Reference, waitFor } from "@revideo/core";
+import { getTimingFunction } from "../helpers/timing.utils";
+import { fitElement } from "../helpers/element.utils";
+import { DEFAULT_POSITION, DEFAULT_TIMING_FUNCTION, FRAME_SHAPE } from "../helpers/constants";
+import { FrameEffect } from "../helpers/types";
+import { View2D } from "@revideo/2d";
+import { createSignal } from "@revideo/core";
+import { logger } from "../helpers/log.utils";
+
+/**
+ * Applies frame effects to a view or element
+ * @param {Object} params - Parameters for applying frame effects
+ * @param {View2D} params.view - The 2D view to apply effects to
+ * @param {FrameEffect[]} params.effects - Array of effects to apply
+ * @returns {void}
+ */
+export function applyFrameEffects({ view, effects }: { view: View2D; effects: FrameEffect[] }) {
+  // ... existing code ...
+}
+
+export function* addFrameEffect({
+  containerRef,
+  elementRef,
+  frameElement,
+}: {
+  containerRef: Reference<any>;
+  elementRef: Reference<any>;
+  frameElement: any;
+}) {
+  let frameEffects = [];
+  const initProps = {
+    frame: {
+      size: containerRef().size(),
+      pos: containerRef().position(),
+      radius: containerRef().radius(),
+      scale: containerRef().scale(),
+      rotation: containerRef().rotation(),
+    },
+    element: {
+      size: containerRef().size(),
+      pos: elementRef().position(),
+      scale: elementRef().scale(),
+    },
+  };
+
+  for (let i = 0; i < frameElement?.frameEffects?.length; i++) {
+    const frameEffect = frameElement.frameEffects[i];
+    const restore =
+      i === frameElement.frameEffects.length - 1 ||
+      frameElement.frameEffects[i + 1].s !== frameEffect.e;
+    frameEffects.push(
+      getFrameEffect({
+        containerRef,
+        elementRef,
+        frameEffect,
+        element: frameElement,
+        restore,
+        initProps,
+      })
+    );
+  }
+  yield* all(...frameEffects);
+}
+
+function* getFrameEffect({
+  containerRef,
+  elementRef,
+  frameEffect,
+  element,
+  initProps,
+  restore = true,
+}: {
+  containerRef: Reference<any>;
+  elementRef: Reference<any>;
+  frameEffect: FrameEffect;
+  element: any;
+  initProps: any;
+  restore?: boolean;
+}) {
+  yield* waitFor(frameEffect.s);
+  const props = frameEffect.props;
+  const sequence = [];
+
+  if (restore) {
+    sequence.push(
+      restoreState({
+        containerRef,
+        elementRef,
+        frameEffect,
+        element,
+        initProps,
+      })
+    );
+  }
+
+  sequence.push(
+    containerRef().size(
+      props.frameSize,
+      props.transitionDuration,
+      getTimingFunction(props.transitionEasing ?? DEFAULT_TIMING_FUNCTION)
+    )
+  );
+
+  sequence.push(
+    containerRef().position(
+      props.framePosition ?? { x: 0, y: 0 },
+      props.transitionDuration,
+      getTimingFunction(props.transitionEasing ?? DEFAULT_TIMING_FUNCTION)
+    )
+  );
+
+  sequence.push(
+    elementRef().position(
+      props.elementPosition ?? DEFAULT_POSITION,
+      props.transitionDuration,
+      getTimingFunction(props.transitionEasing ?? DEFAULT_TIMING_FUNCTION)
+    )
+  );
+
+  switch (props.frameShape) {
+    case FRAME_SHAPE.CIRCLE:
+      sequence.push(
+        containerRef().radius(
+          props.frameSize[0] / 2,
+          props.transitionDuration,
+          getTimingFunction(props.transitionEasing ?? DEFAULT_TIMING_FUNCTION)
+        )
+      );
+      break;
+    default:
+      sequence.push(
+        containerRef().radius(
+          props.radius ?? 0,
+          props.transitionDuration,
+          getTimingFunction(props.transitionEasing ?? DEFAULT_TIMING_FUNCTION)
+        )
+      );
+  }
+
+  if (props.objectFit) {
+    sequence.push(
+      fitElement({
+        elementRef,
+        containerSize: { x: props.frameSize[0], y: props.frameSize[1] },
+        elementSize: initProps.element.size,
+        objectFit: props.objectFit,
+      })
+    );
+  }
+  yield* all(...sequence);
+}
+
+function* restoreState({
+  containerRef,
+  elementRef,
+  frameEffect,
+  element,
+  initProps,
+}: {
+  containerRef: Reference<any>;
+  elementRef: Reference<any>;
+  frameEffect: any;
+  element: any;
+  initProps: any;
+}) {
+  yield* waitFor(frameEffect.e - frameEffect.s);
+  let sequence = [];
+  sequence.push(containerRef().size(initProps.frame.size));
+  sequence.push(containerRef().position(initProps.frame.pos));
+  sequence.push(containerRef().scale(initProps.frame.scale));
+  sequence.push(containerRef().rotation(initProps.frame.rotation));
+  sequence.push(containerRef().radius(initProps.frame.radius));
+  sequence.push(elementRef().size(initProps.element.size));
+  sequence.push(elementRef().position(initProps.element.pos));
+  sequence.push(elementRef().scale(initProps.element.scale));
+  sequence.push(
+    fitElement({
+      elementRef,
+      containerSize: initProps.frame.size,
+      elementSize: initProps.element.size,
+      objectFit: element.objectFit ?? "none",
+    })
+  );
+  yield* all(...sequence);
+}