@realsee/dnalogel 3.69.2 → 3.69.3-alpha.1

package/libs/DigitalHumanPlugin/index.js

@@ -0,0 +1,318 @@
+var b = Object.defineProperty;
+var L = (h, t, e) => t in h ? b(h, t, { enumerable: !0, configurable: !0, writable: !0, value: e }) : h[t] = e;
+var n = (h, t, e) => (L(h, typeof t != "symbol" ? t + "" : t, e), e);
+var y = (h, t, e) => new Promise((s, i) => {
+  var r = (f) => {
+    try {
+      g(e.next(f));
+    } catch (A) {
+      i(A);
+    }
+  }, T = (f) => {
+    try {
+      g(e.throw(f));
+    } catch (A) {
+      i(A);
+    }
+  }, g = (f) => f.done ? s(f.value) : Promise.resolve(f.value).then(r, T);
+  g((e = e.apply(h, t)).next());
+});
+import { GLTFLoader as x } from "@realsee/five/gltf-loader";
+import { DigitalStateMachine as G } from "./stateMachine.js";
+import * as a from "three";
+const H = new a.TextureLoader();
+new a.Matrix4().fromArray([1, 0, 0, 0, 0, 0, -1, 0, 0, 1, 0, 0, 0, 0, 0, 1]);
+const C = 0.5;
+class B {
+  constructor() {
+    n(this, "tracePoints");
+    n(this, "curve");
+    this.tracePoints = [], this.curve = new a.CatmullRomCurve3(this.tracePoints);
+  }
+  addPoint(t) {
+    this.tracePoints.push(t), this.curve = new a.CatmullRomCurve3(this.tracePoints, !1);
+  }
+  clear() {
+    this.tracePoints = [], this.curve = new a.CatmullRomCurve3(this.tracePoints, !1);
+  }
+}
+class S {
+  constructor(t, e) {
+    n(this, "model");
+    n(this, "keyframesManager");
+    n(this, "keyframes");
+    n(this, "items");
+    n(this, "effects");
+    n(this, "digitalPlayGroud");
+    n(this, "mixer");
+    n(this, "action");
+    n(this, "state");
+    n(this, "animationFrameId");
+    n(this, "itemMatrixMap");
+    n(this, "trace");
+    n(this, "curveLine");
+    n(this, "effectsGroup");
+    n(this, "effectObjects");
+    n(this, "container");
+    this.model = t, this.model.visible = !1, this.model.mixer = null, this.mixer = new a.AnimationMixer(this.model), this.action = null, this.keyframesManager = new G(), this.keyframes = [], this.items = [], this.effects = [], this.digitalPlayGroud = e, this.state = {
+      currentTime: 0,
+      currentAnimation: null
+    }, this.animationFrameId = null, this.itemMatrixMap = /* @__PURE__ */ new Map(), this.trace = new B(), this.curveLine = new a.Line(), this.curveLine.material = new a.LineBasicMaterial({ color: 14467195, linewidth: 5 }), this.effectsGroup = new a.Group(), this.effectObjects = /* @__PURE__ */ new Map(), this.container = new a.Group(), this.container.add(this.model), this.container.add(this.curveLine), this.container.add(this.effectsGroup);
+  }
+  playAnimation(t, e, s) {
+    const i = this.action, r = this.mixer.clipAction(t);
+    r.reset(), e || (r.setLoop(a.LoopOnce, 1), r.clampWhenFinished = !0), i && (r.enabled = !0, r.setEffectiveTimeScale(1), r.crossFadeFrom(i, s / 1e3, !0)), r.play(), i && setTimeout(() => {
+      i.stop();
+    }, s), this.action = r;
+  }
+  pauseAnimation() {
+    this.action && (this.action.paused = !0);
+  }
+  stopAnimation() {
+    this.state.currentAnimation = null, this.mixer.stopAllAction();
+  }
+  attachItem() {
+  }
+  init() {
+    this.state.currentTime = 0, this.model.position.copy(new a.Vector3().fromArray(this.keyframes[0].position)), this.model.quaternion.copy(new a.Quaternion().fromArray(this.keyframes[0].quaternion)), this.model.visible = this.keyframes[0].visible;
+  }
+  reset() {
+    this.state.currentTime = 0, this.model.position.copy(new a.Vector3().fromArray(this.keyframes[0].position)), this.model.quaternion.copy(new a.Quaternion().fromArray(this.keyframes[0].quaternion)), this.model.visible = !1, this.container.visible = !1, this.animationFrameId && cancelAnimationFrame(this.animationFrameId), this.stopAnimation(), this.clearTrace();
+  }
+  play(t) {
+    return y(this, null, function* () {
+      const { keyframes: e, items: s, effects: i } = t || {};
+      return this.keyframes = e || this.keyframes, this.items = s || this.items, this.effects = i || this.effects, new Promise((r, T) => {
+        this.state.currentTime || this.init();
+        let g = null;
+        for (let m = 0; m < this.items.length; m++) {
+          const c = this.digitalPlayGroud.getDigitalItem(this.items[m].model);
+          if (c) {
+            let o = null;
+            const p = this.model.children[0].children.find((u) => u instanceof a.Bone), M = this.model.children[0].children.find((u) => u instanceof a.SkinnedMesh);
+            if (!p || !M)
+              throw new Error("模型骨骼缺失");
+            if (this.model.children[0].children[2].traverse((u) => {
+              u instanceof a.Bone && u.name === "RightHand" && (o = u);
+            }), !o)
+              throw new Error("没有找到绑定的骨骼");
+            o.add(c);
+            const l = new a.Vector3(), d = new a.Quaternion(), w = new a.Vector3();
+            new a.Matrix4().fromArray(this.items[m].initialMatrix).decompose(l, d, w), c.position.copy(l), c.quaternion.copy(d), c.scale.copy(w), c.updateMatrixWorld(!0);
+          }
+        }
+        const f = /* @__PURE__ */ new Map();
+        for (let m = 0; m < this.effects.length; m++) {
+          const c = this.effects[m];
+          if (c.type === "frameSequence") {
+            const o = c.keyframes;
+            for (let p = 0; p < o.length; p++)
+              H.load(o[p].image, (M) => {
+                M.name = o[p].image, f.set(o[p].image, M);
+              });
+          }
+        }
+        const A = (m) => {
+          var M;
+          g === null && (g = m);
+          const c = m - g;
+          g = m, this.state.currentTime += c;
+          const o = this.keyframesManager.getStateByTime(this.state.currentTime, this.keyframes, "model");
+          this.trace.addPoint(o.translation);
+          const p = new a.BufferGeometry().setFromPoints(this.trace.tracePoints);
+          if (this.curveLine.geometry = p, this.model.position.copy(o.translation), this.model.quaternion.copy(o.quaternion), this.model.scale.copy(o.scale), this.model.visible = o.visible, o.isFinal) {
+            r(!0), this.animationFrameId !== null && (cancelAnimationFrame(this.animationFrameId), this.animationFrameId = null);
+            return;
+          } else if (o.animation) {
+            const l = this.digitalPlayGroud.getAnimation(o.animation.url);
+            l !== this.state.currentAnimation && (this.state.currentAnimation = l, this.playAnimation(
+              l,
+              o.animation.repeat,
+              (M = o.animation.switchDuration) != null ? M : C
+            )), this.mixer.update(c / 1e3), o.animation.isOver === !0 && (this.action.paused = !0, this.action.time = this.state.currentAnimation.duration, this.mixer.update(0));
+          }
+          for (let l = 0; l < this.items.length; l++) {
+            const d = this.digitalPlayGroud.getDigitalItem(this.items[l].model);
+            d && (this.keyframesManager.getStateByTime(
+              this.state.currentTime,
+              this.items[l].keyframes,
+              "item"
+            ).visible === !0 ? d.visible = !0 : d.visible = !1);
+          }
+          for (let l = 0; l < this.effects.length; l++) {
+            const d = this.effects[l];
+            if (d.type === "frameSequence") {
+              const w = d.type + l;
+              if (!this.effectObjects.has(w)) {
+                const I = new a.PlaneGeometry(1, 1), F = new a.MeshBasicMaterial({
+                  transparent: !0
+                }), P = new a.Mesh(I, F);
+                P.rotateX(-Math.PI / 2), P.position.copy(new a.Vector3().fromArray(d.position)), this.effectObjects.set(w, P), this.effectsGroup.add(P);
+              }
+              const k = this.effectObjects.get(w), u = this.keyframesManager.getStateByTime(
+                this.state.currentTime,
+                this.effects[l].keyframes,
+                "effect"
+              ), v = f.get(u.image);
+              k.material.map = v, k.visible = u.visible;
+            }
+          }
+          this.model.needsRender = !0, this.animationFrameId = requestAnimationFrame(A);
+        };
+        this.animationFrameId = requestAnimationFrame(A);
+      });
+    });
+  }
+  // 用于分段时 更新轨迹
+  traceUpDate() {
+    this.trace.clear();
+    for (let t = 0; t <= this.state.currentTime; t += 100) {
+      const e = this.keyframesManager.getStateByTime(t, this.keyframes, "model");
+      this.trace.addPoint(e.translation);
+      const s = new a.BufferGeometry().setFromPoints(this.trace.tracePoints);
+      this.curveLine.geometry = s;
+    }
+  }
+  // 清楚轨迹
+  clearTrace() {
+    this.trace.clear();
+    const t = new a.BufferGeometry().setFromPoints(this.trace.tracePoints);
+    this.curveLine.geometry = t;
+  }
+}
+class E {
+  constructor() {
+    n(this, "gltfLoader");
+    n(this, "digitalHumanMap");
+    n(this, "digitalItemMap");
+    n(this, "skeletalAnimationMap");
+    n(this, "script");
+    n(this, "currentChapter");
+    this.gltfLoader = new x(), this.digitalHumanMap = /* @__PURE__ */ new Map(), this.digitalItemMap = /* @__PURE__ */ new Map(), this.skeletalAnimationMap = /* @__PURE__ */ new Map(), this.script = null, this.currentChapter = null;
+  }
+  setScript(t) {
+    this.script = t;
+  }
+  loadModel(t) {
+    return y(this, null, function* () {
+      if (this.digitalHumanMap.has(t)) {
+        const e = this.digitalHumanMap.get(t);
+        return new Promise((s, i) => {
+          s(e);
+        });
+      } else
+        return new Promise((e, s) => {
+          t.endsWith(".gltf") || t.endsWith(".glb") ? this.gltfLoader.load(t, ({ scene: i }) => {
+            const r = new S(i, this);
+            this.digitalHumanMap.set(t, r), e(r);
+          }) : s("不支持的模型格式");
+        });
+    });
+  }
+  loadModels(t) {
+    return y(this, null, function* () {
+      return Promise.all(t.map((e) => this.loadModel(e)));
+    });
+  }
+  loadItems(t) {
+    return y(this, null, function* () {
+      return Promise.all(t.map((e) => this.loadItem(e)));
+    });
+  }
+  loadItem(t) {
+    return y(this, null, function* () {
+      if (this.digitalItemMap.has(t)) {
+        const e = this.digitalItemMap.get(t);
+        return new Promise((s, i) => {
+          s(e);
+        });
+      } else
+        return new Promise((e, s) => {
+          t.endsWith(".gltf") || t.endsWith(".glb") ? this.gltfLoader.load(t, ({ scene: i }) => {
+            this.digitalItemMap.set(t, i), e(i);
+          }) : s("不支持的模型格式");
+        });
+    });
+  }
+  //加载骨骼动画，暂且认为一个glb只包含一个骨骼动画
+  loadAnimation(t) {
+    return y(this, null, function* () {
+      if (this.skeletalAnimationMap.has(t)) {
+        const e = this.skeletalAnimationMap.get(t);
+        return new Promise((s, i) => {
+          s(e);
+        });
+      } else
+        return new Promise((e, s) => {
+          this.gltfLoader.load(t, ({ animations: i }) => {
+            const r = i[0];
+            this.skeletalAnimationMap.set(t, r), e(r);
+          });
+        });
+    });
+  }
+  loadAnimations(t) {
+    return y(this, null, function* () {
+      return Promise.all(t.map((e) => this.loadAnimation(e)));
+    });
+  }
+  getAnimation(t) {
+    return this.skeletalAnimationMap.get(t) || null;
+  }
+  getDigitalHuman(t) {
+    return this.digitalHumanMap.get(t) || null;
+  }
+  getDigitalItem(t) {
+    return this.digitalItemMap.get(t) || null;
+  }
+  playByChapter(t) {
+    return y(this, null, function* () {
+      this.currentChapter = t;
+      const e = t.players.filter((s) => s.type === "model");
+      for (let s = 0; s < e.length; s++) {
+        const i = e[s], r = this.digitalHumanMap.get(i.model);
+        console.log("zff ~ DigitalPlayGroud ~ playByChapter ~ digitalPlayer:", r), r && r.play({ keyframes: i.keyframes, items: i.items, effects: i.effects });
+      }
+    });
+  }
+  pause() {
+    if (!this.currentChapter)
+      return;
+    const t = this.currentChapter.players.filter((e) => e.type === "model");
+    for (let e = 0; e < t.length; e++) {
+      const s = t[e], i = this.digitalHumanMap.get(s.model);
+      i != null && i.animationFrameId && cancelAnimationFrame(i.animationFrameId);
+    }
+  }
+  reset() {
+    this.digitalHumanMap.forEach((t) => {
+      t.reset();
+    });
+  }
+  play() {
+    this.currentChapter && this.digitalHumanMap.forEach((t) => {
+      t.play();
+    });
+  }
+  // 设置时间轴
+  setTimelineTo(t) {
+    this.currentChapter && this.digitalHumanMap.forEach((e) => {
+      e.state.currentTime = t, e.traceUpDate();
+    });
+  }
+  // 清楚单个数字人轨迹
+  clearTrace(t) {
+    var e;
+    (e = this.digitalHumanMap.get(t)) == null || e.clearTrace();
+  }
+  // 清楚所有轨迹
+  clearAllTrace() {
+    this.digitalHumanMap.forEach((t) => {
+      t.clearTrace();
+    });
+  }
+}
+export {
+  S as DigitalHuman,
+  E as DigitalPlayGroud
+};

package/libs/DigitalHumanPlugin/stateMachine.d.ts

@@ -0,0 +1,142 @@
+import * as THREE from 'three';
+interface DigitalScript {
+    name: string;
+    uuid: string;
+    assets: {
+        models: string[];
+        animations: string[];
+    };
+    chapters: DigitalChapter[];
+}
+interface DigitalDataPlayer {
+    name: string;
+    type: 'data';
+    data: DigitalModelKeyframe[];
+}
+interface DigitalModelPlayerItem {
+    name: string;
+    initialMatrix: number[];
+    attachedBone: string;
+    model: string;
+    keyframes: DigitalItemKeyframe[];
+}
+interface DigitalEffectKeyframe {
+    timeStamp: number;
+    image: string;
+    visible: boolean;
+}
+interface DigitatModelPlayerEffect {
+    type: 'frameSequence';
+    position: number[];
+    keyframes: DigitalEffectKeyframe[];
+}
+interface DigitalModelPlayer {
+    name: string;
+    type: 'model';
+    model: string;
+    keyframes: DigitalModelKeyframe[];
+    items: DigitalModelPlayerItem[];
+    effects: DigitatModelPlayerEffect[];
+}
+interface DigitalSubTitlePlayer {
+    name: string;
+    type: 'subTitle';
+    keyframes: DigitalSubTitleKeyframe[];
+}
+interface DigitalCameraPlayer {
+    name: string;
+    type: 'camera';
+    keyframes: DigitalCameraKeyframe[];
+}
+interface DigitalChapter {
+    name: string;
+    players: Array<DigitalModelPlayer | DigitalSubTitlePlayer | DigitalCameraPlayer>;
+}
+interface DigitalItemKeyframe {
+    timeStamp: number;
+    visible: boolean;
+}
+interface DigitalCameraKeyframe {
+    timeStamp: number;
+    matrix: number[];
+    fov: number;
+}
+interface DigitalSubTitleKeyframe {
+    visible: boolean;
+    timeStamp: number;
+    title: string;
+    content: string;
+    audio?: string;
+    image?: string;
+}
+interface DigitalModelKeyframe {
+    visible: boolean;
+    timeStamp: number;
+    animation?: {
+        url: string;
+        repeat: boolean;
+        isOver?: boolean;
+        switchDuration?: number;
+    };
+    position: number[];
+    interpolationType?: 'linear' | 'parabola';
+    quaternion: number[];
+    scale: number[];
+    adddon?: {
+        text?: string;
+        audio?: string;
+    };
+}
+interface DigitalDataKeyframe {
+    timeStamp: number;
+    data: any;
+}
+interface DigitalModelState {
+    translation: THREE.Vector3;
+    quaternion: THREE.Quaternion;
+    scale: THREE.Vector3;
+    animation?: DigitalModelKeyframe['animation'];
+    keyframeIndex: number;
+    visible: boolean;
+    isFinal: boolean;
+}
+interface DigitalSubTitleState {
+    title: string;
+    content: string;
+    audio?: string;
+    image?: string;
+    keyframeIndex: number;
+}
+interface DigitalCameraState {
+    matrix: number[];
+    fov: number;
+    transitionType: 'linear' | 'instant';
+}
+interface DigitalDataState {
+    currentFrameData: DigitalDataKeyframe['data'];
+    nextFrameData: DigitalDataKeyframe['data'] | null;
+    progress: number;
+}
+interface DigitalItemState {
+    timeStamp: number;
+    visible: boolean;
+}
+interface DigitalEffectState {
+    image: string;
+    visible: boolean;
+}
+type KeyframeType<T> = T extends 'model' ? DigitalModelKeyframe : T extends 'camera' ? DigitalCameraKeyframe : T extends 'subtitle' ? DigitalSubTitleKeyframe : T extends 'data' ? DigitalDataKeyframe : T extends 'item' ? DigitalItemKeyframe : T extends 'effect' ? DigitalEffectKeyframe : never;
+type StateType<T> = T extends 'model' ? DigitalModelState : T extends 'camera' ? DigitalCameraState : T extends 'subtitle' ? DigitalSubTitleState : T extends 'data' ? DigitalDataState : T extends 'item' ? DigitalItemState : T extends 'effect' ? DigitalEffectState : never;
+declare class DigitalStateMachine {
+    constructor();
+    getStateByTime<T extends 'model' | 'camera' | 'subtitle' | 'data' | 'item' | 'effect'>(time: number, keyframes: Array<KeyframeType<T>>, type: T): StateType<T>;
+    private getDataStateByTime;
+    private getEffectStateByTime;
+    private getItemStateByTime;
+    private getCameraStateByTime;
+    private getSubtitleStateByTime;
+    private getModelStateByTime;
+    private getKeyframeIndexByTime;
+}
+export { DigitalStateMachine };
+export type { DigitatModelPlayerEffect, DigitalEffectKeyframe, DigitalDataState, DigitalModelPlayerItem, DigitalDataPlayer, DigitalCameraKeyframe, DigitalChapter, DigitalModelKeyframe, DigitalScript, DigitalSubTitleKeyframe, DigitalModelPlayer, DigitalSubTitlePlayer, DigitalCameraPlayer, DigitalModelState, DigitalSubTitleState, DigitalCameraState, KeyframeType, StateType, };

package/libs/DigitalHumanPlugin/stateMachine.js

@@ -0,0 +1,153 @@
+import * as n from "three";
+class T {
+  constructor() {
+  }
+  getStateByTime(r, t, e) {
+    switch (e) {
+      case "model":
+        return this.getModelStateByTime(r, t);
+      case "camera":
+        return this.getCameraStateByTime(r, t);
+      case "subtitle":
+        return this.getSubtitleStateByTime(r, t);
+      case "data":
+        return this.getDataStateByTime(r, t);
+      case "item":
+        return this.getItemStateByTime(r, t);
+      case "effect":
+        return this.getEffectStateByTime(r, t);
+      default:
+        throw new Error(`Unsupported type: ${e}`);
+    }
+  }
+  getDataStateByTime(r, t) {
+    const e = this.getKeyframeIndexByTime(r, t);
+    if (e === t.length - 1)
+      return {
+        currentFrameData: t[e].data,
+        nextFrameData: null,
+        progress: 0
+      };
+    {
+      const a = t[e].data, o = t[e + 1].data, i = t[e + 1].timeStamp - t[e].timeStamp, l = (r - t[e].timeStamp) / i;
+      return {
+        currentFrameData: a,
+        nextFrameData: o,
+        progress: l
+      };
+    }
+  }
+  getEffectStateByTime(r, t) {
+    const e = this.getKeyframeIndexByTime(r, t);
+    return {
+      image: t[e].image,
+      visible: t[e].visible
+    };
+  }
+  getItemStateByTime(r, t) {
+    const e = this.getKeyframeIndexByTime(r, t);
+    return {
+      timeStamp: t[e].timeStamp,
+      visible: t[e].visible
+    };
+  }
+  getCameraStateByTime(r, t) {
+    const e = this.getKeyframeIndexByTime(r, t);
+    return {
+      matrix: t[e].matrix,
+      fov: t[e].fov,
+      transitionType: "linear"
+    };
+  }
+  getSubtitleStateByTime(r, t) {
+    const e = this.getKeyframeIndexByTime(r, t);
+    return {
+      title: t[e].title,
+      content: t[e].content,
+      audio: t[e].audio,
+      image: t[e].image,
+      keyframeIndex: e
+    };
+  }
+  getModelStateByTime(r, t) {
+    var l;
+    const e = this.getKeyframeIndexByTime(r, t);
+    if (e === t.length - 1)
+      return {
+        translation: new n.Vector3().fromArray(t[e].position),
+        quaternion: new n.Quaternion().fromArray(t[e].quaternion),
+        scale: new n.Vector3().fromArray(t[e].scale),
+        keyframeIndex: e,
+        animation: t[e].animation,
+        visible: t[e].visible,
+        isFinal: !0
+      };
+    const a = t[e], o = t[e + 1], i = r - a.timeStamp;
+    if (((l = o.interpolationType) != null ? l : "linear") === "parabola") {
+      const u = o.timeStamp - a.timeStamp, s = Math.min(i / u, 1), p = S(
+        new n.Vector3().fromArray(a.position),
+        new n.Vector3().fromArray(o.position)
+      )(s), y = s, w = new n.Quaternion().copy(
+        new n.Quaternion().fromArray(a.quaternion).slerp(new n.Quaternion().fromArray(o.quaternion), y)
+      ), x = new n.Vector3().lerpVectors(
+        new n.Vector3().fromArray(a.scale),
+        new n.Vector3().fromArray(o.scale),
+        s
+      );
+      return {
+        translation: p,
+        quaternion: w,
+        scale: x,
+        keyframeIndex: e,
+        animation: a.animation,
+        visible: a.visible,
+        isFinal: !1
+      };
+    } else {
+      const u = o.timeStamp - a.timeStamp, s = Math.min(i / u, 1), d = new n.Vector3().lerpVectors(
+        new n.Vector3().fromArray(a.position),
+        new n.Vector3().fromArray(o.position),
+        s
+      ), p = s, y = new n.Quaternion().copy(
+        new n.Quaternion().fromArray(a.quaternion).slerp(new n.Quaternion().fromArray(o.quaternion), p)
+      ), w = new n.Vector3().lerpVectors(
+        new n.Vector3().fromArray(a.scale),
+        new n.Vector3().fromArray(o.scale),
+        s
+      );
+      return {
+        translation: d,
+        quaternion: y,
+        scale: w,
+        keyframeIndex: e,
+        animation: a.animation,
+        visible: a.visible,
+        isFinal: !1
+      };
+    }
+  }
+  getKeyframeIndexByTime(r, t) {
+    r < 0 && (r = 0);
+    let e = 0, a = t.length - 1, o = -1;
+    for (; e <= a; ) {
+      const i = Math.floor((e + a) / 2);
+      if (t[i].timeStamp === r) {
+        o = i;
+        break;
+      } else
+        t[i].timeStamp < r ? e = i + 1 : a = i - 1;
+    }
+    return o !== -1 ? o : a;
+  }
+}
+function S(c, r) {
+  return (e) => {
+    const a = Math.sqrt(Math.pow(r.x - c.x, 2) + Math.pow(r.z - c.z, 2)), o = e * a, i = new n.Vector2(c.x, c.z).add(
+      new n.Vector2(r.x, r.z).sub(new n.Vector2(c.x, c.z)).normalize().multiplyScalar(o)
+    ), m = (r.y - c.y) / (Math.pow(r.x - c.x, 2) + Math.pow(r.z - c.z, 2)) * Math.pow(o, 2) + c.y;
+    return new n.Vector3(i.x, m, i.y);
+  };
+}
+export {
+  T as DigitalStateMachine
+};