@volcengine/veplayer 1.15.1 → 1.15.2-rc.1

package/src/plugins/external/aiSubtitleIconPlugin.ts

@@ -0,0 +1,270 @@
+import { Plugin, Util, Sniffer } from 'xgplayer';
+import OpenIcon from '../../assets/icons/pc/subtitleopen.svg';
+import CloseIcon from '../../assets/icons/pc/subtitleclose.svg';
+import OptionList from 'xgplayer/es/plugins/common/optionList';
+import LiveSubTitlesPlugin from './aiSubtitlePlugin';
+
+const { POSITIONS } = Plugin;
+
+export default class LiveSubtitlesIconPlugin extends Plugin {
+  static get pluginName() {
+    return 'LiveSubtitlesIconPlugin';
+  }
+
+  static get defaultConfig() {
+    return {
+      position: POSITIONS.CONTROLS_RIGHT,
+      index: 2.7,
+      defaultOpen: true,
+      default: 'en',
+      list: [
+        {
+          key: 'zh',
+          showText: '中文',
+        },
+        {
+          key: 'en',
+          showText: 'En',
+        },
+      ],
+    };
+  }
+
+  public isMobile = Sniffer.device === 'mobile';
+  public isOpenState = true;
+  public optionList: any = null;
+  public currentLang = '';
+
+  constructor(configs: any) {
+    super(configs);
+  }
+
+  afterCreate() {
+    this.appendChild('.xgplayer-icon', (this.icons as any).aiSubOpen);
+    this.appendChild('.xgplayer-icon', (this.icons as any).aiSubClose);
+    this.renderOptionList();
+    this.initEvents();
+
+    this.currentLang = this.config.default;
+
+    if (this.config.defaultOpen) {
+      this.open(true);
+    } else {
+      this.close();
+    }
+  }
+
+  afterPlayerInit() {
+    // 播放器调用start初始化播放源之后的逻辑
+    if (this.config.defaultOpen) {
+      this.changeSubTitleLang(this.config.default, true);
+    }
+  }
+
+  registerIcons() {
+    return {
+      aiSubOpen: {
+        icon: OpenIcon,
+        class: 'xgplayer-aisub-open-svg',
+      },
+      aiSubClose: {
+        icon: CloseIcon,
+        class: 'xgplayer-aisub-close-svg',
+      },
+    };
+  }
+
+  registerLanguageTexts() {
+    return {
+      sub: {
+        jp: 'AI 字幕',
+        en: 'AI Subtitles',
+        zh: 'AI 字幕',
+        'zh-hk': 'AI 字幕',
+      },
+      subOpen: {
+        jp: '字幕をオンにする',
+        en: 'Show subtitles',
+        zh: '开启字幕',
+        'zh-hk': '開啟字幕',
+      },
+      subClose: {
+        jp: 'クローズドキャプション',
+        en: 'Hide subtitles',
+        zh: '关闭字幕',
+        'zh-hk': '關閉字幕',
+      },
+    };
+  }
+
+  children() {
+    return {
+      LiveSubTitlesPlugin: {
+        plugin: LiveSubTitlesPlugin,
+        options: {
+          root: this.player.root,
+          showLang: this.currentLang,
+        },
+      },
+    };
+  }
+
+  initEvents() {
+    this.bind(['click'], this.handleButtonChange.bind(this));
+    this.bind(['mouseenter'], this.showPanel.bind(this));
+    this.bind(['mouseleave'], this.hidePanel.bind(this));
+  }
+
+  handleButtonChange() {
+    this.close = this.close.bind(this);
+    this.open = this.open.bind(this);
+
+    if (this.isMobile) {
+      this.showMobilePanel();
+    } else {
+      const fn = this.isOpenState ? this.close : this.open;
+      fn();
+    }
+  }
+
+  showMobilePanel() {
+    const $content = this.renderItemList();
+    (this.player as any).panel?.showPanel(
+      $content,
+      (this.langText as any).sub,
+      true,
+    );
+  }
+
+  renderItemList() {
+    const list = [...this.config.list];
+    const $content = Util.createDom('div', '', {}, 'ai-panel-content');
+
+    list.push({
+      key: '',
+      showText: (this.langText as any).subClose,
+    });
+
+    list.forEach((item: any) => {
+      const $item = this.renderItem(item);
+      $content.appendChild($item);
+    });
+
+    return $content;
+  }
+
+  renderItem(item: any) {
+    const isSelect =
+      (item.key === '' && !this.isOpenState) ||
+      (this.currentLang === item.key && this.isOpenState);
+
+    const $item = Util.createDom(
+      'div',
+      '',
+      {},
+      `panel-Item ${isSelect ? 'select-in' : ''}`,
+    );
+
+    $item.innerHTML = item.showText;
+    $item.addEventListener('click', e => {
+      e.stopPropagation();
+      e.preventDefault();
+      this.changeSubTitleLang(item.key);
+      this.updatePanel();
+    });
+
+    return $item;
+  }
+
+  updatePanel() {
+    const children = this.renderItemList();
+    (this.player as any).panel?.updatePanel(children);
+  }
+
+  showPanel() {
+    !this.isMobile && this.isOpenState && this.optionList?.show();
+  }
+
+  hidePanel() {
+    !this.isMobile && this.isOpenState && this.optionList?.hide();
+  }
+
+  open(isInit = false) {
+    this.isOpenState = true;
+    Util.removeClass(this.root, 'isClose');
+    Util.addClass(this.root, 'isOpen');
+    !isInit && this.showPanel();
+    this.toggleSubTitle();
+  }
+
+  close() {
+    this.isOpenState = false;
+    Util.removeClass(this.root, 'isOpen');
+    Util.addClass(this.root, 'isClose');
+    this.optionList?.hide();
+    this.toggleSubTitle();
+    // this.hidePanel();
+  }
+
+  renderOptionList() {
+    if (this.config.list?.length < 2 || this.isMobile) {
+      return;
+    }
+
+    const list = this.config.list.map((item: any) => {
+      if (item.key === this.config.default) {
+        item.isCurrent = true;
+        item.selected = true;
+      }
+      return item;
+    });
+    const options = {
+      config: {
+        data: list,
+        className: '',
+        onItemClick: (e: any, data: any) => {
+          e.stopPropagation();
+          e.preventDefault();
+          this.changeSubTitleLang(data.to?.key);
+        },
+      },
+      root: this.root,
+    };
+
+    this.optionList = new OptionList(options);
+  }
+
+  changeSubTitleLang(key: string, isInit = false) {
+    this.currentLang = key;
+    if (!key) {
+      this.close();
+    } else {
+      this.open(isInit);
+    }
+  }
+
+  toggleSubTitle() {
+    this.player
+      .getPlugin('LiveSubtitlesPlugin')
+      ?.setCurrentLang(this.isOpenState ? this.currentLang : '');
+  }
+
+  destroy() {
+    this.unbind(['click'], this.handleButtonChange.bind(this));
+    this.unbind(['mouseenter'], this.showPanel.bind(this));
+    this.unbind(['mouseleave'], this.hidePanel.bind(this));
+    this.optionList?.destroy();
+  }
+
+  render() {
+    return `
+    <xg-icon class="xgplayer-aisub">
+      <div class="xgplayer-icon">
+      </div>
+      <div class="xg-tips" lang-key="subOpen">${
+        (this.langText as any).subOpen
+      }</div>
+    </xg-icon>
+    `;
+  }
+}

package/src/plugins/external/aiSubtitlePlugin.ts

@@ -0,0 +1,452 @@
+import { PluginEvents } from '@/constants/event';
+import { Uint8ArrayToString } from '@/utils/u8a';
+import { Plugin, Events } from 'xgplayer';
+import { VE_DEBUG } from '@/utils';
+
+interface ISubTitles {
+  timestamp: number;
+  text: string;
+  duration: number;
+  url: string;
+  lang: string;
+  delimiter?: string;
+  textArr?: string[];
+  wordpiece?: any[];
+}
+
+interface ISubTitlesState {
+  showLang: string;
+  subTitleList: ISubTitles[];
+  rtmpDtsGap: number;
+  showingSubtitle: string;
+  currentSub: any;
+  dtsBase: number;
+}
+
+class LiveSubTitlesPlugin extends Plugin {
+  static get pluginName() {
+    return 'LiveSubtitlesPlugin';
+  }
+
+  static get defaultConfig() {
+    return {
+      position: Plugin.POSITIONS.ROOT,
+      showLang: '',
+    };
+  }
+
+  constructor(configs: any) {
+    super(configs);
+    this.state.showLang = configs.showLang;
+  }
+
+  public state: ISubTitlesState = {
+    showLang: '',
+    subTitleList: [],
+    rtmpDtsGap: 0,
+    dtsBase: 0,
+    showingSubtitle: '',
+    currentSub: null,
+  };
+
+  public clearLineTimer: any = null;
+  public showTimer: any = null;
+
+  beforePlayerInit() {
+    this.on(Events.TIME_UPDATE, this.handleVideoTimeupdate.bind(this));
+    this.on(Events.PAUSE, this.onPause.bind(this));
+    this.on(Events.PLAY, this.onPlay.bind(this));
+    this.on(Events.DESTROY, this.destroy.bind(this));
+    this.on(Events.URL_CHANGE, this.clearRtmpDtsGap.bind(this));
+
+    this.on(Events.SEI_PARSED, this.handleSEIParsed.bind(this));
+    // rc版本监听
+    this.on('core_event', (data: any) => {
+      // 通过判断eventName来区分内核事件
+      if (data.eventName === 'core.sei') {
+        this.handleSEIParsed(data.sei, data.originPts);
+      }
+    });
+  }
+
+  onPause() {
+    this.clearSubTitle(false);
+  }
+
+  onPlay() {
+    this.clearSubTitle(true);
+  }
+
+  destroy() {
+    this.clearSubTitle(true);
+    this.off(Events.TIME_UPDATE, this.handleVideoTimeupdate.bind(this));
+    this.off(Events.SEI_PARSED, this.handleSEIParsed.bind(this));
+    this.off(Events.PAUSE, this.onPause.bind(this));
+    this.off(Events.PLAY, this.onPlay.bind(this));
+    this.off(Events.DESTROY, this.destroy.bind(this));
+    this.off(Events.URL_CHANGE, this.clearRtmpDtsGap.bind(this));
+  }
+
+  clearRtmpDtsGap() {
+    this.state.rtmpDtsGap = 0;
+    this.state.dtsBase = 0;
+    this.clearSubTitle(true);
+  }
+
+  clearSubTitle(isClearScreen: boolean) {
+    this.state.subTitleList = [];
+    this.state.showingSubtitle = '';
+    this.state.currentSub = null;
+    clearTimeout(this.showTimer);
+    clearTimeout(this.clearLineTimer);
+    if (isClearScreen) {
+      const $contianer = this.find('.xgplayer-subtitles-inner');
+      if ($contianer) {
+        this.resetDom($contianer);
+      }
+    }
+  }
+
+  getIsSoft() {
+    const { tagName } = (this.player.video as any) ?? {};
+    const isSoft = tagName === 'LIVE-VIDEO';
+    return isSoft;
+  }
+
+  handleVideoTimeupdate(data: any) {
+    const currentTime = (this.player.video as any).currentTime;
+    const isSoft = this.getIsSoft();
+    const curPts = data?.detail?.pts / 1000;
+    const curTime = isSoft ? curPts : currentTime;
+
+    if (!this.state.showLang) return;
+    if (!curTime) return;
+    if (!this.player.isPlaying) return;
+
+    const subTitle = this.findSubTitleByTime(curTime);
+
+    if (!subTitle) return;
+    // 重新匹配到的字幕是正在展示的
+    if (subTitle.text === this.state.showingSubtitle) return;
+    if (!subTitle.wordpiece?.length) return;
+
+    this.renderSubtitle(subTitle);
+    this.player.emit(PluginEvents.AI_SUBTITLE_MATCH, {
+      subTitle,
+      lang: this.state.showLang,
+    });
+
+    if (this.state.subTitleList.length > 1e3) {
+      this.state.subTitleList = this.state.subTitleList.filter(
+        (subTitle: ISubTitles) =>
+          subTitle.timestamp + subTitle.duration / 1000 >= curTime,
+      );
+    }
+  }
+
+  renderSubtitle(subTitle: ISubTitles) {
+    const $subContianer: any = this.root;
+    const $contianer: any = this.find('.xgplayer-subtitles-inner');
+    const $helper: any = this.find('.xgplayer-subtitles-helper');
+    if (!$contianer) return;
+
+    if (this.clearLineTimer) {
+      clearTimeout(this.clearLineTimer);
+    }
+    // 如果上一句未展示完，则把剩下的先展示完
+    if (this.showTimer) {
+      clearTimeout(this.showTimer);
+      this.showSurplus();
+    }
+
+    let subTitleDom: any = this.createNewDom($contianer);
+    this.state.showingSubtitle = subTitle.text;
+    this.state.currentSub = {
+      subObj: subTitle,
+      dom: subTitleDom,
+      index: 0,
+    };
+
+    const start = Date.now();
+    let textContent = '';
+    let i = 0;
+
+    const recursion = () => {
+      const { wordpiece = [], delimiter } = subTitle;
+      // 最后一个字
+      if (i === wordpiece.length) {
+        textContent = '';
+        // 轮训展示完成
+        this.showTimer = null;
+        // 结束后一秒清空字幕容器
+        this.clearLineTimer = setTimeout(() => {
+          this.resetDom($contianer);
+          this.clearLineTimer = null;
+        }, 1 * 1000);
+        return;
+      }
+
+      // 下一个字的剩余展示时长
+      let delayTime =
+        wordpiece[i].duration - (Date.now() - start - wordpiece[i].OffsetMs);
+      if (delayTime < 0) {
+        // 如果不足，则往后补足一个字
+        while (delayTime < 0 && i < wordpiece.length - 1) {
+          i = i + 1;
+          textContent = textContent + delimiter + wordpiece[i].text;
+          delayTime =
+            wordpiece[i].duration -
+            (Date.now() - start - wordpiece[i].OffsetMs);
+        }
+      } else {
+        textContent = textContent + delimiter + wordpiece[i].text;
+      }
+
+      // 探测该行有没有超出
+      $helper.textContent = textContent;
+      const subContianerWidth = $subContianer?.offsetWidth ?? 300;
+      if ($helper.offsetWidth > subContianerWidth) {
+        // 记录当前dom
+        subTitleDom = this.createNewDom($contianer);
+        this.state.currentSub.dom = subTitleDom;
+        textContent = wordpiece[i].text;
+      }
+
+      subTitleDom.textContent = textContent;
+      this.showTimer = setTimeout(recursion, delayTime);
+      i = i + 1;
+      // 记录当前展示至第i个字
+      this.state.currentSub.index = i;
+    };
+    recursion();
+  }
+
+  handleSEIParsed(sei: any, originPts?) {
+    const { dts, code } = sei;
+    if (code !== 5) return;
+    try {
+      const content =
+        typeof sei.content === 'string'
+          ? sei.content
+          : Uint8ArrayToString(sei.content);
+      const seiObj = JSON.parse(content);
+      const { rtmp_dts: rtmpDts } = seiObj;
+      if (rtmpDts && dts) {
+        const realDts = originPts ? originPts : dts;
+        this.state.rtmpDtsGap = rtmpDts - realDts;
+      }
+    } catch (e) {
+      // DO NOTHING
+    }
+  }
+
+  insertSubTitle(...subTitles: ISubTitles[]) {
+    subTitles.forEach(subTitle => {
+      subTitle.timestamp = this.adjustSubTitleTime(subTitle.timestamp);
+    });
+
+    this.state.subTitleList.push(...subTitles);
+  }
+
+  setCurrentLang(lang: string) {
+    this.state.showLang = lang;
+
+    // 通知外部AI字幕语言改变
+    this.player.emit(PluginEvents.AI_SUBTITLE_LANG_CHANGE, lang);
+  }
+
+  adjustSubTitleTime(time: number) {
+    const isSoft = this.getIsSoft();
+    if (isSoft) {
+      return (time - this.state.rtmpDtsGap) / 1000;
+    }
+
+    //compat alpha版本
+    const coreAlpha =
+      this.player?.plugins?.flvlive || this.player?.plugins?.hlslive;
+    if (coreAlpha) {
+      const videoLargeGap =
+        coreAlpha?._context?.getInstance('COMPATIBILITY')?._videoLargeGap ?? 0;
+
+      const remuxer = coreAlpha?._context?.getInstance('MP4_REMUXER');
+
+      //compat alpha.120以下版本
+      const videoDtsBase =
+        remuxer.remuxer?.videoDtsBase ?? remuxer?.videoDtsBase ?? 0;
+
+      this.state.dtsBase = videoDtsBase - videoLargeGap;
+
+      return (
+        (time + videoLargeGap - videoDtsBase - this.state.rtmpDtsGap) / 1000
+      );
+    }
+
+    //compat next播放器版本
+    const coreRc = this.player?.plugins?.flv || this.player?.plugins?.hls;
+    if (coreRc) {
+      const videoDtsBase =
+        coreRc.core._bufferService?._demuxer?._fixer?._baseDts;
+      this.state.dtsBase = videoDtsBase;
+      return (time - videoDtsBase - this.state.rtmpDtsGap) / 1000;
+    }
+
+    return (time - this.state.rtmpDtsGap) / 1000;
+  }
+
+  findSubTitleByTime(currentTime: number) {
+    const subTitleList = this.state.subTitleList;
+    const subIndex = subTitleList.findIndex(subTitle => {
+      const player: any = this.player;
+      const langMatch = subTitle.lang === this.state.showLang;
+      const urlMatch = (player?.__curUrl || player.config.url) === subTitle.url;
+
+      const { timestamp, duration } = subTitle;
+      // console.log(timestamp + duration / 1000, currentTime, timestamp);
+      const dtsMatch =
+        timestamp + duration / 1000 >= currentTime && timestamp <= currentTime;
+      // console.log(dtsMatch, urlMatch, langMatch);
+      return dtsMatch && urlMatch && langMatch;
+    });
+
+    if (subIndex < 0) {
+      return;
+    }
+
+    // 移出该字幕之前的字幕，须保证字幕是按时间顺序排序的
+    const deleteList = subTitleList.splice(0, subIndex + 1);
+    const matchSubTitle = deleteList[deleteList.length - 1];
+
+    const subTitle = this.splitSubTitle(matchSubTitle);
+    return subTitle;
+  }
+
+  splitSubTitle(subTitle: ISubTitles) {
+    const delimiter = subTitle.lang === 'en' ? ' ' : '';
+    const textArr = subTitle.text.split(delimiter);
+    if (!subTitle.wordpiece) {
+      const perTime = subTitle.duration / textArr.length;
+      const wordpiece = textArr.map((txt, i) => ({
+        text: txt,
+        OffsetMs: perTime * i,
+        duration: perTime,
+      }));
+
+      return {
+        ...subTitle,
+        delimiter,
+        textArr,
+        wordpiece,
+      };
+    }
+
+    return {
+      ...subTitle,
+      delimiter,
+      textArr,
+    };
+  }
+
+  showSurplus() {
+    if (!this.state.currentSub) {
+      return;
+    }
+    const { index, subObj, dom } = this.state.currentSub;
+    const { wordpiece, delimiter } = subObj ?? {};
+    if (index >= wordpiece.length) {
+      return;
+    }
+    const $helper: any = this.find('.xgplayer-subtitles-helper');
+    const $contianer: any = this.find('.xgplayer-subtitles-inner');
+    const $subContianer: any = this.root;
+    const subContianerWidth = $subContianer?.offsetWidth ?? 300;
+    const end = wordpiece.length;
+    let textContent = dom.textContent;
+    wordpiece.slice(index, end).forEach((item: any) => {
+      textContent = textContent + delimiter + item.text;
+      $helper.textContent = textContent;
+
+      if ($helper.offsetWidth > subContianerWidth) {
+        this.state.currentSub.dom = this.createNewDom($contianer);
+        textContent = item.text;
+      }
+      this.state.currentSub.dom.textContent = textContent;
+    });
+  }
+
+  createNewDom(parent: HTMLElement) {
+    if (!parent) return;
+
+    if (parent.children.length > 2 && parent.firstElementChild) {
+      parent.removeChild(parent.firstElementChild);
+    }
+    const dom = document.createElement('span');
+    parent.appendChild(dom);
+    if (parent.children.length > 2) {
+      const contains1 = parent.classList.contains('scroll-1');
+      const contains2 = parent.classList.contains('scroll-2');
+      if (contains1) {
+        parent.classList.replace('scroll-1', 'scroll-2');
+      }
+      if (contains2) {
+        parent.classList.replace('scroll-2', 'scroll-1');
+      }
+      if (!contains1 && !contains2) {
+        parent.classList.add('scroll-1');
+      }
+    }
+
+    return dom;
+  }
+
+  test() {
+    const sub = {
+      duration: 10000,
+      lang: 'zh',
+      text: '你呼吸 已改变 停滞于某段流年 离别的你我 才明白挥霍有期限,逝去的人不曾走远 像 突如其来温暖我的晴天 像 倾盆大雨我躲避的屋檐',
+      timestamp: 0,
+      url: 'xxx',
+    };
+    const subTitle = this.splitSubTitle(sub);
+    this.renderSubtitle(subTitle);
+
+    setTimeout(() => {
+      this.test();
+    }, 10000);
+  }
+
+  getPlayTime() {
+    const isSoft = this.getIsSoft();
+    this.on(Events.TIME_UPDATE, (data: any) => {
+      if (!isSoft) {
+        const info: any = {
+          currentTime: (this.player.video as any).currentTime * 1000,
+          dtsBase: this.state.dtsBase,
+          rtmpDtsGap: this.state.rtmpDtsGap,
+        };
+        info.timestamp = info.currentTime + info.dtsBase + info.rtmpDtsGap;
+        VE_DEBUG.log('直播当前时间戳：', info.timestamp, info);
+      } else {
+        const info: any = {
+          pts: data?.detail?.pts,
+          rtmpDtsGap: this.state.rtmpDtsGap,
+        };
+        info.timestamp = info.pts + info.rtmpDtsGap;
+        VE_DEBUG.log('直播当前时间戳：', info.timestamp, info);
+      }
+    });
+  }
+
+  resetDom(el: HTMLElement) {
+    if (!el) return;
+    this.state.currentSub = null;
+    if (!el) return;
+    el.innerHTML = '';
+    el.classList.remove('scroll-1', 'scroll-2');
+  }
+
+  render() {
+    return '<xg-subtitles class="xgplayer-subtitles"><xg-subtitles-inner class="xgplayer-subtitles-inner"></xg-subtitles-inner><xg-subtitles-helper class="xgplayer-subtitles-helper"></xg-helper></xg-subtitles>';
+  }
+}
+
+export default LiveSubTitlesPlugin;