@vunk/plus 1.10.2 → 1.11.0

package/components/metahuman-broadcasting/index.mjs

@@ -1,11 +1,11 @@
-import { defineComponent, ref, shallowReactive, onMounted, openBlock, createElementBlock, Fragment, createVNode, mergeProps, createBlock, createCommentVNode } from 'vue';
+import { defineComponent, ref, onMounted, openBlock, createElementBlock, Fragment, createVNode, mergeProps, withCtx, renderSlot } from 'vue';
 import { VkBroadcastingMarkdown } from '@vunk/plus/components/broadcasting-markdown';
-import { VkPixiFrameBitmap, VkPixiFrameCore } from '@vunk/plus/components/pixi-frame';
-import { StreamingInferenceService, getStremingStartData, processStreaming, blobToAudioBuffer } from '@vunk/plus/shared/audioToFrames';
-export { workerConfig } from '@vunk/plus/shared/audioToFrames';
-import { useDeferred, useModelComputed } from '@vunk/core/composables';
+import { VkPixiFrameBitmap, VkPixiFrameBitmapGenerator } from '@vunk/plus/components/pixi-frame';
+import { blobToAudioBuffer } from '@vunk/plus/shared/data';
+import { useModelComputed } from '@vunk/core/composables';
 import { setData } from '@vunk/core/shared';
 import { TickerStatus } from '@vunk/shared/enum';
+import { FeatureExtractor, InferenceEngine, loadDataset } from 'sophontalk-services';
 import { _ as _export_sfc } from '../_plugin-vue_export-helper.mjs';
 
 const props = {
@@ -58,6 +58,10 @@ const props = {
   playAfterCache: {
     type: Number,
     default: 100
+  },
+  disabled: {
+    type: Boolean,
+    default: false
   }
 };
 const emits = {
@@ -77,58 +81,47 @@ var _sfc_main = /* @__PURE__ */ defineComponent({
     const props = __props;
     const emit = __emit;
     const paragraphData = ref([]);
-    const streamingInferenceService = new StreamingInferenceService(props.modelUrl);
-    const frameUrls = ref([]);
-    const silentFrameUrls = shallowReactive([]);
-    const startedDef = useDeferred();
+    const extractor = new FeatureExtractor();
+    const engine = new InferenceEngine({
+      onFrame,
+      onError(error) {
+        console.error("Inference engine error:", error);
+      },
+      onProgress
+    });
+    const silentFrameGenerator = engine.createSilentFrameGenerator();
+    const slientFrameStatus = ref(TickerStatus.pending);
+    const frames = ref([]);
     const frameStatus = useModelComputed({
       default: TickerStatus.pending,
       key: "status"
     }, props, emit);
-    const slientFrameStatus = ref(TickerStatus.pending);
     onMounted(async () => {
-      await streamingInferenceService.when();
-      const { blendingMaskBitmap, dataset, zipBlob, zip } = await getStremingStartData({
-        datasetUrl: props.datasetUrl,
-        sourceUrl: props.sourceUrl
+      const { dataset, zipBuffer } = await loadDataset({
+        jsonUrl: props.datasetUrl,
+        zipUrl: props.sourceUrl
       });
-      loadAllSilentFrames(dataset, zip);
-      await streamingInferenceService.startStreaming({
-        blendingMaskBitmap,
+      engine.init({
+        modelPath: props.modelUrl,
         dataset,
-        zipBlob
-      }, {
-        onFrame(frame) {
-          frameUrls.value.push(frame);
-        },
-        onProgress(processed, total) {
-          if (total < props.playAfterCache && processed === total || processed === props.playAfterCache) {
-            frameStatus.value = TickerStatus.play;
-          }
-        }
+        zipBuffer
       });
-      startedDef.resolve();
+      await engine.when();
+      slientFrameStatus.value = TickerStatus.play;
     });
-    async function requestProcessStreaming(buffer) {
-      await streamingInferenceService.when();
-      await startedDef.promise;
-      try {
-        await processStreaming(buffer, {
-          onChunkComplete(result) {
-            streamingInferenceService.addChunk(result);
-          }
-        });
-      } catch (error) {
-        console.error("\u97F3\u9891\u5904\u7406\u9519\u8BEF:", error);
-      }
-    }
     async function processingParagraph(item) {
       if (!item.blob) {
         return;
       }
       try {
         const audioBuffer = await blobToAudioBuffer(item.blob);
-        await requestProcessStreaming(audioBuffer);
+        const { dimensions, features } = await extractor.process(audioBuffer);
+        await engine.when();
+        engine.run({
+          audioFeatures: features,
+          audioDimensions: dimensions,
+          reset: false
+        });
       } catch (error) {
         console.error("Error processing paragraph:", error);
       }
@@ -136,30 +129,23 @@ var _sfc_main = /* @__PURE__ */ defineComponent({
     function allParagraphCompleted() {
       if (paragraphData.value.length) {
         frameStatus.value = TickerStatus.stop;
-        frameUrls.value.length = 0;
+        frames.value.length = 0;
       }
     }
-    const ready = ref(false);
-    async function loadAllSilentFrames(dataset, zip) {
-      for (const image of dataset.images) {
-        const imageFile = zip.file(image.full_image);
-        if (imageFile) {
-          const blob = await imageFile.async("blob");
-          const url = URL.createObjectURL(blob);
-          silentFrameUrls.push(url);
-        }
-        if (slientFrameStatus.value === TickerStatus.pending) {
-          slientFrameStatus.value = TickerStatus.play;
-        }
+    function onProgress(processed, total) {
+      if (total < props.playAfterCache && processed === total || processed === props.playAfterCache) {
+        frameStatus.value = TickerStatus.play;
       }
-      ready.value = true;
     }
-    const __returned__ = { props, emit, paragraphData, streamingInferenceService, frameUrls, silentFrameUrls, startedDef, frameStatus, slientFrameStatus, requestProcessStreaming, processingParagraph, allParagraphCompleted, ready, loadAllSilentFrames, get VkBroadcastingMarkdown() {
+    function onFrame(frame) {
+      frames.value.push(frame);
+    }
+    const __returned__ = { props, emit, paragraphData, extractor, engine, silentFrameGenerator, slientFrameStatus, frames, frameStatus, processingParagraph, allParagraphCompleted, onProgress, onFrame, get VkBroadcastingMarkdown() {
       return VkBroadcastingMarkdown;
     }, get VkPixiFrameBitmap() {
       return VkPixiFrameBitmap;
-    }, get VkPixiFrameCore() {
-      return VkPixiFrameCore;
+    }, get VkPixiFrameBitmapGenerator() {
+      return VkPixiFrameBitmapGenerator;
     }, get setData() {
       return setData;
     }, get TickerStatus() {
@@ -181,24 +167,28 @@ function _sfc_render(_ctx, _cache, $props, $setup, $data, $options) {
         "text-to-speech": _ctx.textToSpeech,
         source: _ctx.source,
         processing: $setup.processingParagraph,
+        disabled: _ctx.disabled,
         onSetData: _cache[0] || (_cache[0] = ($event) => $setup.setData($setup.paragraphData, $event)),
         onComplete: $setup.allParagraphCompleted
-      }), null, 16, ["status", "data", "text-to-speech", "source"]),
+      }), {
+        default: withCtx(({ paragraphs }) => [
+          renderSlot(_ctx.$slots, "default", { paragraphs })
+        ]),
+        _: 3
+        /* FORWARDED */
+      }, 16, ["status", "data", "text-to-speech", "source", "disabled"]),
       createVNode($setup["VkPixiFrameBitmap"], {
         status: $setup.frameStatus,
         "onUpdate:status": _cache[1] || (_cache[1] = ($event) => $setup.frameStatus = $event),
-        data: $setup.frameUrls,
-        onSetData: _cache[2] || (_cache[2] = ($event) => $setup.setData($setup.frameUrls, $event))
+        data: $setup.frames,
+        onSetData: _cache[2] || (_cache[2] = ($event) => $setup.setData($setup.frames, $event))
       }, null, 8, ["status", "data"]),
-      $setup.ready ? (openBlock(), createBlock($setup["VkPixiFrameCore"], {
-        key: 0,
+      createVNode($setup["VkPixiFrameBitmapGenerator"], {
         status: $setup.slientFrameStatus,
         "onUpdate:status": _cache[3] || (_cache[3] = ($event) => $setup.slientFrameStatus = $event),
-        data: $setup.silentFrameUrls,
-        loop: true,
-        prerender: true,
+        generator: $setup.silentFrameGenerator,
         visible: $setup.frameStatus !== $setup.TickerStatus.playing
-      }, null, 8, ["status", "data", "visible"])) : createCommentVNode("v-if", true)
+      }, null, 8, ["status", "generator", "visible"])
     ],
     64
     /* STABLE_FRAGMENT */

package/components/metahuman-broadcasting/src/ctx.d.ts

@@ -51,6 +51,10 @@ export declare const props: {
         type: NumberConstructor;
         default: number;
     };
+    disabled: {
+        type: BooleanConstructor;
+        default: boolean;
+    };
 };
 export declare const emits: {
     'update:status': null;