@vonage/ml-transformers 1.3.1 → 2.0.2

package/dist/types/src/transformers/BackgroundTransformer.d.ts

@@ -1,35 +1,37 @@
 /// <reference types="dom-webcodecs" />
 /// <reference types="offscreencanvas" />
-import { MediaProcessorConfig } from '../../main';
+import { BackgroundOptions } from '../../main';
 import { PostProcessInterface } from './PostProcessInterface';
-import { VonageTFLiteInfo } from './VonageTFLite';
+export declare type ProcessFrame = (image: ImageBitmap) => (void);
 declare class BackgroundTransformer {
     static BLUR_PROCESS_GAP_MICRO_SECOND: number;
     static VIRTUAL_PROCESS_GAP_MICRO_SECOND: number;
-    voangeTFLiteInfo_: VonageTFLiteInfo;
-    frameCounter_: number;
+    static NEW_INPUT_NUMBER_OF_RUNS: number;
+    static SELFIE_WIDTH: number;
+    static SELFIE_HIGHT: number;
     lastProcessedTimestamp_: number;
     timerGap_: number;
-    segmentationMaskCanvas_: OffscreenCanvas;
-    segmentationMaskCtx_: OffscreenCanvasRenderingContext2D | null;
-    segmentationPixelCount_: number;
-    segmentationMask_: ImageData;
-    segmentationMaskFrameCanvas_: OffscreenCanvas;
-    segmentationMaskFrameCtx_: OffscreenCanvasRenderingContext2D | null;
+    processFrameCb?: ProcessFrame;
+    segmentationMaskImage_?: ImageBitmap;
+    segmentationMaskCanvas_?: OffscreenCanvas;
+    segmentationMaskCtx_?: OffscreenCanvasRenderingContext2D | null;
     resultCanvas_: OffscreenCanvas;
     resultCtx_: OffscreenCanvasRenderingContext2D | null;
-    inputMemoryOffset_: number;
-    outputMemoryOffset_: number;
     postProcessInterface_?: PostProcessInterface;
-    config_: MediaProcessorConfig;
-    constructor(config: MediaProcessorConfig);
-    init(): Promise<void>;
-    start(): Promise<void>;
+    newInputCounter: number;
+    constructor();
+    init(framProcessor: ProcessFrame): void;
+    setBackgroundOptions(backgroundOptions: BackgroundOptions): Promise<void>;
+    close(): Promise<void>;
+    start(): void;
     transform(frame: any, controller: TransformStreamDefaultController): Promise<void>;
     processFrame(controller: TransformStreamDefaultController, image: ImageBitmap, timestamp: number): void;
     processSource(image: ImageBitmap): void;
     flush(): void;
     getTransformerType(): string;
-    setVideoBGReadable(videoBGReadable: ReadableStream): void;
+    resetNewInputCouner(): void;
+    setVideoBGReadable(videoBGReadable: ReadableStream): Promise<void>;
+    setVirtualBGImage(image: ImageBitmap): Promise<void>;
+    setSegmentationMaskImage(image: ImageBitmap): void;
 }
 export default BackgroundTransformer;

package/dist/types/src/transformers/BlurPostProcess.d.ts

@@ -5,6 +5,7 @@ import { BlurRadius } from "../types";
 declare class BlurPostProcess implements PostProcessInterface {
     blurFilter_: string;
     constructor(radius?: BlurRadius);
+    destroy(): Promise<void>;
     runPostProcessing(resultCtx: OffscreenCanvasRenderingContext2D | null, image: ImageBitmap): void;
 }
 export default BlurPostProcess;

package/dist/types/src/transformers/MediapipeHelper.d.ts

@@ -0,0 +1,12 @@
+import { MediapipeConfig, MediaPipeModelType } from '../types';
+declare class MediapipeHelper {
+    private mediapipeMap_;
+    private static defaultAssetsBaseUrl;
+    constructor();
+    private getVersion;
+    initialize(config: MediapipeConfig): Promise<void>;
+    close(modelType?: MediaPipeModelType): Promise<void>;
+    send(image: HTMLVideoElement | HTMLImageElement | HTMLCanvasElement | ImageBitmap | OffscreenCanvas, modelType?: MediaPipeModelType): Promise<void>;
+    reset(modelType?: MediaPipeModelType): void;
+}
+export default MediapipeHelper;

package/dist/types/src/transformers/PostProcessInterface.d.ts

@@ -2,4 +2,5 @@
 /// <reference types="offscreencanvas" />
 export interface PostProcessInterface {
     runPostProcessing(resultCtx: OffscreenCanvasRenderingContext2D | null, image: ImageBitmap): void;
+    destroy(): Promise<void>;
 }

package/dist/types/src/transformers/SilueteBlurPostProcess.d.ts

@@ -5,6 +5,7 @@ import { BlurRadius } from '../types';
 declare class SilueteBlurPostProcess implements PostProcessInterface {
     blurFilter_: string;
     constructor(radius?: BlurRadius);
+    destroy(): Promise<void>;
     runPostProcessing(resultCtx: OffscreenCanvasRenderingContext2D | null, image: ImageBitmap): void;
 }
 export default SilueteBlurPostProcess;

package/dist/types/src/transformers/VideoPostProcess.d.ts

@@ -1,11 +1,14 @@
 /// <reference types="dom-webcodecs" />
 /// <reference types="offscreencanvas" />
 import { PostProcessInterface } from './PostProcessInterface';
-import { Queue } from './Queue';
 export declare class VideoPostProcess implements PostProcessInterface {
-    queue_: Queue<any>;
-    frameReader_: any;
-    constructor(videoBGReadable: ReadableStream);
-    processFrame(result: any): void;
+    private queue_;
+    private frameReader_?;
+    private static NUMBER_OF_FRAMES_TO_CLEAR;
+    constructor();
+    private clearQueue;
+    setVideoReadable(videoBGReadable: ReadableStream): Promise<void>;
+    destroy(): Promise<void>;
+    private processFrame;
     runPostProcessing(resultCtx: OffscreenCanvasRenderingContext2D | null, image: ImageBitmap): void;
 }

package/dist/types/src/transformers/VirtualPostProcess.d.ts

@@ -2,10 +2,10 @@
 /// <reference types="offscreencanvas" />
 import { PostProcessInterface } from "./PostProcessInterface";
 declare class VirtualPostProcess implements PostProcessInterface {
-    imageCanvas_: OffscreenCanvas;
-    imageCtx_: OffscreenCanvasRenderingContext2D | null;
-    imageBlob_: any;
-    constructor(imageUri: string);
-    runPostProcessing(resultCtx: OffscreenCanvasRenderingContext2D | null, image: ImageBitmap): void;
+    bgImage_?: ImageBitmap;
+    constructor();
+    destroy(): Promise<void>;
+    setVirtualBGImage(image: ImageBitmap): void;
+    runPostProcessing(resultCtx: OffscreenCanvasRenderingContext2D | null, image: ImageBitmap): Promise<void>;
 }
 export default VirtualPostProcess;

package/dist/types/src/types.d.ts

@@ -1,3 +1,10 @@
+import { Results as FaceDetectionResults, Options as FaceDetectionOptions } from '@mediapipe/face_detection';
+import { Results as FaceMeshResults, Options as FaceMeshOptions } from '@mediapipe/face_mesh';
+import { Results as HandsResults, Options as HandsOptions } from '@mediapipe/hands';
+import { Results as HolisticResults, Options as HolisticOptions } from '@mediapipe/holistic';
+import { Results as ObjectronResults, Options as ObjectronOptions } from '@mediapipe/objectron';
+import { Results as SelfieSegmentationResults, Options as SelfieSegmentationOptions } from '@mediapipe/selfie_segmentation';
+import { Results as PoseResults, Options as PoseOptions } from '@mediapipe/pose';
 /**
  * VonageTransformerType which type of transformer to create.
  */
@@ -16,24 +23,32 @@ export declare enum BlurRadius {
     High = "High"
 }
 /**
- * MediaProcessorBaseConfig specifies the absolute URL for the path where the Wasm modules used by the transformers from this library are stored.
- *
- * @example
- *
- * ```ts
- * {
- *   wasmAssetUriPath: 'http://localhost:8080/assets'
- *   tfliteAssetUriPath: 'http://localhost:8080/assets'
- * }
- * ```
- * wasmAssetUriPath - should be the uri path to `tflite-simd.wasm` which included in the package under ./dist
- * tfliteAssetUriPath - should be the uri path to `selfie_segmentation_landscape.tflite` which included in the package under ./dist/models
- * the libary will load this files while starting the transformer process. the load will be on web worker.
- *
- */
+* MediaProcessorBaseConfig specifies the absolute URL for the path where the mediapipe assets used by the transformers from this library are stored.
+* Vonage hosts static assets of mediapipe that will be used in case this property is not set.
+* Vonage promises to take care of maintaining all versioning.
+*
+* @example
+*
+* ```ts
+* {
+*   mediapipeBaseAssetsUri: 'http://localhost:8080/assets' | undeifined
+* }
+* ```
+* mediapipeBaseAssetsUri - should be the uri path to mediapipe assets.
+* *** WARNING *** - using this option moves the responsibility of the assets and the versioning to the user.
+* mediapipe selfie segmintaion got those assets:
+* selfie_segmentation_landscape.tflite
+* selfie_segmentation_solution_simd_wasm_bin.js
+* selfie_segmentation_solution_simd_wasm_bin.wasm
+* selfie_segmentation_solution_wasm_bin.js
+* selfie_segmentation_solution_wasm_bin.wasm
+* selfie_segmentation.binarypb
+* selfie_segmentation.tflite
+* please keep in mind that the assets list can change between versions!
+* all assets can be taken from `node_modules/@mediapipe/selfie_segmentation`
+*/
 export declare type MediaProcessorBaseConfig = {
-    wasmAssetUriPath: string;
-    tfliteAssetUriPath: string;
+    mediapipeBaseAssetsUri?: string;
 };
 /**
  * BackgroundBlurConfig specifies the configuration for a transformer logic that performs background blurring.
@@ -67,3 +82,42 @@ export declare type SilueteBlurConfig = {
     transformerType: 'SilueteBlur';
     radius?: BlurRadius;
 };
+export type { FaceDetectionResults, FaceDetectionOptions, FaceMeshResults, FaceMeshOptions, HandsResults, HandsOptions, HolisticResults, HolisticOptions, ObjectronResults, ObjectronOptions, SelfieSegmentationResults, SelfieSegmentationOptions, PoseResults, PoseOptions };
+/**
+ * MediaPipeResults types of results object of MediaPipe
+ */
+export declare type MediaPipeResults = FaceDetectionResults | FaceMeshResults | HandsResults | HolisticResults | ObjectronResults | SelfieSegmentationResults;
+/**
+ * ResultsListener callback function from MediaPipe process
+ * @results - The results object from MediaPipe
+ * @returns - can return a promise of void
+ */
+export declare type MediaPipeResultsListener = <T extends MediaPipeResults>(results: T) => (Promise<void> | void);
+/**
+ * MediaPipeModelType supported models types
+ */
+export declare type MediaPipeModelType = "face_mesh" | "face_detection" | "hands" | "holistic" | "objectron" | "selfie_segmentation" | 'pose';
+/**
+ * defines one mediapipe model config
+ * @modelType - which model is required
+ * @listener - callback function from the model
+ * @options - define options for the mediapipe model that is used. for more info check https://google.github.io/mediapipe/getting_started/javascript.html
+ * @assetsUri (optional) - can be set to get wasm/tflite/js/binarypb assets. Vonage provides static assets.
+ * *** WARNING *** - using this option moves the responsibility of the assets and the versioning to the user.
+ * please keep in mind that the assets list can change between versions!
+ * assets are different between mediapipe models.
+ */
+export declare type MediaPipeModelConfig = {
+    modelType: MediaPipeModelType;
+    listener: MediaPipeResultsListener;
+    options: FaceDetectionOptions | FaceMeshOptions | HandsOptions | HolisticOptions | ObjectronOptions | SelfieSegmentationOptions;
+    assetsUri?: string;
+};
+/**
+ * MediapipeConfig specified config of mediapipe helper:
+ * this structure allows the user to create few mediapipe models that will run in parallel.
+ * @modelTypesArray - array of mediapipe models to be loaded and called.
+ */
+export declare type MediapipeConfig = {
+    mediaPipeModelConfigArray: Array<MediaPipeModelConfig>;
+};

package/dist/types/src/vonage/VonageAssetsLoaderHelper.d.ts

@@ -0,0 +1,5 @@
+declare class VonageAssetsLoaderHelper {
+    static createVideoReadable(videoUri: string): Promise<ReadableStream>;
+    static createImageCanvas(imageUri: string): Promise<ImageBitmap>;
+}
+export default VonageAssetsLoaderHelper;

package/dist/types/src/vonage/VonageMediaProcessorThread.d.ts

@@ -1,5 +1,6 @@
-export declare type CallbackType = 'success' | 'error' | 'media_processor_error_event' | 'media_processor_warn_event';
+export declare type CallbackType = 'success' | 'error' | 'media_processor_error_event' | 'media_processor_warn_event' | 'media_processor_pipeline_event';
 export declare type CallbackMessage = {
     callbackType: CallbackType;
-    message: any;
+    message?: any;
+    error?: string;
 };

package/dist/types/src/vonage/VonageMediaProcessorWorker.d.ts

@@ -1,16 +1,27 @@
 import { EventDataMap, MediaProcessorInterface } from '@vonage/media-processor';
-import { MediaProcessorConfig } from '../../main';
+import { BackgroundOptions, MediaProcessorConfig } from '../../main';
 import Emittery from 'emittery';
+import MediapipeHelper from '../transformers/MediapipeHelper';
+interface InnerEventDataMap {
+    initResults: string;
+    transformResults: string;
+    destroyResults: string;
+    setBackgroundOptionsResults: string;
+    setVideoBGReadableResults: string;
+    setVirtualBGImageResults: string;
+}
 declare class VonageMediaProcessorWorker extends Emittery<EventDataMap> implements MediaProcessorInterface {
-    worker_: any;
-    config_: MediaProcessorConfig;
-    onMessage_: Promise<void>;
+    worker_?: any;
     trackExpectedRate_: number;
     workerInited_: boolean;
-    constructor(config: MediaProcessorConfig);
+    mediapipeHelper_: MediapipeHelper;
+    innerEmitty_: Emittery<InnerEventDataMap>;
+    constructor();
+    init(config: MediaProcessorConfig): Promise<void>;
     transform(readable: ReadableStream, writable: WritableStream): Promise<void>;
     destroy(): Promise<void>;
     setTrackExpectedRate(trackExpectedRate: number): void;
     postTrackExpectedRate(): void;
+    setBackgroundOptions(backgroundOptions: BackgroundOptions): Promise<void>;
 }
 export default VonageMediaProcessorWorker;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@vonage/ml-transformers",
-  "version": "1.3.1",
+  "version": "2.0.2",
   "author": "Guy Mininberg <guy.mininberg@vonage.com>",
   "contributors": [
     "Guy Mininberg <guy.mininberg@vonage.com>",
@@ -37,7 +37,15 @@
     "axios": "^0.25.0",
     "uuid": "^8.3.2",
     "typescript-optional": "3.0.0-alpha.3",
-    "@vonage/media-processor": "1.2.1"
+    "@vonage/media-processor": "1.2.7",
+    "@mediapipe/selfie_segmentation": "0.1.1632777926",
+    "@mediapipe/hands": "0.4.1646424915",
+    "@mediapipe/objectron": "0.4.1636596145",
+    "@mediapipe/face_mesh": "0.4.1633559619",
+    "@mediapipe/face_detection": "0.4.1646425229",
+    "@mediapipe/holistic": "0.5.1635989137",
+    "@mediapipe/pose": "0.5.1635988162",
+    "@mediapipe/drawing_utils": "0.3.1620248257"
   },
   "files": [
     "dist"
@@ -54,11 +62,11 @@
   "devDependencies": {
     "@tsconfig/node12": "^1.0.1",
     "typescript": "^4.5.4",
-    "vite": "^2.7.2",
+    "vite": "2.9.9",
     "npm-run-all": "^4.1.5",
     "typedoc": "^0.22.12"
   },
-  "repository": "https://github.com/vonage/ml-transformers",
+  "repository": "https://github.com/Vonage/vonage-media-transformers-samples",
   "publishConfig": {
     "registry": "https://registry.npmjs.org"
   }

package/dist/types/src/transformers/VonageTFLite.d.ts

@@ -1,28 +0,0 @@
-/// <reference types="emscripten" />
-export interface TFLite extends EmscriptenModule {
-    _getModelBufferMemoryOffset(): number;
-    _getInputMemoryOffset(): number;
-    _getInputHeight(): number;
-    _getInputWidth(): number;
-    _getInputChannelCount(): number;
-    _getOutputMemoryOffset(): number;
-    _getOutputHeight(): number;
-    _getOutputWidth(): number;
-    _getOutputChannelCount(): number;
-    _loadModel(bufferSize: number): number;
-    _runInference(): number;
-}
-export declare class VonageTFLiteInfo {
-    selectedTFLite_: TFLite;
-    isSIMDSupported_: boolean;
-    isThreadsSupported_: boolean;
-    inputWidth_: number;
-    inputHeight_: number;
-    inputChannelCount_: number;
-    inputMemoryOffset_: number;
-    outputWidth_: number;
-    outputHeight_: number;
-    outputChannelCount_: number;
-    outputMemoryOffset_: number;
-}
-export declare function VonageTFLite(wasmUri: string, tfliteUri: string): Promise<VonageTFLiteInfo>;