@vonage/ml-transformers 7.0.0-alpha.1 → 7.0.0-alpha.3

package/dist/types/lib/main.d.ts

@@ -1,126 +1,8 @@
 import { WarningType, ErrorFunction, PipelineInfoData } from "@vonage/media-processor";
-import { BlurRadius, BackgroundBlurConfig, VirtualBackgroundConfig, AssetBackgroundConfig, VideoBackgroundConfig, MediaProcessorBaseConfig, SilhouetteBlurConfig, MediapipeConfig, MediaPipeResults, MediaPipeResultsListener, MediaPipeModelType, FaceDetectionResults, FaceDetectionOptions, FaceMeshResults, FaceMeshOptions, HandsResults, HandsOptions, HolisticResults, HolisticOptions, ObjectronResults, ObjectronOptions, PoseResults, PoseOptions, SelfieSegmentationResults, SelfieSegmentationOptions, MediaPipeModelConfig, SupportedBrowserFeatures, FeatureSupport, WebglSelfieSegmentationType, BackgroundOptions, BackgroundTransformerType, RenderingOptions, RenderingType, SegmentationOptions } from "./src/types";
+import { BlurRadius, BackgroundBlurConfig, VirtualBackgroundConfig, AssetBackgroundConfig, VideoBackgroundConfig, MediaProcessorBaseConfig, SilhouetteBlurConfig, SupportedBrowserFeatures, FeatureSupport, WebglSelfieSegmentationType, BackgroundOptions, BackgroundTransformerType, RenderingOptions, RenderingType, SegmentationOptions } from "./src/types";
 import { isSupported, getSupportedBrowserFeatures } from "./src/utils/support";
-import { MediapipeHelper } from "./src/utils/mediapipe";
 import { VonageMediaProcessor } from "./src/facades/vonage-media-processor";
 import { BackgroundTransformer } from "./src/transformers/background-transformer";
-import VonageSelfieSegmentation from "./libs/selfie-segmentation-helper";
-/**
- * MediaPipe Selfie Segmentation static metadata.
- * @returns
- * An object containing MediaPipe Selfie Segmentation metadata:
- * - VERSION
- */
-declare function getVonageSelfieSegmentation(): VonageSelfieSegmentation;
-import VonageHands from "./libs/hands-helper";
-/**
- * Vonage Mediapipe objects info (it will be used by the @mediapipe/drawing_utils package which is not included by this library).
- * @param VonageSelfiesegmentation:
- * - VERSION
- * @param VonageHands:
- * - VERSION
- * - HAND_CONNECTIONS
- */
-declare function getVonageHands(): VonageHands;
-import VonageHolistic from "./libs/holistic-helper";
-/**
- * MediaPipe Holistic static metadata.
- * This metadata is needed while using MediaPipe.
- * [drawing utils](https://www.npmjs.com/package/@mediapipe/drawing_utils).
- * And [control utils](https://www.npmjs.com/package/@mediapipe/control_utils).
- * @returns
- * An object containing MediaPipe Holistic metadata:
- * - VERSION
- * - FACE_GEOMETRY
- * - FACEMESH_LIPS
- * - FACEMESH_LEFT_EYE
- * - FACEMESH_LEFT_EYEBROW
- * - FACEMESH_LEFT_IRIS
- * - FACEMESH_RIGHT_EYE
- * - FACEMESH_RIGHT_EYEBROW
- * - FACEMESH_RIGHT_IRIS
- * - FACEMESH_FACE_OVAL
- * - FACEMESH_CONTOURS
- * - FACEMESH_TESSELATION
- * - HAND_CONNECTIONS
- * - POSE_CONNECTIONS
- * - POSE_LANDMARKS
- * - POSE_LANDMARKS_LEFT
- * - POSE_LANDMARKS_RIGHT
- * - POSE_LANDMARKS_NEUTRAL
- */
-declare function getVonageHolistic(): VonageHolistic;
-import VonageFaceDetection from "./libs/face-detection-helper";
-/**
- * MediaPipe Face Detection static metadata.
- * This metadata is needed while using MediaPipe.
- * [drawing utils](https://www.npmjs.com/package/@mediapipe/drawing_utils).
- * And [control utils](https://www.npmjs.com/package/@mediapipe/control_utils).
- * @returns
- *  An object containing MediaPipe Face Detection metadata:
- * - VERSION
- * - FACEDETECTION_LIPS
- * - FACEDETECTION_LEFT_EYE
- * - FACEDETECTION_LEFT_EYEBROW
- * - FACEDETECTION_RIGHT_EYE
- * - FACEDETECTION_RIGHT_EYEBROW
- * - FACEDETECTION_FACE_OVAL
- * - FACEDETECTION_CONTOURS
- * - FACEDETECTION_TESSELATION
- */
-declare function getVonageFaceDetection(): VonageFaceDetection;
-import VonageFaceMash from "./libs/face-mash-helper";
-/**
- * MediaPipe Face Mash static metadata.
- * This metadata is needed while using MediaPipe.
- * [drawing utils](https://www.npmjs.com/package/@mediapipe/drawing_utils).
- * And [control utils](https://www.npmjs.com/package/@mediapipe/control_utils).
- * @returns
- * An object containing MediaPipe Face Mash metadata:
- * - VERSION
- * - FACE_GEOMETRY
- * - FACEMESH_LIPS
- * - FACEMESH_LEFT_EYE
- * - FACEMESH_LEFT_EYEBROW
- * - FACEMESH_LEFT_IRIS
- * - FACEMESH_RIGHT_EYE
- * - FACEMESH_RIGHT_EYEBROW
- * - FACEMESH_RIGHT_IRIS
- * - FACEMESH_FACE_OVAL
- * - FACEMESH_CONTOURS
- * - FACEMESH_TESSELATION
- */
-declare function getVonageFaceMash(): VonageFaceMash;
-import VonageObjectron from "./libs/objectron-helper";
-/**
- * MediaPipe Objectron static metadata.
- * This metadata is needed while using MediaPipe.
- * [drawing utils](https://www.npmjs.com/package/@mediapipe/drawing_utils).
- * And [control utils](https://www.npmjs.com/package/@mediapipe/control_utils).
- * @returns
- * An object containing MediaPipe Objectron metadata:
- * - VERSION
- * - VERSION
- * - BOX_CONNECTIONS
- * - BOX_KEYPOINTS
- */
-declare function getVonageObjectron(): VonageObjectron;
-import VonagePose from "./libs/pose-helper";
-/**
- * MediaPipe Pose static metadata.
- * This metadata is needed while using MediaPipe.
- * [drawing utils](https://www.npmjs.com/package/@mediapipe/drawing_utils).
- * And [control utils](https://www.npmjs.com/package/@mediapipe/control_utils).
- * @returns
- * An object containing MediaPipe Pose metadata:
- * - VERSION
- * - POSE_CONNECTIONS
- * - POSE_LANDMARKS
- * - POSE_LANDMARKS_LEFT
- * - POSE_LANDMARKS_RIGHT
- * - POSE_LANDMARKS_NEUTRAL
- */
-declare function getVonagePose(): VonagePose;
 /**
  * MediaProcessorConfig specifies the transformer logic to be performed.
  */
@@ -132,5 +14,5 @@ type MediaProcessorConfig = BackgroundOptions;
  */
 export declare function createVonageMediaProcessor(config: MediaProcessorConfig): Promise<VonageMediaProcessor>;
 export type { EventDataMap, WarnData, ErrorData, EventMetaData, DropInfo, } from "@vonage/media-processor";
-export type { MediaProcessorBaseConfig, BackgroundBlurConfig, VirtualBackgroundConfig, VideoBackgroundConfig, AssetBackgroundConfig, SilhouetteBlurConfig, MediaProcessorConfig, BackgroundOptions, MediapipeConfig, MediaPipeResults, MediaPipeModelType, MediaPipeResultsListener, FaceDetectionResults, FaceDetectionOptions, FaceMeshResults, FaceMeshOptions, HandsResults, HandsOptions, HolisticResults, HolisticOptions, ObjectronResults, ObjectronOptions, SelfieSegmentationOptions, SelfieSegmentationResults, PoseResults, PoseOptions, MediaPipeModelConfig, SupportedBrowserFeatures, FeatureSupport, RenderingOptions, SegmentationOptions, };
-export { VonageMediaProcessor, BlurRadius, isSupported, getSupportedBrowserFeatures, MediapipeHelper, getVonageSelfieSegmentation, getVonageFaceDetection, getVonageFaceMash, getVonageHands, getVonageHolistic, getVonageObjectron, getVonagePose, WarningType, PipelineInfoData, ErrorFunction, BackgroundTransformer, WebglSelfieSegmentationType, BackgroundTransformerType, RenderingType, };
+export type { MediaProcessorBaseConfig, BackgroundBlurConfig, VirtualBackgroundConfig, VideoBackgroundConfig, AssetBackgroundConfig, SilhouetteBlurConfig, MediaProcessorConfig, BackgroundOptions, SupportedBrowserFeatures, FeatureSupport, RenderingOptions, SegmentationOptions, };
+export { VonageMediaProcessor, BlurRadius, isSupported, getSupportedBrowserFeatures, WarningType, PipelineInfoData, ErrorFunction, BackgroundTransformer, WebglSelfieSegmentationType, BackgroundTransformerType, RenderingType, };

package/dist/types/lib/src/renderers/canvas2d/renderers/canvas2d-renderer.d.ts

@@ -1,11 +1,9 @@
 import { RendererInterface } from "../../interfaces/renderer";
-import { FlickeringOptions } from "../../webgl/pipelines/improve-segmentation-mask";
 import { ResolvedWebglQuery } from "../../webgl/webgl-profiler";
 export declare abstract class Canvas2dRenderer implements RendererInterface {
     protected canvas: OffscreenCanvas;
     protected context: OffscreenCanvasRenderingContext2D;
     constructor();
-    setFlickeringOptions(options: FlickeringOptions): void;
     enablePostProcessing(): void;
     disablePostProcessing(): void;
     profileWebgl(duration: number): Promise<ResolvedWebglQuery[]>;

package/dist/types/lib/src/renderers/webgl/pipelines/improve-segmentation-mask.d.ts

@@ -1,17 +1,5 @@
 import { TextureSource } from "../../../types";
 import { WebglPipeline } from "../webgl-pipeline";
-/**
- * @internal
- */
-export interface FlickeringOptions {
-    currentThresholdMinimum: number;
-    currentThresholdMaximum: number;
-    previousThresholdMinimum: number;
-    previousThresholdMaximum: number;
-    thresholdMinimum: number;
-    thresholdMaximum: number;
-    thresholdFactor: number;
-}
 export declare class ImproveSegmentationMaskPipeline extends WebglPipeline {
     private readonly context;
     inputImage?: ImageBitmap;

package/dist/types/lib/src/renderers/webgl/programs/blur/blur.d.ts

@@ -14,7 +14,4 @@ export declare class Blur extends WebglPipelineProgram<BlurOptions> {
     protected getFragmentShader(): string;
     protected getVertexShader(): string;
     private createLoopFunction;
-    private getFactor;
-    private getLinearFactor;
-    private getGaussianFactor;
 }

package/dist/types/lib/src/transformers/background-transformer.d.ts

@@ -3,13 +3,13 @@ import { ResolvedWebglQuery } from "../renderers/webgl/webgl-profiler";
 export declare class BackgroundTransformer {
     private selfieSegmentation?;
     private backgroundFilter?;
+    private backgroundOptions?;
     private reporter;
     private isEnabled;
     init(id: string, config: MediaProcessorConfig): Promise<void>;
     transform?(frame: VideoFrame, controller: TransformStreamDefaultController): Promise<void>;
-    private backgroundOptions?;
     setBackgroundOptions(options: BackgroundOptions): Promise<void>;
-    setVideoBGReadable(stream: ReadableStream): Promise<void>;
+    setVideoBGReadable(stream: ReadableStream): void;
     setVirtualBGImage(image: ImageBitmap): Promise<void>;
     profile(duration: number): Promise<ResolvedWebglQuery[]>;
     getTransformerType(): string;

package/dist/types/lib/src/types.d.ts

@@ -1,10 +1,3 @@
-import { Options as FaceDetectionOptions, Results as FaceDetectionResults } from "@mediapipe/face_detection";
-import { Options as FaceMeshOptions, Results as FaceMeshResults } from "@mediapipe/face_mesh";
-import { Options as HandsOptions, Results as HandsResults } from "@mediapipe/hands";
-import { Options as HolisticOptions, Results as HolisticResults } from "@mediapipe/holistic";
-import { Options as ObjectronOptions, Results as ObjectronResults } from "@mediapipe/objectron";
-import { Options as PoseOptions, Results as PoseResults } from "@mediapipe/pose";
-import { Options as SelfieSegmentationOptions, Results as SelfieSegmentationResults } from "@mediapipe/selfie_segmentation";
 import { TextureFunc } from "twgl.js";
 /**
  * BlurRadius specifies how much bluring filter to apply by a given transformer.
@@ -153,55 +146,6 @@ export interface SilhouetteBlurConfig extends MediaProcessorBaseConfig {
  * BackgroundOptions background options
  */
 export type BackgroundOptions = BackgroundBlurConfig | VirtualBackgroundConfig | VideoBackgroundConfig | SilhouetteBlurConfig;
-export type { FaceDetectionResults, FaceDetectionOptions, FaceMeshResults, FaceMeshOptions, HandsResults, HandsOptions, HolisticResults, HolisticOptions, ObjectronResults, ObjectronOptions, SelfieSegmentationResults, SelfieSegmentationOptions, PoseResults, PoseOptions, };
-/**
- * MediaPipeResults types of results object of MediaPipe
- */
-export type MediaPipeResults = FaceDetectionResults | FaceMeshResults | HandsResults | HolisticResults | ObjectronResults | SelfieSegmentationResults | PoseResults;
-/**
- * ResultsListener callback function from MediaPipe process
- * @results - The results object from MediaPipe
- * @returns - can return a promise of void
- */
-export type MediaPipeResultsListener = <T extends MediaPipeResults>(results: T) => Promise<void> | void;
-/**
- * MediaPipeModelType supported models types
- */
-export type MediaPipeModelType = "face_mesh" | "face_detection" | "hands" | "holistic" | "objectron" | "selfie_segmentation" | "pose";
-/**
- * defines one mediapipe model config
- * @modelType - which model is required
- * @listener - callback function from the model
- * @options - define options for the mediapipe model that is used. for more info check https://google.github.io/mediapipe/getting_started/javascript.html
- * @assetsUri (optional) - can be set to get wasm/tflite/js/binarypb assets. Vonage provides static assets.
- * *** WARNING *** - using this option moves the responsibility of the assets and the versioning to the user.
- * please keep in mind that the assets list can change between versions!
- * assets are different between mediapipe models.
- */
-export type MediaPipeModelConfig = {
-    modelType: MediaPipeModelType;
-    listener: MediaPipeResultsListener;
-    options: FaceDetectionOptions | FaceMeshOptions | HandsOptions | HolisticOptions | ObjectronOptions | SelfieSegmentationOptions | PoseOptions;
-    assetsUri?: string;
-};
-/**
- * MediapipeConfig specified config of mediapipe helper:
- * this structure allows the user to create few mediapipe models that will run in parallel.
- * @modelTypesArray - array of mediapipe models to be loaded and called.
- */
-export type MediapipeConfig = {
-    mediaPipeModelConfigArray: Array<MediaPipeModelConfig>;
-};
-/**
- * Vector containing 3 numbers
- * @internal
- */
-export type vec3 = [number, number, number];
-/**
- * Vector containing 4 numbers
- * @internal
- */
-export type vec4 = [number, number, number, number];
 /**
  * Valid data to feed webgl texture
  * @internal

package/dist/types/lib/src/utils/webgl.d.ts

@@ -1,10 +1,4 @@
-import { TextureSource, vec4 } from "../types";
-/**
- * Render a VideoFrame within a canvas
- * @param frame VideoFrame to render
- * @param context Canvas context to render in
- */
-export declare function frameToCanvas(frame: VideoFrame, context: OffscreenCanvasRenderingContext2D): void;
+import { TextureSource } from "../types";
 /**
  * Create and fill a webgl texture
  * If source is a texture, it will just be returned without creating a new texture
@@ -13,15 +7,6 @@ export declare function frameToCanvas(frame: VideoFrame, context: OffscreenCanva
  * @returns Created texture, or source if it's a WebGLTexture
  */
 export declare function texture(context: WebGLRenderingContext, source?: TextureSource | WebGLTexture): () => WebGLTexture;
-/**
- * Create a webgl texture and fill it with a color
- * @param context Webgl context to create the texture on
- * @param width Width of the texture
- * @param height Height of the texture
- * @param color Color to fill the texture with
- * @returns Created texture
- */
-export declare function textureWithColor(context: WebGLRenderingContext, width: number, height: number, color: vec4): () => WebGLTexture;
 /**
  * Minimum size for a webgl texture
  */

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@vonage/ml-transformers",
-    "version": "7.0.0-alpha.1",
+    "version": "7.0.0-alpha.3",
     "author": "Vonage <vcp_webrtc@vonage.com>",
     "keywords": [
         "vonage",
@@ -32,24 +32,17 @@
         "version:published": "npm show @vonage/ml-transformers version"
     },
     "dependencies": {
-        "@mediapipe/face_detection": "0.4.1646425229",
-        "@mediapipe/face_mesh": "0.4.1633559619",
-        "@mediapipe/hands": "0.4.1646424915",
-        "@mediapipe/holistic": "0.5.1635989137",
-        "@mediapipe/objectron": "0.4.1636596145",
-        "@mediapipe/pose": "0.5.1635988162",
-        "@mediapipe/selfie_segmentation": "0.1.1632777926",
+        "@mediapipe/tasks-vision": "0.10.20",
         "@types/dom-mediacapture-transform": "^0.1.2",
         "@types/emscripten": "^1.39.6",
         "@types/offscreencanvas": "^2019.6.4",
         "@vonage/js-onewebrtc-telemetry": "1.1.2",
         "@vonage/js-workerizer": "^1.1.2",
-        "@vonage/media-processor": "3.0.0-alpha.2",
+        "@vonage/media-processor": "3.0.1-alpha.0",
         "twgl.js": "^5.1.0",
         "typescript-optional": "3.0.0-alpha.3",
         "uuid": "^13.0.0",
-        "wasm-feature-detect": "^1.2.11",
-        "@mediapipe/tasks-vision": "0.10.20"
+        "wasm-feature-detect": "^1.2.11"
     },
     "files": [
         "dist"