@vonage/ml-transformers 4.2.1 → 4.3.1

package/dist/types/src/facades/vonage-media-processor.d.ts

@@ -1,6 +1,8 @@
 import { EventDataMap, MediaProcessorConnector } from "@vonage/media-processor";
 import Emittery from "emittery";
 import { BackgroundOptions, MediaProcessorConfig } from "../../main";
+import { ResolvedWebglQuery } from "../webgl/webgl-profiler";
+import { WebglProfilerReporter } from "../webgl/webgl-profiler-reporter";
 /**
  * Class wrapping features provided by ml-transformers.
  */
@@ -50,6 +52,9 @@ export declare class VonageMediaProcessor extends Emittery<EventDataMap> {
      * @returns - `MediaProcessorConnectorInterface` feed this return value to any vonage SDK that supports this API
      */
     getConnector(): MediaProcessorConnector;
+    profile(duration: number): Promise<ResolvedWebglQuery[]>;
+    static profile(duration: number): Promise<WebglProfilerReporter>;
+    private static instances;
     /**
      * Asynchronous constructor of VonageMediaProcessor
      * @param config Initial MediaProcessorConfig to use

package/dist/types/src/filters/background-filter.d.ts

@@ -1,4 +1,5 @@
 import { BackgroundOptions } from "../../main";
+import { ResolvedWebglQuery } from "../webgl/webgl-profiler";
 import { FlickeringOptions } from "../webgl/pipelines/improve-segmentation-mask";
 export interface BackgroundFilterConfig {
     maskWidth: number;
@@ -7,21 +8,24 @@ export interface BackgroundFilterConfig {
     foregroundHeight: number;
 }
 export declare class BackgroundFilter {
+    private id;
     private improveMaskSegmentation;
     private backgroundResizer;
     private backgroundImage;
     private backgroundVideo;
+    private profiler?;
     private activePipeline?;
     canvas: OffscreenCanvas;
     private options?;
-    readonly context: WebGLRenderingContext;
+    readonly context: WebGL2RenderingContext;
     private virtualBGImage?;
-    constructor(config: BackgroundFilterConfig);
+    constructor(id: string, config: BackgroundFilterConfig);
     setBackgroundOptions(options: BackgroundOptions): void;
     setFlickeringOptions(options: FlickeringOptions): void;
     setVideoBGReadable(videoBGReadable: ReadableStream): void;
     setVirtualBGImage(image: ImageBitmap): Promise<void>;
     process(foreground: ImageBitmap, mask?: ImageBitmap): Promise<void>;
     resizeForeground(width: number, height: number): Promise<void>;
+    profile(duration: number): Promise<ResolvedWebglQuery[]>;
     private getBlurSize;
 }

package/dist/types/src/processors/processor-main.d.ts

@@ -2,6 +2,7 @@ import { EventDataMap, MediaProcessorInterface } from "@vonage/media-processor";
 import Emittery from "emittery";
 import { BackgroundOptions, MediaProcessorConfig } from "../../main";
 import { FlickeringOptions } from "../webgl/pipelines/improve-segmentation-mask";
+import { ResolvedWebglQuery } from "../webgl/webgl-profiler";
 export declare enum WorkerOperations {
     init = "init",
     transform = "transform",
@@ -22,11 +23,13 @@ export interface InnerEventDataMap {
 export declare class ProcessorMain extends Emittery<EventDataMap> implements MediaProcessorInterface {
     private rate;
     private worker?;
+    private static initCount;
     init(config: MediaProcessorConfig): Promise<void>;
     setTrackExpectedRate(rate: number): Promise<void>;
     setBackgroundOptions(options: BackgroundOptions): Promise<void>;
     transform(readable: ReadableStream<any>, writable: WritableStream<any>): Promise<void>;
     destroy(): Promise<void>;
     setFlickeringOptions(options: FlickeringOptions): Promise<void>;
+    profile(duration: number): Promise<ResolvedWebglQuery[]>;
     private listenWorker;
 }

package/dist/types/src/processors/processor-worker.d.ts

@@ -11,7 +11,7 @@ export declare class ProcessorWorker {
     private rate;
     private resolveOnMediaProcessor?;
     private eventsQueue;
-    init(config: MediaProcessorConfig): Promise<void>;
+    init(id: string, config: MediaProcessorConfig): Promise<void>;
     onMediaProcessorEvent(): Promise<MediaProcessorEvent>;
     setTrackExpectedRate(rate: number): Promise<void>;
     transform(readable: ReadableStream<any>, writable: WritableStream<any>): Promise<void>;
@@ -21,4 +21,5 @@ export declare class ProcessorWorker {
     terminate(): Promise<void>;
     setFlickeringOptions(options: FlickeringOptions): void;
     setVonageMetadata(metadata: VonageMetadata): void;
+    profile(duration: number): Promise<string>;
 }

package/dist/types/src/transformers/background-transformer.d.ts

@@ -1,14 +1,16 @@
 /// <reference types="dom-webcodecs" />
 import { BackgroundOptions, MediaProcessorConfig } from "../../main";
 import { FlickeringOptions } from "../webgl/pipelines/improve-segmentation-mask";
+import { ResolvedWebglQuery } from "../webgl/webgl-profiler";
 export declare class BackgroundTransformer {
     private selfieSegmentation;
     private backgroundFilter?;
-    init(config: MediaProcessorConfig): Promise<void>;
+    init(id: string, config: MediaProcessorConfig): Promise<void>;
     private previousDimensions;
     transform?(frame: VideoFrame, controller: TransformStreamDefaultController): Promise<void>;
     setBackgroundOptions(options: BackgroundOptions): Promise<void>;
     setVideoBGReadable(stream: ReadableStream): Promise<void>;
     setVirtualBGImage(image: ImageBitmap): Promise<void>;
     setFlickeringOptions(options: FlickeringOptions): void;
+    profile(duration: number): Promise<ResolvedWebglQuery[]>;
 }

package/dist/types/src/types.d.ts

@@ -190,4 +190,8 @@ export interface SupportedBrowserFeatures {
      * GPU processing support, optional
      */
     gpu: FeatureSupport;
+    /**
+     * Webgl context support, required
+     */
+    webgl: FeatureSupport;
 }

package/dist/types/src/utils/async.d.ts

@@ -0,0 +1 @@
+export declare function wait(ms: number): Promise<void>;

package/dist/types/src/utils/webgl.d.ts

@@ -15,7 +15,7 @@ export declare function frameToCanvas(frame: VideoFrame, context: OffscreenCanva
  * @param source Source used to fill the texture
  * @returns Created texture, or source if it's a WebGLTexture
  */
-export declare function texture(context: WebGLRenderingContext, source?: TextureSource | WebGLTexture): WebGLTexture;
+export declare function texture(context: WebGLRenderingContext, source?: TextureSource | WebGLTexture): () => WebGLTexture;
 /**
  * Create a webgl texture and fill it with a color
  * @param context Webgl context to create the texture on
@@ -24,4 +24,8 @@ export declare function texture(context: WebGLRenderingContext, source?: Texture
  * @param color Color to fill the texture with
  * @returns Created texture
  */
-export declare function textureWithColor(context: WebGLRenderingContext, width: number, height: number, color: vec4): WebGLTexture;
+export declare function textureWithColor(context: WebGLRenderingContext, width: number, height: number, color: vec4): () => WebGLTexture;
+/**
+ * Minimum size for a webgl texture
+ */
+export declare const WEBGL_MINIMUM_TEXTURE_SIZE: number;

package/dist/types/src/webgl/pipelines/background-blur.d.ts

@@ -4,6 +4,8 @@ export declare class BackgroundBlur extends WebglTransformerPipeline {
     private readonly context;
     inputImage?: ImageBitmap | TextureSource | WebGLTexture;
     inputMask?: ImageBitmap | TextureSource | WebGLTexture;
+    private blur;
     constructor(context: WebGLRenderingContext, radius: number);
     setData(image?: ImageBitmap | TextureSource | WebGLTexture, mask?: ImageBitmap | TextureSource | WebGLTexture): void;
+    resizeOutput(width: number, height: number): void;
 }

package/dist/types/src/webgl/pipelines/improve-segmentation-mask.d.ts

@@ -1,5 +1,8 @@
 import { TextureSource } from "../../types";
 import { WebglPipeline } from "../webgl-pipeline";
+/**
+ * @internal
+ */
 export interface FlickeringOptions {
     currentThresholdMinimum: number;
     currentThresholdMaximum: number;

package/dist/types/src/webgl/webgl-pipeline-program.d.ts

@@ -1,6 +1,7 @@
 import * as twgl from "twgl.js";
 import { FramebufferInfo } from "twgl.js";
 import { UniformDataMap } from "../types";
+import { WebglProfiler } from "./webgl-profiler";
 export interface WebglPipelineProgramOptions {
     context: WebGLRenderingContext;
     width: number;
@@ -15,6 +16,7 @@ export interface WebglPipelineProgramOptions {
  *  - vec2 canvas: canvas dimension, in px
  */
 export declare abstract class WebglPipelineProgram<O extends WebglPipelineProgramOptions = WebglPipelineProgramOptions> {
+    id: string;
     /**
      * Webgl context used by the program
      */
@@ -39,6 +41,7 @@ export declare abstract class WebglPipelineProgram<O extends WebglPipelineProgra
      * Buffers infos used by the program
      */
     private bufferInfo;
+    private profiler?;
     constructor(options: O);
     resizeOutput(width: number, height: number): void;
     /**
@@ -79,4 +82,5 @@ export declare abstract class WebglPipelineProgram<O extends WebglPipelineProgra
      * @param uniforms Data to used for the uniforms
      */
     run(uniforms: UniformDataMap): void;
+    setProfiler(profiler?: WebglProfiler): void;
 }

package/dist/types/src/webgl/webgl-pipeline.d.ts

@@ -1,4 +1,5 @@
 import { UniformDataMap } from "../types";
+import { WebglProfiler } from "./webgl-profiler";
 import { WebglPipelineProgram } from "./webgl-pipeline-program";
 /**
  * Program pipeline run infos
@@ -13,6 +14,10 @@ export interface WebglPipelineStep {
      * @returns Object containing for each uniform a pair key/value name/value
      */
     getUniforms: () => UniformDataMap;
+    /**
+     * If set to true, the step won't be rescale if the output is resized
+     */
+    preventResize?: boolean;
 }
 /**
  * Webgl pipeline. Run pipeline program one after one
@@ -32,4 +37,7 @@ export declare class WebglPipeline {
      */
     run(): Promise<void>;
     resizeOutput(width: number, height: number): void;
+    private profiler?;
+    setProfiler(profiler?: WebglProfiler): void;
+    setId(id: string): void;
 }

package/dist/types/src/webgl/webgl-profiler-reporter.d.ts

@@ -0,0 +1,11 @@
+import { ResolvedWebglQuery } from "./webgl-profiler";
+/**
+ * @internal
+ */
+export declare class WebglProfilerReporter {
+    private queries;
+    constructor(queries: ResolvedWebglQuery[][]);
+    downloadSpeedscope(): Promise<void>;
+    private download;
+    private convertToSpeedscope;
+}

package/dist/types/src/webgl/webgl-profiler.d.ts

@@ -0,0 +1,40 @@
+/**
+ * @internal
+ */
+export declare enum WebglQueryAction {
+    PUSH = "PUSH",
+    POP = "POP"
+}
+/**
+ * @internal
+ */
+export interface WebglQuery {
+    name: string;
+    action: WebglQueryAction;
+    query: WebGLQuery;
+}
+/**
+ * @internal
+ */
+export interface ResolvedWebglQuery extends WebglQuery {
+    duration: number;
+    timestamp: number;
+}
+/**
+ * @internal
+ */
+export declare class WebglProfiler {
+    private context;
+    private extension;
+    private queries;
+    private activeQuery?;
+    constructor(context: WebGL2RenderingContext);
+    start(): void;
+    stop(): void;
+    pushContext(name: string): void;
+    popContext(name: string): void;
+    private createQuery;
+    getResolvedQueries(): Promise<ResolvedWebglQuery[]>;
+    private resolveQuery;
+    private awaitQueryAvailable;
+}

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@vonage/ml-transformers",
-    "version": "4.2.1",
+    "version": "4.3.1",
     "author": "Guy Mininberg <guy.mininberg@vonage.com>",
     "contributors": [
         "Guy Mininberg <guy.mininberg@vonage.com>",
@@ -26,7 +26,7 @@
         "build": "npm-run-all build:js build:static doc",
         "build:js": "tsc && vite build && tsc --declaration --emitDeclarationOnly --outDir dist/types",
         "build:static": "node ./scripts/build-static.js && minify ./dist/ml-transformers.static.js > ./dist/ml-transformers.min.js ",
-        "doc": "typedoc --excludePrivate --excludeProtected --disableSources --out dist/docs lib/main.ts",
+        "doc": "typedoc --excludePrivate --excludeInternal --excludeProtected --disableSources --out dist/docs lib/main.ts",
         "dev": "vite",
         "preview": "vite preview",
         "e2e:app": "vite ./tests/e2e/apps/vite",