@webex/web-media-effects-types 2.20.8 → 2.21.0

package/index.d.ts

@@ -1,8 +1,8 @@
 import TypedEmitter from 'typed-emitter';
+import { IFrameInferenceResult, Generator, Quality, BlurStrength, Pipeline } from '@webex/ladon-ts';
+export { BlurStrength, Generator, Quality } from '@webex/ladon-ts';
 import * as js_logger from 'js-logger';
 export { default as Logger } from 'js-logger';
-import { Generator, Quality, BlurStrength, Pipeline } from '@webex/ladon-ts';
-export { BlurStrength, Generator, Quality } from '@webex/ladon-ts';
 
 declare const EventEmitter_base: new <TT>() => TypedEmitter<TT>;
 declare class EventEmitter<T> extends EventEmitter_base<T> {
@@ -26,6 +26,14 @@ declare enum EffectEnv {
     Prod = "prod",
     Int = "int"
 }
+declare enum EffectState {
+    Idle = "idle",
+    Init = "init",
+    Disabled = "disabled",
+    Enabled = "enabled",
+    Disposed = "disposed",
+    Error = "error"
+}
 declare enum EffectEvent {
     TrackUpdated = "track-updated",
     Enabled = "enabled",
@@ -42,6 +50,7 @@ interface EffectOptions {
     [key: string]: unknown;
 }
 declare abstract class BaseEffect extends EventEmitter<EffectEventCallback> implements TrackEffect {
+    protected state: EffectState;
     protected effectTrack?: MediaStreamTrack;
     protected outputTrack?: MediaStreamTrack;
     protected inputStream?: MediaStream;
@@ -150,63 +159,106 @@ declare class NoiseReductionEffect extends BaseMicrophoneEffect {
 }
 declare const createNoiseReductionEffect: (options?: NoiseReductionEffectOptions, preloadAssets?: boolean) => Promise<NoiseReductionEffect>;
 
-declare const logger: js_logger.ILogger;
-//# sourceMappingURL=logger.d.ts.map
-
-interface RateEstimatorOptions {
-    hysteresisMargin: number;
-    lowDuration: number;
-    lowThreshold: number;
-    minSamples: number;
-    maxSamples: number;
+declare enum BeRightBackAction {
+    TurnOn = "turn-on",
+    TurnOff = "turn-off"
 }
-declare enum RateEstimatorEvent {
-    RateOk = "rate-ok",
-    RateLow = "rate-low",
-    RateLagging = "rate-lagging"
+declare enum BeRightBackState {
+    Idle = "idle",
+    Off = "off",
+    On = "on"
 }
-interface RateEstimatorEventCallback {
-    [RateEstimatorEvent.RateLow]: (rate: number) => void;
-    [RateEstimatorEvent.RateOk]: (rate: number) => void;
-    [RateEstimatorEvent.RateLagging]: (rate: number) => void;
+interface BeRightBackOptions {
+    motionIouThreshold: number;
+    onHysteresisMaxMs: number;
+    offHysteresisMaxMs: number;
 }
-declare enum RateEstimatorStatus {
-    Idle = "idle",
-    Init = "init",
-    Lagging = "lagging",
-    Low = "low",
-    Ok = "ok"
+interface BeRightBackDebugInfo {
+    action: BeRightBackAction;
+    isMoving: boolean;
+    offHysteresisDurationMs: number;
+    onHysteresisDurationMs: number;
+    numFaces: number;
+    state: BeRightBackState;
+    timeDiffMs: number;
 }
-interface RateEstimatedValues {
-    rate?: number;
-    status: RateEstimatorStatus;
-    threshold: number;
+declare enum BeRightBackEvent {
+    On = "on",
+    Off = "off",
+    StateChange = "state-change"
 }
-declare class RateEstimator extends EventEmitter<RateEstimatorEventCallback> {
-    private timestamps;
-    private targetRate?;
-    private estimatedRate?;
-    private lowRateStartTime?;
+interface BeRightBackEventCallback {
+    [BeRightBackEvent.StateChange]: (newState: BeRightBackState, oldState: BeRightBackState) => void;
+    [BeRightBackEvent.On]: () => void;
+    [BeRightBackEvent.Off]: () => void;
+}
+declare enum BeRightBackPluginMode {
+    Off = "off",
+    Conservative = "conservative",
+    Aggressive = "aggressive"
+}
+interface BeRightBackPluginOptions {
+    mode: BeRightBackPluginMode;
+    debug?: boolean;
+    coreOptions?: Partial<BeRightBackOptions>;
+}
+
+declare class BeRightBack extends EventEmitter<BeRightBackEventCallback> {
+    private debugBuffer;
+    private state;
     private options;
-    status: RateEstimatorStatus;
-    constructor(targetRate: number, options?: Partial<RateEstimatorOptions>);
-    getEstimatedValues(): RateEstimatedValues;
-    getOptions(): Readonly<RateEstimatorOptions>;
-    getTimestamps(): Readonly<number[]>;
-    addTimestamp(timestamp: DOMHighResTimeStamp): void;
-    private calculateAndUpdateRate;
-    private evaluateRate;
-    private isRateLow;
-    private handleLowRate;
-    private handleReturnToNormalRate;
-    reset(): void;
-    updateOptions(newOptions: Partial<RateEstimatorOptions>): void;
+    private onHysteresisDurationMs;
+    private offHysteresisDurationMs;
+    private lastTimestampMs;
+    private isEnabled;
+    private isFirstFrame;
+    constructor(options?: Partial<BeRightBackOptions>);
+    destroy(): void;
+    getDebugInfo(): BeRightBackDebugInfo[];
+    getLatestDebugInfo(): BeRightBackDebugInfo;
+    getState(): BeRightBackState;
+    setEnable(enable: boolean): void;
+    updateOptions(newOptions: Partial<BeRightBackOptions>): void;
+    processFrame(faces: Float32Array, shape: readonly number[], motion: number, timestampMs: number): BeRightBackState;
+    private initializeFirstFrameState;
+    private determineActionFromInferenceResult;
+    private handleHysteresisAndStateChange;
+    private accumulateHysteresisAndAttemptTurnOn;
+    private accumulateHysteresisAndAttemptTurnOff;
+    private changeState;
+    private resetHysteresis;
+}
+
+interface Plugin {
+    isInitialized: boolean;
+    initialize(effect: VirtualBackgroundEffect): void;
+    dispose(effect: VirtualBackgroundEffect): void;
+}
+type BeforeInferenceResult = {
+    shouldInfer: boolean;
+    modifiedCanvas?: HTMLCanvasElement;
+};
+type BeforeInferenceCallback = (timestamp: number, lastResult: IFrameInferenceResult, inputCanvas?: HTMLCanvasElement) => Promise<BeforeInferenceResult>;
+type AfterInferenceCallback = (timestamp: number, result?: IFrameInferenceResult) => Promise<void>;
+interface BeforeInferencePlugin extends Plugin {
+    onBeforeInference: BeforeInferenceCallback;
+}
+interface AfterInferencePlugin extends Plugin {
+    onAfterInference: AfterInferenceCallback;
+}
+type PluginType = BeforeInferencePlugin | AfterInferencePlugin;
+declare class PluginManager {
+    private plugins;
+    private effect;
+    register(name: string, plugin: PluginType): void;
+    initialize(effect: VirtualBackgroundEffect): void;
+    dispose(): void;
+    getPlugin<T extends PluginType>(name: string): T | undefined;
+    getPlugins(): Map<string, Plugin>;
 }
 
 declare function debounce<F extends (...args: Parameters<F>) => ReturnType<F>>(func: F, delay: number): (...args: Parameters<F>) => void;
 
-declare const DEFAULT_FRAME_RATE = 24;
-
 declare enum VirtualBackgroundMode {
     Blur = "BLUR",
     Image = "IMAGE",
@@ -224,8 +276,12 @@ interface VirtualBackgroundEffectOptions extends EffectOptions {
     mirror: boolean;
     env?: EffectEnv;
     preventBackgroundThrottling?: boolean;
-}
-declare const defaultOptions: VirtualBackgroundEffectOptions;
+    useLocalSegmentationWorker?: boolean;
+    canvasResolutionScaling?: number;
+}
+
+declare const DEFAULT_FRAME_RATE = 24;
+
 declare class VirtualBackgroundEffect extends BaseCameraEffect {
     static readonly kind = "virtual-background-effect";
     static readonly BlurStrength: typeof BlurStrength;
@@ -244,11 +300,16 @@ declare class VirtualBackgroundEffect extends BaseCameraEffect {
     private preloadFuture?;
     private loadFuture?;
     private disposeFuture?;
-    private onFrameProcessedCallback?;
     private lastFrameTime;
     private frameInterval;
     private restoreEffectOnTrackEnable;
     private timerId?;
+    private lastFrame?;
+    private pluginManager;
+    private beforeInferenceCallbacks;
+    private afterInferenceCallbacks;
+    private offscreenCanvas?;
+    private inputCanvasContext?;
     get isReady(): boolean;
     get isLoaded(): boolean;
     get frameRate(): number;
@@ -266,6 +327,9 @@ declare class VirtualBackgroundEffect extends BaseCameraEffect {
     private loadModel;
     private handleFrame;
     private manageEffectRestoration;
+    private prepareInputCanvas;
+    private executeBeforeInferenceCallbacks;
+    private executeAfterInferenceCallbacks;
     private setupDom;
     private detectQuirks;
     private handleOrientationChange;
@@ -275,10 +339,221 @@ declare class VirtualBackgroundEffect extends BaseCameraEffect {
     enable(): Promise<boolean>;
     disable(): Promise<boolean>;
     dispose(): Promise<void>;
-    setOnFrameProcessedCallback(callback: (timestamp: number) => void): void;
+    addBeforeInferenceCallback(callback: BeforeInferenceCallback): void;
+    addAfterInferenceCallback(callback: AfterInferenceCallback): void;
+    removeBeforeInferenceCallback(callback: BeforeInferenceCallback): void;
+    removeAfterInferenceCallback(callback: AfterInferenceCallback): void;
+    setOnFrameProcessedCallback(callback: AfterInferenceCallback): void;
     updateOptions(options: Partial<VirtualBackgroundEffectOptions>): Promise<void>;
     updateSize(width: number, height: number): Promise<void>;
+    registerPlugin: (name: string, plugin: PluginType) => void;
+    initializePlugins: () => void;
+    getPlugin: <T extends PluginType>(name: string) => T | undefined;
+    getPluginManager: () => PluginManager;
 }
 declare const createVirtualBackgroundEffect: (options?: VirtualBackgroundEffectOptions, preloadAssets?: boolean) => Promise<VirtualBackgroundEffect>;
 
-export { BaseCameraEffect, BaseEffect, BaseMicrophoneEffect, DEFAULT_FRAME_RATE, EffectEnv, EffectEvent, EffectEventCallback, EffectOptions, GainEffect, MediaStreamOrTrack, NoiseReductionEffect, NoiseReductionEffectOptions, NoiseReductionMode, RateEstimatedValues, RateEstimator, RateEstimatorEvent, RateEstimatorEventCallback, RateEstimatorOptions, RateEstimatorStatus, TrackEffect, VirtualBackgroundEffect, VirtualBackgroundEffectOptions, VirtualBackgroundMode, createEffect, createNoiseReductionEffect, createVirtualBackgroundEffect, defaultOptions, logger };
+declare abstract class BaseInferencePlugin<T, O> {
+    protected core: T;
+    protected options: O;
+    protected isEffectEnabled: boolean;
+    protected isInitialized: boolean;
+    constructor(options?: Partial<O>);
+    initialize(effect: VirtualBackgroundEffect): void;
+    dispose(): void;
+    updateOptions(newOptions: Partial<O>): void;
+    protected getDefaultOptions(options?: Partial<O>): O;
+    protected createCore(): T;
+    protected onCoreCreated(): void;
+    protected onEffectEnabled(): void;
+    protected onEffectDisabled(): void;
+    protected cleanup(): void;
+    protected onOptionsUpdated(): void;
+    protected abstract destroyCore(): void;
+}
+
+declare abstract class BaseEventEmitterPlugin<T extends EventEmitter<unknown>, O> extends BaseInferencePlugin<T, O> {
+    on?: T['on'];
+    off?: T['off'];
+    once?: T['once'];
+    constructor(options?: Partial<O>);
+    destroyCore(): void;
+}
+
+declare abstract class BaseAfterInferencePlugin<T extends EventEmitter<unknown> = EventEmitter<unknown>, O extends object = Record<string, never>> extends BaseEventEmitterPlugin<T, O> {
+    constructor(options?: Partial<O>);
+    initialize(effect: VirtualBackgroundEffect): void;
+    dispose(): void;
+    abstract onAfterInference(timestamp: number, result?: IFrameInferenceResult): Promise<void>;
+}
+
+declare class BeRightBackPlugin extends BaseAfterInferencePlugin<BeRightBack, BeRightBackPluginOptions> {
+    protected getDefaultOptions(): BeRightBackPluginOptions;
+    protected createCore(): BeRightBack;
+    onAfterInference(timestamp: number, result?: IFrameInferenceResult): Promise<void>;
+    onOptionsUpdated(): void;
+    getDebugInfo(): BeRightBackDebugInfo[];
+    getLatestDebugInfo(): BeRightBackDebugInfo;
+}
+
+declare function getOptionsByMode(mode: BeRightBackPluginMode): BeRightBackOptions;
+declare function makeBeRightBackOptions(mode: BeRightBackPluginMode, coreOptions?: Partial<BeRightBackOptions>): BeRightBackOptions;
+
+interface FrameSkipperOptions {
+    baseMinSkipTime: number;
+    baseMaxSkipTime: number;
+    historySize: number;
+    skipTimeIncrement: number;
+    forcedInferenceInterval: number;
+    highMotionThreshold: number;
+    smoothingFactor: number;
+}
+declare enum FrameSkipperInferenceReason {
+    Initial = "initial",
+    HighMotion = "high-motion",
+    TimeForced = "time-forced",
+    SkipIntervalElapsed = "skip-interval-elapsed"
+}
+interface FrameSkipperDebugInfo {
+    averageMotion: number;
+    currentTime: number;
+    currentSkipTime: number;
+    isHighMotion: boolean;
+    isSkipIntervalElapsed: boolean;
+    isTimeForced: boolean;
+    lastInferenceReason: FrameSkipperInferenceReason;
+    lastInferenceTime: number;
+    motionValue: number;
+    shouldInfer: boolean;
+    smoothedMotion: number;
+    timeSinceLastInference: number;
+}
+declare enum FrameSkipperPluginMode {
+    Off = "off",
+    Conservative = "conservative",
+    Aggressive = "aggressive"
+}
+interface FrameSkipperPluginOptions {
+    mode: FrameSkipperPluginMode;
+    debug?: boolean;
+    coreOptions?: Partial<FrameSkipperOptions>;
+}
+
+declare class AdaptiveFrameSkipper extends EventEmitter<unknown> {
+    private options;
+    private debugBuffer;
+    private motionData;
+    private lastInferenceTime;
+    private currentSkipTime;
+    private lastSmoothedMotion;
+    constructor(options?: FrameSkipperOptions);
+    destroy(): void;
+    getDebugInfo(): FrameSkipperDebugInfo[];
+    getLatestDebugInfo(): FrameSkipperDebugInfo;
+    shouldPerformInference(motionValue: number, motionTimestamp: number): boolean;
+    updateOptions(newOptions: FrameSkipperOptions): void;
+    private updateMotionData;
+    private adjustSkipTime;
+    private calculateSmoothedMotion;
+    private calculateAverageMotion;
+}
+
+declare abstract class BaseBeforeInferencePlugin<T extends EventEmitter<unknown> = EventEmitter<unknown>, O extends object = Record<string, never>> extends BaseEventEmitterPlugin<T, O> {
+    constructor(options: O);
+    initialize(effect: VirtualBackgroundEffect): void;
+    abstract onBeforeInference(timestamp: number, lastResult: IFrameInferenceResult, inputCanvas?: HTMLCanvasElement): Promise<BeforeInferenceResult>;
+}
+
+declare class FrameSkipperPlugin extends BaseBeforeInferencePlugin<AdaptiveFrameSkipper, FrameSkipperPluginOptions> {
+    protected getDefaultOptions(): FrameSkipperPluginOptions;
+    protected createCore(): AdaptiveFrameSkipper;
+    destroyCore(): void;
+    onBeforeInference(timestamp: number, lastResult: IFrameInferenceResult): Promise<BeforeInferenceResult>;
+    onOptionsUpdated(): void;
+    getDebugInfo(): FrameSkipperDebugInfo[];
+    getLatestDebugInfo(): FrameSkipperDebugInfo;
+}
+
+declare function makeFrameSkipperOptions(mode: FrameSkipperPluginMode, coreOptions?: Partial<FrameSkipperOptions>): FrameSkipperOptions;
+
+declare enum RateEstimatorStatus {
+    Idle = "idle",
+    Init = "init",
+    Lagging = "lagging",
+    Low = "low",
+    Ok = "ok"
+}
+interface RateEstimatedValues {
+    rate?: number;
+    status: RateEstimatorStatus;
+    threshold: number;
+}
+interface RateEstimatorOptions {
+    hysteresisMargin: number;
+    lowDuration: number;
+    lowThreshold: number;
+    minSamples: number;
+    maxSamples: number;
+}
+declare enum RateEstimatorEvent {
+    RateOk = "rate-ok",
+    RateLow = "rate-low",
+    RateLagging = "rate-lagging"
+}
+interface RateEstimatorEventCallback {
+    [RateEstimatorEvent.RateLow]: (rate: number) => void;
+    [RateEstimatorEvent.RateOk]: (rate: number) => void;
+    [RateEstimatorEvent.RateLagging]: (rate: number) => void;
+}
+interface RateEstimationPluginOptions {
+    targetRate: number;
+    debug?: boolean;
+    coreOptions?: Partial<RateEstimatorOptions>;
+}
+
+declare class RateEstimator extends EventEmitter<RateEstimatorEventCallback> {
+    private timestamps;
+    private estimatedRate?;
+    private lowRateStartTime?;
+    private options;
+    status: RateEstimatorStatus;
+    constructor(targetRate: number, options?: Partial<RateEstimatorOptions>);
+    dispose(): void;
+    getEstimatedValues(): RateEstimatedValues;
+    getOptions(): Readonly<RateEstimatorOptions>;
+    getTimestamps(): Readonly<number[]>;
+    addTimestamp(timestamp: DOMHighResTimeStamp): void;
+    private calculateAndUpdateRate;
+    private evaluateRate;
+    private isRateLow;
+    private handleLowRate;
+    private handleReturnToNormalRate;
+    reset(): void;
+    updateOptions(newOptions: Partial<RateEstimatorOptions>): void;
+}
+
+declare class RateEstimationPlugin extends BaseAfterInferencePlugin<RateEstimator, RateEstimationPluginOptions> {
+    protected createCore(): RateEstimator;
+    onCoreCreated(): void;
+    onEffectDisabled(): void;
+    onEffectEnabled(): void;
+    destroyCore(): void;
+    onAfterInference(timestamp: number): Promise<void>;
+    getRate(): number | undefined;
+    getThreshold(): number | undefined;
+    getStatus(): RateEstimatorStatus | undefined;
+}
+
+declare class SkippedFrameRatePlugin extends BaseAfterInferencePlugin {
+    private frameHistory;
+    private readonly frameWindowSize;
+    onAfterInference(timestamp: number, result?: IFrameInferenceResult): Promise<void>;
+    getSkippedPercentage(): number;
+    getSkippedFrameCount(): number;
+    getTotalFrameCount(): number;
+}
+
+declare const logger: js_logger.ILogger;
+//# sourceMappingURL=logger.d.ts.map
+
+export { AdaptiveFrameSkipper, BaseCameraEffect, BaseEffect, BaseMicrophoneEffect, BeRightBack, BeRightBackAction, BeRightBackDebugInfo, BeRightBackEvent, BeRightBackEventCallback, BeRightBackOptions, BeRightBackPlugin, BeRightBackPluginMode, BeRightBackPluginOptions, BeRightBackState, DEFAULT_FRAME_RATE, EffectEnv, EffectEvent, EffectEventCallback, EffectOptions, EffectState, FrameSkipperDebugInfo, FrameSkipperInferenceReason, FrameSkipperOptions, FrameSkipperPlugin, FrameSkipperPluginMode, FrameSkipperPluginOptions, GainEffect, MediaStreamOrTrack, NoiseReductionEffect, NoiseReductionEffectOptions, NoiseReductionMode, RateEstimatedValues, RateEstimationPlugin, RateEstimationPluginOptions, RateEstimator, RateEstimatorEvent, RateEstimatorEventCallback, RateEstimatorOptions, RateEstimatorStatus, SkippedFrameRatePlugin, TrackEffect, VirtualBackgroundEffect, createEffect, createNoiseReductionEffect, createVirtualBackgroundEffect, getOptionsByMode, logger, makeBeRightBackOptions, makeFrameSkipperOptions };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@webex/web-media-effects-types",
-  "version": "2.20.8",
+  "version": "2.21.0",
   "description": "TypeScript definitions for @webex/web-media-effects",
   "main": "",
   "types": "index.d.ts",