longpipe 0.0.11 → 0.0.13

package/dist/index.d.cts

@@ -62,6 +62,56 @@ interface UpsampleParams {
     outH: number;
     outW: number;
 }
+interface FaceBoxParams {
+    win: number;
+    thresh: number;
+    boxScale: number;
+}
+interface FaceBoxesParams extends FaceBoxParams {
+    maxFaces: number;
+    tol: number;
+}
+interface CropResampleParams {
+    outH: number;
+    outW: number;
+    mean: [number, number, number];
+    std: [number, number, number];
+    slot?: number;
+}
+interface LandmarkOverlayParams {
+    count: number;
+    thresh: number;
+    pointSize: number;
+    color: [number, number, number];
+    slot?: number;
+    drawImage?: boolean;
+}
+interface ReframeStateParams {
+    zoom: number;
+    gravity: number;
+    margin: number;
+    deadband: number;
+    ease: number;
+    aspect: number;
+}
+interface FaceTouchupStageOp extends Op {
+    setActiveSlots(n: number): void;
+}
+interface FaceTopology {
+    count: number;
+    uv: Float32Array;
+    idx: Float32Array;
+    weightMask: ImageBitmap;
+}
+type FaceTouchupStyle = "freq-sep" | "bilateral";
+interface FaceTouchupParams {
+    strength: number;
+    amount: number;
+    detail: number;
+    thresh: number;
+    style?: FaceTouchupStyle;
+    slots?: number;
+}
 interface UpsampleConv1x1Params {
     outH: number;
     outW: number;
@@ -102,6 +152,12 @@ interface Backend {
         Tanh: (input: Tensor) => Op;
         ElementwiseMul: (a: Tensor, b: Tensor) => Op;
         Warp: (source: Tensor, flow: Tensor, params: WarpParams) => Op;
+        FaceBoxFromHeatmaps: (heatmaps: Tensor, params: FaceBoxParams) => Op;
+        FaceBoxesFromHeatmaps: (heatmaps: Tensor, params: FaceBoxesParams) => Op;
+        ReframeState: (boxes: Tensor, prev: Tensor, cmd: Tensor, params: ReframeStateParams) => Op;
+        Reframe: (src: Tensor, rect: Tensor) => Op;
+        FaceTouchupStage: (frame: Tensor, landmarks: Tensor, box: Tensor, topo: FaceTopology, params: FaceTouchupParams) => FaceTouchupStageOp;
+        CropResample: (frame: Tensor, box: Tensor, params: CropResampleParams) => Op;
         Stabilize: (flow: Tensor, pred: Tensor, ref: Tensor, envPrev: Tensor, params: StabilizeParams) => Op;
         GatesFused: (uIn: Tensor, hPrev: Tensor, weights: Conv2DWeights) => Op;
         CandUpdateFused: (uIn: Tensor, hPrev: Tensor, gatesOut: Tensor, weights: Conv2DWeights, gamma: ArrayLike<number>) => Op;
@@ -123,6 +179,8 @@ interface Backend {
         CompositeImage: (image: Tensor, alpha: Tensor, bg: Tensor, target?: RenderTarget) => Presenter;
         CompositeImageBilinear: (image: Tensor, alpha: Tensor, bg: Tensor, target?: RenderTarget) => Presenter;
         CompositePassthrough: (image: Tensor, target?: RenderTarget) => Presenter;
+        LandmarkOverlay: (image: Tensor, landmarks: Tensor, box: Tensor, params: LandmarkOverlayParams, target?: RenderTarget) => Presenter;
+        FaceTouchup: (frame: Tensor, landmarks: Tensor, box: Tensor, topo: FaceTopology, params: FaceTouchupParams, target?: RenderTarget) => Presenter;
     };
     attachCanvas(name: RenderTarget, canvas: HTMLCanvasElement | OffscreenCanvas): void;
     readback(tensor: Tensor): Promise<Float32Array>;
@@ -194,7 +252,7 @@ interface RendererStats {
     inputPath: string;
     outputPath: string;
 }
-type ErrorSource = 'backend-lost' | 'worker' | 'adaptive' | 'background' | 'unknown';
+type ErrorSource = 'backend-lost' | 'worker' | 'adaptive' | 'background' | 'audio' | 'unknown';
 interface PipelineError {
     message: string;
     source: ErrorSource;
@@ -202,23 +260,65 @@ interface PipelineError {
     cause?: unknown;
 }
 
+type DenoiseModel = 'rnnoise' | 'dfn' | 'dfnint8';
+type DenoiseTier = 'high' | 'mid' | 'low';
+type DenoiseModelOption = 'auto' | DenoiseTier | DenoiseModel;
+
+interface DenoiseOptions {
+    model?: DenoiseModelOption;
+    postFilterBeta?: number;
+    gruLeak?: number;
+    enabled?: boolean;
+}
+type AudioInput = 'passthrough' | 'drop' | 'denoise' | {
+    denoise: DenoiseOptions;
+};
 type AudioMode = 'passthrough' | 'drop';
 
+interface AudioStats {
+    model: DenoiseModel;
+    p50Ms: number | null;
+    p95Ms: number | null;
+    latencyMs: number;
+    active: boolean;
+    sampleRate: number;
+}
+
 interface PipelineOptions {
     background?: BackgroundInput;
     preset?: PresetName | ManualPreset;
     weightsBaseUrl?: string;
-    audio?: AudioMode;
+    dtype?: Dtype;
+    audio?: AudioInput;
     enabled?: boolean;
     outputResolution?: {
         w: number;
         h: number;
     };
     adaptive?: boolean;
+    touchup?: TouchupOptions;
+    reframe?: ReframeConfig;
     onReady?: () => void;
     onError?: (err: PipelineError) => void;
     debug?: boolean;
 }
+interface TouchupOptions {
+    strength?: number;
+    amount?: number;
+    detail?: number;
+    style?: FaceTouchupStyle;
+}
+type ReframeConfig = boolean | ReframeOptions;
+interface ReframeOptions {
+    zoom?: number;
+    gravity?: number;
+    margin?: number;
+    auto?: boolean | ReframeAutoOptions;
+}
+interface ReframeAutoOptions {
+    deadband?: number;
+    ease?: number;
+}
 declare class Pipeline implements PromiseLike<Pipeline> {
     readonly stream: MediaStream;
     readonly ready: Promise<void>;
@@ -227,12 +327,19 @@ declare class Pipeline implements PromiseLike<Pipeline> {
     private inputCleanup;
     private outputCleanup;
     private adaptive;
+    private denoiser;
+    private touchupAssets;
+    private weightsBaseUrl;
+    private resolvedDtype;
     private bgCleanup;
     private previewBgCleanup;
     constructor(inputStream: MediaStream, options: PipelineOptions);
     private bootstrap;
     then<T1 = this, T2 = never>(onFulfilled?: ((value: this) => T1 | PromiseLike<T1>) | null, onRejected?: ((reason: unknown) => T2 | PromiseLike<T2>) | null): PromiseLike<T1 | T2>;
     setBackground(input: BackgroundInput): Promise<void>;
+    setTouchup(opts: TouchupOptions | null): Promise<void>;
+    setReframe(cfg: ReframeConfig | null): Promise<void>;
+    reframe(): Promise<void>;
     setPreset(p: PresetName | ManualPreset, weights?: ArrayBuffer): void;
     attachPreview(canvas: HTMLCanvasElement): void;
     setPreview(input: {
@@ -242,6 +349,8 @@ declare class Pipeline implements PromiseLike<Pipeline> {
     clearPreview(): void;
     setEnabled(on: boolean): void;
     getStats(): Promise<RendererStats>;
+    setDenoise(input: boolean | DenoiseOptions): void;
+    getAudioStats(): AudioStats | null;
     destroy(): void;
 }
 
@@ -283,4 +392,4 @@ declare class WebGPUBackend implements Backend {
     destroy(): void;
 }
 
-export { type Activation, type AudioMode, type Backend, type Background, type BackgroundInput, type BlurInput, type ColorInput, type Conv2dParams, type DepthwiseParams, Pipeline as EffectsPipeline, type ErrorSource, type ImageInput, type ManualPreset, type ModelName, type Op, type PipelineError, type PipelineOptions, type PresetName, type Tensor, type VideoInput, WebGPUBackend };
+export { type Activation, type AudioInput, type AudioMode, type AudioStats, type Backend, type Background, type BackgroundInput, type BlurInput, type ColorInput, type Conv2dParams, type DenoiseModel, type DenoiseModelOption, type DenoiseOptions, type DenoiseTier, type DepthwiseParams, Pipeline as EffectsPipeline, type ErrorSource, type FaceTouchupStyle, type ImageInput, type ManualPreset, type ModelName, type Op, type PipelineError, type PipelineOptions, type PresetName, type Tensor, type TouchupOptions, type VideoInput, WebGPUBackend };

package/dist/index.d.ts

@@ -62,6 +62,56 @@ interface UpsampleParams {
     outH: number;
     outW: number;
 }
+interface FaceBoxParams {
+    win: number;
+    thresh: number;
+    boxScale: number;
+}
+interface FaceBoxesParams extends FaceBoxParams {
+    maxFaces: number;
+    tol: number;
+}
+interface CropResampleParams {
+    outH: number;
+    outW: number;
+    mean: [number, number, number];
+    std: [number, number, number];
+    slot?: number;
+}
+interface LandmarkOverlayParams {
+    count: number;
+    thresh: number;
+    pointSize: number;
+    color: [number, number, number];
+    slot?: number;
+    drawImage?: boolean;
+}
+interface ReframeStateParams {
+    zoom: number;
+    gravity: number;
+    margin: number;
+    deadband: number;
+    ease: number;
+    aspect: number;
+}
+interface FaceTouchupStageOp extends Op {
+    setActiveSlots(n: number): void;
+}
+interface FaceTopology {
+    count: number;
+    uv: Float32Array;
+    idx: Float32Array;
+    weightMask: ImageBitmap;
+}
+type FaceTouchupStyle = "freq-sep" | "bilateral";
+interface FaceTouchupParams {
+    strength: number;
+    amount: number;
+    detail: number;
+    thresh: number;
+    style?: FaceTouchupStyle;
+    slots?: number;
+}
 interface UpsampleConv1x1Params {
     outH: number;
     outW: number;
@@ -102,6 +152,12 @@ interface Backend {
         Tanh: (input: Tensor) => Op;
         ElementwiseMul: (a: Tensor, b: Tensor) => Op;
         Warp: (source: Tensor, flow: Tensor, params: WarpParams) => Op;
+        FaceBoxFromHeatmaps: (heatmaps: Tensor, params: FaceBoxParams) => Op;
+        FaceBoxesFromHeatmaps: (heatmaps: Tensor, params: FaceBoxesParams) => Op;
+        ReframeState: (boxes: Tensor, prev: Tensor, cmd: Tensor, params: ReframeStateParams) => Op;
+        Reframe: (src: Tensor, rect: Tensor) => Op;
+        FaceTouchupStage: (frame: Tensor, landmarks: Tensor, box: Tensor, topo: FaceTopology, params: FaceTouchupParams) => FaceTouchupStageOp;
+        CropResample: (frame: Tensor, box: Tensor, params: CropResampleParams) => Op;
         Stabilize: (flow: Tensor, pred: Tensor, ref: Tensor, envPrev: Tensor, params: StabilizeParams) => Op;
         GatesFused: (uIn: Tensor, hPrev: Tensor, weights: Conv2DWeights) => Op;
         CandUpdateFused: (uIn: Tensor, hPrev: Tensor, gatesOut: Tensor, weights: Conv2DWeights, gamma: ArrayLike<number>) => Op;
@@ -123,6 +179,8 @@ interface Backend {
         CompositeImage: (image: Tensor, alpha: Tensor, bg: Tensor, target?: RenderTarget) => Presenter;
         CompositeImageBilinear: (image: Tensor, alpha: Tensor, bg: Tensor, target?: RenderTarget) => Presenter;
         CompositePassthrough: (image: Tensor, target?: RenderTarget) => Presenter;
+        LandmarkOverlay: (image: Tensor, landmarks: Tensor, box: Tensor, params: LandmarkOverlayParams, target?: RenderTarget) => Presenter;
+        FaceTouchup: (frame: Tensor, landmarks: Tensor, box: Tensor, topo: FaceTopology, params: FaceTouchupParams, target?: RenderTarget) => Presenter;
     };
     attachCanvas(name: RenderTarget, canvas: HTMLCanvasElement | OffscreenCanvas): void;
     readback(tensor: Tensor): Promise<Float32Array>;
@@ -194,7 +252,7 @@ interface RendererStats {
     inputPath: string;
     outputPath: string;
 }
-type ErrorSource = 'backend-lost' | 'worker' | 'adaptive' | 'background' | 'unknown';
+type ErrorSource = 'backend-lost' | 'worker' | 'adaptive' | 'background' | 'audio' | 'unknown';
 interface PipelineError {
     message: string;
     source: ErrorSource;
@@ -202,23 +260,65 @@ interface PipelineError {
     cause?: unknown;
 }
 
+type DenoiseModel = 'rnnoise' | 'dfn' | 'dfnint8';
+type DenoiseTier = 'high' | 'mid' | 'low';
+type DenoiseModelOption = 'auto' | DenoiseTier | DenoiseModel;
+
+interface DenoiseOptions {
+    model?: DenoiseModelOption;
+    postFilterBeta?: number;
+    gruLeak?: number;
+    enabled?: boolean;
+}
+type AudioInput = 'passthrough' | 'drop' | 'denoise' | {
+    denoise: DenoiseOptions;
+};
 type AudioMode = 'passthrough' | 'drop';
 
+interface AudioStats {
+    model: DenoiseModel;
+    p50Ms: number | null;
+    p95Ms: number | null;
+    latencyMs: number;
+    active: boolean;
+    sampleRate: number;
+}
+
 interface PipelineOptions {
     background?: BackgroundInput;
     preset?: PresetName | ManualPreset;
     weightsBaseUrl?: string;
-    audio?: AudioMode;
+    dtype?: Dtype;
+    audio?: AudioInput;
     enabled?: boolean;
     outputResolution?: {
         w: number;
         h: number;
     };
     adaptive?: boolean;
+    touchup?: TouchupOptions;
+    reframe?: ReframeConfig;
     onReady?: () => void;
     onError?: (err: PipelineError) => void;
     debug?: boolean;
 }
+interface TouchupOptions {
+    strength?: number;
+    amount?: number;
+    detail?: number;
+    style?: FaceTouchupStyle;
+}
+type ReframeConfig = boolean | ReframeOptions;
+interface ReframeOptions {
+    zoom?: number;
+    gravity?: number;
+    margin?: number;
+    auto?: boolean | ReframeAutoOptions;
+}
+interface ReframeAutoOptions {
+    deadband?: number;
+    ease?: number;
+}
 declare class Pipeline implements PromiseLike<Pipeline> {
     readonly stream: MediaStream;
     readonly ready: Promise<void>;
@@ -227,12 +327,19 @@ declare class Pipeline implements PromiseLike<Pipeline> {
     private inputCleanup;
     private outputCleanup;
     private adaptive;
+    private denoiser;
+    private touchupAssets;
+    private weightsBaseUrl;
+    private resolvedDtype;
     private bgCleanup;
     private previewBgCleanup;
     constructor(inputStream: MediaStream, options: PipelineOptions);
     private bootstrap;
     then<T1 = this, T2 = never>(onFulfilled?: ((value: this) => T1 | PromiseLike<T1>) | null, onRejected?: ((reason: unknown) => T2 | PromiseLike<T2>) | null): PromiseLike<T1 | T2>;
     setBackground(input: BackgroundInput): Promise<void>;
+    setTouchup(opts: TouchupOptions | null): Promise<void>;
+    setReframe(cfg: ReframeConfig | null): Promise<void>;
+    reframe(): Promise<void>;
     setPreset(p: PresetName | ManualPreset, weights?: ArrayBuffer): void;
     attachPreview(canvas: HTMLCanvasElement): void;
     setPreview(input: {
@@ -242,6 +349,8 @@ declare class Pipeline implements PromiseLike<Pipeline> {
     clearPreview(): void;
     setEnabled(on: boolean): void;
     getStats(): Promise<RendererStats>;
+    setDenoise(input: boolean | DenoiseOptions): void;
+    getAudioStats(): AudioStats | null;
     destroy(): void;
 }
 
@@ -283,4 +392,4 @@ declare class WebGPUBackend implements Backend {
     destroy(): void;
 }
 
-export { type Activation, type AudioMode, type Backend, type Background, type BackgroundInput, type BlurInput, type ColorInput, type Conv2dParams, type DepthwiseParams, Pipeline as EffectsPipeline, type ErrorSource, type ImageInput, type ManualPreset, type ModelName, type Op, type PipelineError, type PipelineOptions, type PresetName, type Tensor, type VideoInput, WebGPUBackend };
+export { type Activation, type AudioInput, type AudioMode, type AudioStats, type Backend, type Background, type BackgroundInput, type BlurInput, type ColorInput, type Conv2dParams, type DenoiseModel, type DenoiseModelOption, type DenoiseOptions, type DenoiseTier, type DepthwiseParams, Pipeline as EffectsPipeline, type ErrorSource, type FaceTouchupStyle, type ImageInput, type ManualPreset, type ModelName, type Op, type PipelineError, type PipelineOptions, type PresetName, type Tensor, type TouchupOptions, type VideoInput, WebGPUBackend };