edgeflowjs 0.1.0 → 0.2.0

package/dist/pipelines/image-segmentation.d.ts

@@ -0,0 +1,221 @@
+/**
+ * edgeFlow.js - Image Segmentation Pipeline
+ *
+ * Interactive image segmentation using SAM (Segment Anything Model).
+ * Supports point prompts and bounding box prompts.
+ */
+import { PipelineConfig, PipelineOptions } from '../core/types.js';
+import { EdgeFlowTensor } from '../core/tensor.js';
+import { BasePipeline, PipelineResult } from './base.js';
+/**
+ * Point prompt for segmentation
+ */
+export interface PointPrompt {
+    /** X coordinate (0-1 normalized) */
+    x: number;
+    /** Y coordinate (0-1 normalized) */
+    y: number;
+    /** 1 for foreground (include), 0 for background (exclude) */
+    label: 0 | 1;
+}
+/**
+ * Box prompt for segmentation
+ */
+export interface BoxPrompt {
+    /** Top-left X (0-1 normalized) */
+    x1: number;
+    /** Top-left Y (0-1 normalized) */
+    y1: number;
+    /** Bottom-right X (0-1 normalized) */
+    x2: number;
+    /** Bottom-right Y (0-1 normalized) */
+    y2: number;
+}
+/**
+ * Model loading progress callback
+ */
+export interface ModelLoadProgress {
+    /** Model name (encoder or decoder) */
+    model: 'encoder' | 'decoder';
+    /** Bytes loaded */
+    loaded: number;
+    /** Total bytes */
+    total: number;
+    /** Progress percentage (0-100) */
+    progress: number;
+}
+/**
+ * Segmentation options
+ */
+export interface ImageSegmentationOptions extends PipelineOptions {
+    /** Point prompts */
+    points?: PointPrompt[];
+    /** Box prompts */
+    boxes?: BoxPrompt[];
+    /** Return all masks or just the best one */
+    returnAllMasks?: boolean;
+    /** Mask threshold (0-1) */
+    maskThreshold?: number;
+}
+/**
+ * Segmentation result
+ */
+export interface ImageSegmentationResult extends PipelineResult {
+    /** Segmentation mask (Uint8Array, 0 or 255) */
+    mask: Uint8Array;
+    /** Mask width */
+    width: number;
+    /** Mask height */
+    height: number;
+    /** Confidence score */
+    score: number;
+    /** All masks if returnAllMasks is true */
+    allMasks?: Array<{
+        mask: Uint8Array;
+        score: number;
+    }>;
+}
+/**
+ * Image input types
+ */
+export type ImageInput = HTMLImageElement | HTMLCanvasElement | ImageBitmap | ImageData | string;
+/**
+ * ImageSegmentationPipeline - Interactive image segmentation
+ *
+ * Uses SAM-style models for point/box prompted segmentation.
+ *
+ * @example
+ * ```typescript
+ * const segmenter = createImageSegmentationPipeline();
+ *
+ * // Load models with progress callback
+ * await segmenter.loadModels((progress) => {
+ *   console.log(`Loading ${progress.model}: ${progress.progress}%`);
+ * });
+ *
+ * // Set image and segment
+ * await segmenter.setImage(imageElement);
+ * const result = await segmenter.segment({
+ *   points: [{ x: 0.5, y: 0.5, label: 1 }]
+ * });
+ * ```
+ */
+export declare class ImageSegmentationPipeline extends BasePipeline<ImageInput, ImageSegmentationResult> {
+    private encoderModel;
+    private decoderModel;
+    private imageEmbedding;
+    private imagePositionalEmbedding;
+    private currentImageSize;
+    private resizedImageSize;
+    private inputSize;
+    private modelsLoaded;
+    private encoderUrl;
+    private decoderUrl;
+    constructor(config: PipelineConfig);
+    /**
+     * Check if models are loaded
+     */
+    get isModelsLoaded(): boolean;
+    /**
+     * Set custom model URLs
+     */
+    setModelUrls(encoder: string, decoder: string): void;
+    /**
+     * Load both encoder and decoder models with progress callback
+     */
+    loadModels(onProgress?: (progress: ModelLoadProgress) => void): Promise<void>;
+    /**
+     * Fetch model with progress tracking
+     */
+    private fetchModelWithProgress;
+    /**
+     * Initialize pipeline (override to skip default model loading)
+     */
+    initialize(): Promise<void>;
+    /**
+     * Load encoder model (processes the image once)
+     */
+    loadEncoder(modelUrl: string): Promise<void>;
+    /**
+     * Load decoder model (processes prompts to generate masks)
+     */
+    loadDecoder(modelUrl: string): Promise<void>;
+    /**
+     * Set and encode the image (call once per image)
+     */
+    setImage(image: ImageInput): Promise<void>;
+    /**
+     * Segment the image with given prompts
+     */
+    segment(options?: ImageSegmentationOptions): Promise<ImageSegmentationResult>;
+    /**
+     * Run segmentation (implements BasePipeline interface)
+     */
+    run(input: ImageInput, options?: ImageSegmentationOptions): Promise<ImageSegmentationResult>;
+    /**
+     * Load image from various sources
+     */
+    private loadImage;
+    /**
+     * Load image from URL
+     */
+    private loadImageFromUrl;
+    /**
+     * Convert HTMLImageElement to ImageData
+     */
+    private imageElementToImageData;
+    /**
+     * Convert canvas to ImageData
+     */
+    private canvasToImageData;
+    /**
+     * Convert ImageBitmap to ImageData
+     */
+    private imageBitmapToImageData;
+    /**
+     * Preprocess image for SAM
+     */
+    private preprocessImage;
+    /**
+     * Prepare decoder inputs (prompts) for SlimSAM
+     *
+     * SlimSAM prompt_encoder_mask_decoder expects these named inputs:
+     * - image_embeddings: [1, 256, 64, 64]
+     * - point_coords: [batch, num_points, 2]
+     * - point_labels: [batch, num_points]
+     * - mask_input: [batch, 1, 256, 256]
+     * - has_mask_input: [batch, 1]
+     * - orig_im_size: [2]
+     * - position_ids: [batch, num_points]
+     */
+    private prepareDecoderInputs;
+    /**
+     * Post-process masks from decoder output
+     */
+    private postprocessMasks;
+    /**
+     * Resize mask from model output size to original image size
+     */
+    private resizeMask;
+    /**
+     * Clear the current image embedding
+     */
+    clearImage(): void;
+    /**
+     * Preprocess (required by BasePipeline)
+     */
+    protected preprocess(input: ImageInput): Promise<EdgeFlowTensor[]>;
+    /**
+     * Postprocess (required by BasePipeline)
+     */
+    protected postprocess(_outputs: EdgeFlowTensor[], _options?: PipelineOptions): Promise<ImageSegmentationResult>;
+    /**
+     * Dispose resources
+     */
+    dispose(): void;
+}
+/**
+ * Create image segmentation pipeline
+ */
+export declare function createImageSegmentationPipeline(config?: Partial<PipelineConfig>): ImageSegmentationPipeline;
+//# sourceMappingURL=image-segmentation.d.ts.map

package/dist/pipelines/image-segmentation.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"image-segmentation.d.ts","sourceRoot":"","sources":["../../src/pipelines/image-segmentation.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EACL,cAAc,EACd,eAAe,EAEhB,MAAM,kBAAkB,CAAC;AAC1B,OAAO,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AACnD,OAAO,EAAE,YAAY,EAAE,cAAc,EAAoB,MAAM,WAAW,CAAC;AAgB3E;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,oCAAoC;IACpC,CAAC,EAAE,MAAM,CAAC;IACV,oCAAoC;IACpC,CAAC,EAAE,MAAM,CAAC;IACV,6DAA6D;IAC7D,KAAK,EAAE,CAAC,GAAG,CAAC,CAAC;CACd;AAED;;GAEG;AACH,MAAM,WAAW,SAAS;IACxB,kCAAkC;IAClC,EAAE,EAAE,MAAM,CAAC;IACX,kCAAkC;IAClC,EAAE,EAAE,MAAM,CAAC;IACX,sCAAsC;IACtC,EAAE,EAAE,MAAM,CAAC;IACX,sCAAsC;IACtC,EAAE,EAAE,MAAM,CAAC;CACZ;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,sCAAsC;IACtC,KAAK,EAAE,SAAS,GAAG,SAAS,CAAC;IAC7B,mBAAmB;IACnB,MAAM,EAAE,MAAM,CAAC;IACf,kBAAkB;IAClB,KAAK,EAAE,MAAM,CAAC;IACd,kCAAkC;IAClC,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED;;GAEG;AACH,MAAM,WAAW,wBAAyB,SAAQ,eAAe;IAC/D,oBAAoB;IACpB,MAAM,CAAC,EAAE,WAAW,EAAE,CAAC;IACvB,kBAAkB;IAClB,KAAK,CAAC,EAAE,SAAS,EAAE,CAAC;IACpB,4CAA4C;IAC5C,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB,2BAA2B;IAC3B,aAAa,CAAC,EAAE,MAAM,CAAC;CACxB;AAED;;GAEG;AACH,MAAM,WAAW,uBAAwB,SAAQ,cAAc;IAC7D,+CAA+C;IAC/C,IAAI,EAAE,UAAU,CAAC;IACjB,iBAAiB;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,kBAAkB;IAClB,MAAM,EAAE,MAAM,CAAC;IACf,uBAAuB;IACvB,KAAK,EAAE,MAAM,CAAC;IACd,0CAA0C;IAC1C,QAAQ,CAAC,EAAE,KAAK,CAAC;QAAE,IAAI,EAAE,UAAU,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;CACvD;AAED;;GAEG;AACH,MAAM,MAAM,UAAU,GAClB,gBAAgB,GAChB,iBAAiB,GACjB,WAAW,GACX,SAAS,GACT,MAAM,CAAC;AAMX;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,qBAAa,yBAA0B,SAAQ,YAAY,CACzD,UAAU,EACV,uBAAuB,CACxB;IACC,OAAO,CAAC,YAAY,CAA4B;IAChD,OAAO,CAAC,YAAY,CAA4B;IAChD,OAAO,CAAC,cAAc,CAA+B;IACrD,OAAO,CAAC,wBAAwB,CAA+B;IAC/D,OAAO,CAAC,gBAAgB,CAAkD;IAC1E,OAAO,CAAC,gBAAgB,CAAkD;IAC1E,OAAO,CAAC,SAAS,CAAgB;IACjC,OAAO,CAAC,YAAY,CAAkB;IAGtC,OAAO,CAAC,UAAU,CAAS;IAC3B,OAAO,CAAC,UAAU,CAAS;gBAEf,MAAM,EAAE,cAAc;IAMlC;;OAEG;IACH,IAAI,cAAc,IAAI,OAAO,CAE5B;IAED;;OAEG;IACH,YAAY,CAAC,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,IAAI;IAKpD;;OAEG;IACG,UAAU,CACd,UAAU,CAAC,EAAE,CAAC,QAAQ,EAAE,iBAAiB,KAAK,IAAI,GACjD,OAAO,CAAC,IAAI,CAAC;IA4ChB;;OAEG;YACW,sBAAsB;IA6CpC;;OAEG;IACY,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;IAM1C;;OAEG;IACG,WAAW,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAMlD;;OAEG;IACG,WAAW,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAMlD;;OAEG;IACG,QAAQ,CAAC,KAAK,EAAE,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC;IAgChD;;OAEG;IACG,OAAO,CAAC,OAAO,GAAE,wBAA6B,GAAG,OAAO,CAAC,uBAAuB,CAAC;IAuCvF;;OAEG;IACY,GAAG,CAChB,KAAK,EAAE,UAAU,EACjB,OAAO,CAAC,EAAE,wBAAwB,GACjC,OAAO,CAAC,uBAAuB,CAAC;IAKnC;;OAEG;YACW,SAAS;IAiBvB;;OAEG;YACW,gBAAgB;IAiB9B;;OAEG;IACH,OAAO,CAAC,uBAAuB;IAS/B;;OAEG;IACH,OAAO,CAAC,iBAAiB;IAKzB;;OAEG;IACH,OAAO,CAAC,sBAAsB;IAS9B;;OAEG;IACH,OAAO,CAAC,eAAe;IAoDvB;;;;;;;;;;;OAWG;IACH,OAAO,CAAC,oBAAoB;IA8D5B;;OAEG;IACH,OAAO,CAAC,gBAAgB;IAmExB;;OAEG;IACH,OAAO,CAAC,UAAU;IAgDlB;;OAEG;IACH,UAAU,IAAI,IAAI;IAOlB;;OAEG;cACsB,UAAU,CAAC,KAAK,EAAE,UAAU,GAAG,OAAO,CAAC,cAAc,EAAE,CAAC;IAMjF;;OAEG;cACsB,WAAW,CAClC,QAAQ,EAAE,cAAc,EAAE,EAC1B,QAAQ,CAAC,EAAE,eAAe,GACzB,OAAO,CAAC,uBAAuB,CAAC;IAUnC;;OAEG;IACM,OAAO,IAAI,IAAI;CAUzB;AAMD;;GAEG;AACH,wBAAgB,+BAA+B,CAC7C,MAAM,GAAE,OAAO,CAAC,cAAc,CAAM,GACnC,yBAAyB,CAQ3B"}