@pixerate/editor 0.179.0 → 0.180.0

package/dist/image-editor/index.js

@@ -948,6 +948,24 @@ function quickMaskPixels(mask, style = {}) {
   }
   return out;
 }
+var MARCHING_ANTS_STEP_MS = 100;
+function prefersReducedMotion() {
+  try {
+    return typeof matchMedia === "function" && matchMedia("(prefers-reduced-motion: reduce)").matches;
+  } catch {
+    return false;
+  }
+}
+function startMarchingAnts(tick, stepMs = MARCHING_ANTS_STEP_MS) {
+  if (prefersReducedMotion()) return () => {
+  };
+  let offset = 0;
+  const id = setInterval(() => {
+    offset = (offset + 1) % 8;
+    tick(offset);
+  }, stepMs);
+  return () => clearInterval(id);
+}
 
 // src/image-editor/layers/render.ts
 function isQuarterTurn(rotate) {
@@ -1353,8 +1371,8 @@ var ImageEditorRenderer = class {
         ctx.rect(region.x, region.y, region.width, region.height);
         ctx.clip();
         if (state.selection?.bounds && display === "quickMask") this.drawQuickMask(ctx, state.selection);
-        else if (state.selection?.bounds && display === "ants") this.drawAnts(ctx, state.selection, projection.scale, options.antsOffset ?? 0);
-        if (options.draftSelectionPath?.length) this.drawDraftPath(ctx, options.draftSelectionPath, projection.scale);
+        else if (state.selection?.bounds && display === "ants") this.drawAnts(ctx, state.selection, projection.scale / u, options.antsOffset ?? 0);
+        if (options.draftSelectionPath?.length) this.drawDraftPath(ctx, options.draftSelectionPath, projection.scale / u);
       } catch {
       }
       ctx.restore();
@@ -1772,7 +1790,10 @@ var ImageEditorRenderer = class {
     }
     ctx.drawImage(canvas, 0, 0, selection.width, selection.height);
   }
-  /** Marching ants: a white line under a black dashed one, one screen pixel wide. */
+  /**
+   * Marching ants: a white line under a black dashed one, one screen pixel wide. `scale` is
+   * screen pixels per natural pixel (the projection scale times the host's CSS scale).
+   */
   drawAnts(ctx, selection, scale, offset) {
     let segs = this.outlineCache.get(selection);
     if (!segs) {
@@ -3082,6 +3103,35 @@ function hueRange(pixels, center) {
   return createSelectionMask(pixels.width, pixels.height, out);
 }
 
+// src/image-editor/selection/segmentation.ts
+var isPoint = (p) => !!p && typeof p === "object" && Number.isFinite(p.x) && Number.isFinite(p.y);
+function polygonsOf(result) {
+  const p = result.polygon;
+  if (!Array.isArray(p) || !p.length) return [];
+  const rings = Array.isArray(p[0]) ? p : [p];
+  return rings.map((ring) => Array.isArray(ring) ? ring.filter(isPoint) : []).filter((ring) => ring.length >= 3);
+}
+function segmentToSelection(width, height, result) {
+  let out = null;
+  for (const ring of polygonsOf(result)) {
+    out = combineSelections(out, polygonSelection(width, height, ring), "add");
+  }
+  if (result.mask && result.mask.width > 0 && result.mask.height > 0) {
+    const raw = selectionFromPixels(result.mask, { channel: "luminance" });
+    const sized = raw.width === width && raw.height === height ? raw : resizeSelection(raw, width, height);
+    out = combineSelections(out, sized, "add");
+  }
+  return out?.bounds ? out : null;
+}
+function segmentsToSelection(width, height, results) {
+  let out = null;
+  for (const r of results) {
+    const sel = segmentToSelection(width, height, r);
+    if (sel) out = combineSelections(out, sel, "add");
+  }
+  return out;
+}
+
 // src/image-editor/selection/refine.ts
 var DEFAULT_REFINE_EDGE_PARAMS = {
   radius: 0,
@@ -4669,6 +4719,8 @@ var ImageEditorController = class {
   viewKey = [];
   /** For `compositePixels` and mask evaluation (the renderer keeps its own for the viewport). */
   compositor = new LayerCompositor();
+  /** The segmentation provider behind `selectSubject` (see `setSegmentationProvider`). */
+  segmentation = null;
   /** The effects provider slot (see `setEffectsProvider`). */
   effects = new EffectsSlot();
   /** Renders provider effects: effect smart filter results and effect layers. */
@@ -4678,6 +4730,7 @@ var ImageEditorController = class {
     this.effects.subscribe(() => this.queueEffectRefresh("all"));
     this.effectsEngine.subscribe(() => this.queueEffectRefresh("incomplete"));
     if (options.effects) this.effects.set(options.effects);
+    if (options.segmentation) this.segmentation = options.segmentation;
     this.maxHistoryDepth = options.maxHistoryDepth ?? 50;
     if (options.textShaper !== void 0) this.textShaperValue = options.textShaper;
     this.maxHistoryBytes = options.maxHistoryBytes ?? 512 * 1024 * 1024;
@@ -4859,6 +4912,16 @@ var ImageEditorController = class {
     this.state.layers = next;
     this.notify();
   }
+  /** The registered segmentation provider, or `null` (then `selectSubject` is unavailable). */
+  get segmentationProvider() {
+    return this.segmentation;
+  }
+  /** Registers (or with `null` removes) the segmentation provider. Not an edit: no undo step. */
+  setSegmentationProvider(provider) {
+    if (this.segmentation === provider) return;
+    this.segmentation = provider;
+    this.notify();
+  }
   /** The registered effects provider, or `null`. */
   get effectsProvider() {
     return this.effects.provider;
@@ -7615,23 +7678,50 @@ var ImageEditorController = class {
    * RGBA pixels, by luminance (white = selected) or a channel. Any size; it is resampled.
    */
   async loadSelectionFromImage(source, options = {}) {
-    let pixels;
-    if (typeof source === "string") {
-      const img = await this.renderer.imageCache.get(source);
-      const w = img.naturalWidth || img.width;
-      const h = img.naturalHeight || img.height;
-      const canvas = this.renderer.createCanvas(w, h);
-      const ctx = canvas.getContext("2d");
-      if (ctx) ctx.drawImage(img, 0, 0, w, h);
-      const data = ctx?.getImageData?.(0, 0, w, h);
-      if (!data) throw new Error("The mask image pixels are not available (no canvas access); pass pixels.");
-      pixels = { width: w, height: h, data: data.data };
-    } else {
-      pixels = source;
-    }
+    const pixels = await this.decodeMaskImage(source);
     this.setSelection(selectionFromPixels(pixels, options), options.mode ?? "replace");
     return this.getSelection();
   }
+  /** A mask image (data URL / URL, or pixels as they are) as RGBA pixels. */
+  async decodeMaskImage(source) {
+    if (typeof source !== "string") return source;
+    const img = await this.renderer.imageCache.get(source);
+    const w = img.naturalWidth || img.width;
+    const h = img.naturalHeight || img.height;
+    const canvas = this.renderer.createCanvas(w, h);
+    const ctx = canvas.getContext("2d");
+    if (ctx) ctx.drawImage(img, 0, 0, w, h);
+    const data = ctx?.getImageData?.(0, 0, w, h);
+    if (!data) throw new Error("The mask image pixels are not available (no canvas access); pass pixels.");
+    return { width: w, height: h, data: data.data };
+  }
+  /**
+   * Select › Subject: asks the segmentation provider for the objects matching `target` ("the
+   * dog", "the sky") in the flattened image and selects their shapes, combined with the
+   * selection by `mode`, as one undo step. Returns what was found (labels and bounds in natural
+   * pixels); an empty list leaves the selection as it was. Throws without a provider.
+   */
+  async selectSubject(target, options = {}) {
+    const provider = this.segmentation;
+    if (!provider) throw new Error("Select Subject needs a segmentation provider (setSegmentationProvider).");
+    const description = target.trim();
+    if (!description) throw new Error("Describe the subject to select.");
+    const { width, height } = this.naturalSize;
+    const image = await this.toDataURL({ format: "image/png" });
+    const results = await provider.segment({ image, width, height, target: description, maxSegments: options.maxSegments, signal: options.signal });
+    options.signal?.throwIfAborted();
+    const decoded = [];
+    for (const r of results ?? []) {
+      decoded.push({ ...r, mask: r.mask !== void 0 ? await this.decodeMaskImage(r.mask) : void 0 });
+    }
+    const segments = decoded.map((r) => {
+      const one = segmentsToSelection(width, height, [r]);
+      return { label: r.label || description, bounds: one?.bounds ?? null, ...r.score !== void 0 ? { score: r.score } : {} };
+    });
+    const mask = segmentsToSelection(width, height, decoded);
+    if (mask) this.shapeCommand(mask, options);
+    return { segments: segments.filter((s) => s.bounds) };
+  }
   /** Quick Mask view (tints the unselected area). View state: no undo step. */
   setQuickMask(on) {
     if (Boolean(this.state.quickMask) === on) return;
@@ -10022,7 +10112,7 @@ var DEPTH_MASK_CONTROLS = [
 // src/image-editor/playback.ts
 var IMAGE_PLAYBACK_KIND = "layer";
 var PAINT = /^(paint_stroke|clone_stamp|heal_brush|spot_heal|dodge|burn|sponge|smudge|blur_stroke|sharpen_stroke|history_brush|set_clone_source|set_clone_transform|fill_pixels|clear_pixels|set_paint|set_retouch_options|load_mixer_brush|clean_mixer_brush)$/;
-var SELECT = /^(select_region|select_by_color|modify_selection|refine_selection|transform_selection|apply_selection_as_mask|layer_via_copy)$/;
+var SELECT = /^(select_region|select_subject|select_by_color|modify_selection|refine_selection|transform_selection|apply_selection_as_mask|layer_via_copy)$/;
 var ANNOTATE = /^(add_shape_layer|update_shape|add_shape_component|set_shape_component|remove_shape_component|merge_shape_components|add_text_layer|set_text|set_text_style|set_paragraph_style|set_text_frame|convert_text_to_shape)$/;
 var CROP = /^(set_crop|reset_crop|set_rotation|flip_image|transform_layer|warp_layer|flip_layer|offset_layer)$/;
 var ADJUST = /^(set_adjustments|add_adjustment_layer|update_adjustment_layer|remove_adjustment_layer|move_adjustment_layer)$/;
@@ -10043,7 +10133,7 @@ function createImagePlaybackLocator(controller) {
   };
 }
 var num = (v) => typeof v === "number" && Number.isFinite(v);
-var isPoint = (p) => !!p && typeof p === "object" && num(p.x) && num(p.y);
+var isPoint2 = (p) => !!p && typeof p === "object" && num(p.x) && num(p.y);
 function trace(points, max = 32) {
   const list = points.length <= max ? points : Array.from({ length: max }, (_, i) => points[Math.round(i * (points.length - 1) / (max - 1))]);
   if (!list.length) return [];
@@ -10079,7 +10169,7 @@ function planImageToolPlayback(tool, args) {
   const steps = [];
   const panel = imagePlaybackPanel(tool);
   if (panel) steps.push({ type: "open", ref: { kind: "panel", id: panel } });
-  const points = Array.isArray(args.points) ? args.points.filter(isPoint) : [];
+  const points = Array.isArray(args.points) ? args.points.filter(isPoint2) : [];
   const box = boxOf(args);
   if (points.length) steps.push(...trace(points));
   else if (box) steps.push({ type: "move", to: box.from }, { type: "press" }, { type: "drag", to: box.to }, { type: "release" });
@@ -10212,15 +10302,26 @@ function createImageSelectionTools(controller) {
   const selectRegion = {
     name: "select_region",
     title: "Select a region",
-    description: 'Selects a rectangle or ellipse (x, y, width, height in natural image pixels), everything ("all"), nothing ("none") or the inverse of the current selection ("invert"). One undoable step.',
+    description: 'Selects a rectangle or ellipse (x, y, width, height in natural image pixels), a polygon (the lasso: points along an outline, closed automatically), everything ("all"), nothing ("none") or the inverse of the current selection ("invert"). To cut out a subject, trace its contour with polygon (then refine_selection for soft edges such as fur or foliage) rather than selecting its bounding box. One undoable step.',
     inputSchema: {
       type: "object",
       properties: {
-        shape: { type: "string", enum: ["rect", "ellipse", "all", "none", "invert"], description: "What to select." },
+        shape: { type: "string", enum: ["rect", "ellipse", "polygon", "all", "none", "invert"], description: "What to select." },
         x: { type: "number", description: "rect / ellipse: left edge in natural image pixels." },
         y: { type: "number", description: "rect / ellipse: top edge in natural image pixels." },
         width: { type: "number", description: "rect / ellipse: width in natural image pixels." },
         height: { type: "number", description: "rect / ellipse: height in natural image pixels." },
+        points: {
+          type: "array",
+          minItems: 3,
+          items: {
+            type: "object",
+            properties: { x: { type: "number" }, y: { type: "number" } },
+            required: ["x", "y"],
+            additionalProperties: false
+          },
+          description: "polygon: at least 3 vertices along the outline, in order, in natural image pixels. Follow the edge closely; more points give a tighter contour."
+        },
         mode: modeSchema,
         feather: { type: "number", minimum: 0, description: "Feather radius in pixels (default 0)." }
       },
@@ -10228,7 +10329,7 @@ function createImageSelectionTools(controller) {
       additionalProperties: false
     },
     annotations: writeAnnotations("Select a region", true),
-    handler: ({ shape, x, y, width, height, mode, feather }) => {
+    handler: ({ shape, x, y, width, height, points, mode, feather }) => {
       const opts = { mode: parseSelectionMode(mode), feather: num2(feather, 0) };
       switch (shape) {
         case "all":
@@ -10250,8 +10351,16 @@ function createImageSelectionTools(controller) {
           else controller.selectEllipse(rect, opts);
           break;
         }
+        case "polygon": {
+          const list = Array.isArray(points) ? points.map((p) => ({ x: num2(p?.x, NaN), y: num2(p?.y, NaN) })) : [];
+          if (list.length < 3 || !list.every((p) => Number.isFinite(p.x) && Number.isFinite(p.y))) {
+            throw new Error("polygon needs points: at least 3 { x, y } vertices.");
+          }
+          controller.selectPolygon(list, opts);
+          break;
+        }
         default:
-          throw new Error(`Unknown shape "${String(shape)}". Use rect, ellipse, all, none or invert.`);
+          throw new Error(`Unknown shape "${String(shape)}". Use rect, ellipse, polygon, all, none or invert.`);
       }
       return summary(controller);
     }
@@ -10391,6 +10500,53 @@ function createImageSelectionTools(controller) {
   };
   return [getSelection, selectRegion, selectByColor, modify, refine, applyMask];
 }
+function createImageSubjectSelectionTool(controller) {
+  return {
+    name: "select_subject",
+    title: "Select a subject",
+    description: 'Select \u203A Subject: an AI segmentation model finds the objects matching target (a description such as "the dog", "the house", "the sky") in the flattened image and selects their outlines, combined with the selection by mode. Use it to cut a subject out by its contour (then layer_via_copy for a layer of just that subject); describe one object precisely to get only that one. One undoable step. Returns the selection and the objects found (label, bounds); none found leaves the selection unchanged.',
+    inputSchema: {
+      type: "object",
+      properties: {
+        target: { type: "string", minLength: 1, description: "What to select, in words." },
+        maxSegments: {
+          type: "integer",
+          minimum: 1,
+          maximum: 32,
+          description: "Select at most this many matching objects (default the model\u2019s; 1 for a single subject)."
+        },
+        mode: modeSchema,
+        feather: { type: "number", minimum: 0, description: "Feather radius in pixels (default 0)." }
+      },
+      required: ["target"],
+      additionalProperties: false
+    },
+    annotations: { ...writeAnnotations("Select a subject"), openWorldHint: true },
+    handler: async ({ target, maxSegments, mode, feather }) => {
+      if (typeof target !== "string" || !target.trim()) throw new Error("target must describe what to select.");
+      const { segments } = await controller.selectSubject(target, {
+        mode: parseSelectionMode(mode),
+        feather: num2(feather, 0),
+        maxSegments: typeof maxSegments === "number" ? Math.max(1, Math.round(maxSegments)) : void 0
+      });
+      return { ...summary(controller), found: segments };
+    }
+  };
+}
+function createImageSegmentationToolProvider(controller, options = {}) {
+  const tool = createImageSubjectSelectionTool(controller);
+  return withImagePlayback(
+    {
+      id: options.id ?? "image-segmentation",
+      namespace: options.namespace,
+      title: "Image segmentation",
+      description: "Select \u203A Subject: select objects in the image by description through an AI segmentation model.",
+      skill: "pixerate-image-editor",
+      tools: () => controller.segmentationProvider ? [tool] : []
+    },
+    options.playbackSurface
+  );
+}
 
 // src/image-editor/capture.ts
 var NO_CANVAS = "Rendering is not available in this environment.";
@@ -11176,7 +11332,7 @@ function createImageLayerTools(controller) {
   const viaCopy = {
     name: "layer_via_copy",
     title: "New layer from the selection",
-    description: "Layer via Copy: copies the selected pixels of a pixel layer (layerId, default the active layer) to a new layer directly above it and makes it active. cut: true also removes them from the source (Layer via Cut). Needs a selection. One undo step. Returns the new layer.",
+    description: "Layer via Copy: copies the selected pixels of a pixel layer (layerId, default the active layer) to a new layer directly above it and makes it active. cut: true also removes them from the source (Layer via Cut). Needs a selection; the new layer takes the selection\u2019s shape, so select the subject\u2019s outline (select_region polygon, select_by_color) to isolate it, not its bounding box. One undo step. Returns the new layer.",
     inputSchema: {
       type: "object",
       properties: {
@@ -14746,6 +14902,7 @@ function layerPanels(layer, options = {}) {
   if (layer.mask) {
     panels.push({ id: "mask", title: "Mask Options" });
   }
+  panels.push({ id: "selection", title: "Selection" });
   return panels;
 }
 
@@ -15880,6 +16037,7 @@ export {
   LayerCompositor,
   LayerEffectError,
   LayerOperationError,
+  MARCHING_ANTS_STEP_MS,
   MAX_BATCH,
   MAX_BRUSH_SIZE,
   MAX_CANVAS_BACKING_SIDE,
@@ -16159,9 +16317,11 @@ export {
   createImagePlaybackLocator,
   createImagePsdTools,
   createImageRetouchTools,
+  createImageSegmentationToolProvider,
   createImageSelectionTools,
   createImageShapeTools,
   createImageSmartObjectTools,
+  createImageSubjectSelectionTool,
   createImageTransformTools,
   createImageTypeTools,
   createMetricTextShaper,
@@ -16602,6 +16762,8 @@ export {
   scalePaintGradient,
   seamlessClone,
   seedFromString,
+  segmentToSelection,
+  segmentsToSelection,
   selectionArea,
   selectionFromCoverage,
   selectionFromPixels,
@@ -16659,6 +16821,7 @@ export {
   srgbEncode,
   srgbToLinear,
   standardBlendMode,
+  startMarchingAnts,
   stepSlotTs,
   strokeGrainCatch,
   styleAt,

package/dist/image-editor/layers/panel.d.cts

@@ -123,6 +123,7 @@ export interface LayerPanelsOptions {
  *
  * `properties` ("Layer Properties") is first for every layer and includes the layer's Layer Style
  * (drop shadow, stroke, glow…: `layerEffectsPanel`) for layers that can take one, with or without a
- * provider. There is no separate `effects` (Layer Effects) or `smart-filters` panel.
+ * provider. There is no separate `effects` (Layer Effects) or `smart-filters` panel. `selection`
+ * is last for every layer, and with no active layer follows `canvas-size`.
  */
 export declare function layerPanels(layer: ImageLayer | null, options?: LayerPanelsOptions): LayerPanelDescriptor[];

package/dist/image-editor/layers/panel.d.ts

@@ -123,6 +123,7 @@ export interface LayerPanelsOptions {
  *
  * `properties` ("Layer Properties") is first for every layer and includes the layer's Layer Style
  * (drop shadow, stroke, glow…: `layerEffectsPanel`) for layers that can take one, with or without a
- * provider. There is no separate `effects` (Layer Effects) or `smart-filters` panel.
+ * provider. There is no separate `effects` (Layer Effects) or `smart-filters` panel. `selection`
+ * is last for every layer, and with no active layer follows `canvas-size`.
  */
 export declare function layerPanels(layer: ImageLayer | null, options?: LayerPanelsOptions): LayerPanelDescriptor[];

package/dist/image-editor/renderer.d.cts

@@ -176,7 +176,10 @@ export declare class ImageEditorRenderer {
     private outlineCache;
     /** Rubylith over the unselected area, drawn in natural image space. */
     private drawQuickMask;
-    /** Marching ants: a white line under a black dashed one, one screen pixel wide. */
+    /**
+     * Marching ants: a white line under a black dashed one, one screen pixel wide. `scale` is
+     * screen pixels per natural pixel (the projection scale times the host's CSS scale).
+     */
     private drawAnts;
     private drawDraftPath;
     private getSourceId;

package/dist/image-editor/renderer.d.ts

@@ -176,7 +176,10 @@ export declare class ImageEditorRenderer {
     private outlineCache;
     /** Rubylith over the unselected area, drawn in natural image space. */
     private drawQuickMask;
-    /** Marching ants: a white line under a black dashed one, one screen pixel wide. */
+    /**
+     * Marching ants: a white line under a black dashed one, one screen pixel wide. `scale` is
+     * screen pixels per natural pixel (the projection scale times the host's CSS scale).
+     */
     private drawAnts;
     private drawDraftPath;
     private getSourceId;

package/dist/image-editor/selection/index.d.cts

@@ -7,3 +7,4 @@ export * from './layer-mask.cjs';
 export * from './overlay.cjs';
 export * from './tool-options.cjs';
 export * from './gesture.cjs';
+export * from './segmentation.cjs';

package/dist/image-editor/selection/index.d.ts

@@ -7,3 +7,4 @@ export * from './layer-mask.js';
 export * from './overlay.js';
 export * from './tool-options.js';
 export * from './gesture.js';
+export * from './segmentation.js';

package/dist/image-editor/selection/overlay.d.cts

@@ -16,3 +16,11 @@ export interface QuickMaskStyle {
  * selection shows through clear, as Photoshop's rubylith overlay.
  */
 export declare function quickMaskPixels(mask: SelectionMask, style?: QuickMaskStyle): Uint8ClampedArray;
+/** How often marching ants step (ms); each step moves the dashes one screen pixel. */
+export declare const MARCHING_ANTS_STEP_MS = 100;
+/**
+ * Marches the ants: calls `tick` with the next `RenderOptions.antsOffset` every `stepMs` until the
+ * returned stop function is called. Hosts run it while a selection is shown and render on each
+ * tick. With reduced motion requested the ants stand still and `tick` is never called.
+ */
+export declare function startMarchingAnts(tick: (offset: number) => void, stepMs?: number): () => void;

package/dist/image-editor/selection/overlay.d.ts

@@ -16,3 +16,11 @@ export interface QuickMaskStyle {
  * selection shows through clear, as Photoshop's rubylith overlay.
  */
 export declare function quickMaskPixels(mask: SelectionMask, style?: QuickMaskStyle): Uint8ClampedArray;
+/** How often marching ants step (ms); each step moves the dashes one screen pixel. */
+export declare const MARCHING_ANTS_STEP_MS = 100;
+/**
+ * Marches the ants: calls `tick` with the next `RenderOptions.antsOffset` every `stepMs` until the
+ * returned stop function is called. Hosts run it while a selection is shown and render on each
+ * tick. With reduced motion requested the ants stand still and `tick` is never called.
+ */
+export declare function startMarchingAnts(tick: (offset: number) => void, stepMs?: number): () => void;

package/dist/image-editor/selection/segmentation.d.cts

@@ -0,0 +1,49 @@
+import type { CropBox, ImagePoint } from '../types.cjs';
+import { type PixelBuffer, type SelectionMask } from './mask.cjs';
+/** What a segmentation provider is asked for (see `SegmentationProvider`). */
+export interface SegmentationRequest {
+    /** The flattened image at its natural size, as a PNG data URL. */
+    image: string;
+    /** The image's natural size in pixels. */
+    width: number;
+    height: number;
+    /** What to find, in words ("the dog", "the sky", "every separate object"). */
+    target: string;
+    /** Return at most this many objects (the provider's default when absent). */
+    maxSegments?: number;
+    signal?: AbortSignal;
+}
+/**
+ * One object a provider found, in natural image pixels: an outline `polygon` (or several, for
+ * an object in pieces), or a `mask` covering the whole image (white = the object, any size;
+ * it is resampled), or both, which are combined.
+ */
+export interface SegmentationResult {
+    label?: string;
+    polygon?: ImagePoint[] | ImagePoint[][];
+    mask?: string | PixelBuffer;
+    /** The object's bounding box, when the provider knows it. */
+    box?: CropBox;
+    /** The provider's confidence, 0..1. */
+    score?: number;
+}
+/**
+ * Finds objects in an image by a description and returns their shapes: an AI segmentation
+ * model (SAM, Gemini, a background remover…) behind `ImageEditorController.selectSubject`.
+ */
+export interface SegmentationProvider {
+    /** Shown in the UI and in tool results (e.g. "Gemini"). */
+    readonly name?: string;
+    segment(request: SegmentationRequest): Promise<SegmentationResult[]>;
+}
+/** A segmentation result with its `mask` already decoded to pixels (strings are decoded by the controller). */
+export type DecodedSegmentationResult = Omit<SegmentationResult, 'mask'> & {
+    mask?: PixelBuffer;
+};
+/**
+ * One segmentation result as a selection over a `width × height` image, or null when it has no
+ * usable shape. Polygons are anti-aliased; several polygons and a mask are added together.
+ */
+export declare function segmentToSelection(width: number, height: number, result: DecodedSegmentationResult): SelectionMask | null;
+/** Every usable result added into one selection (null when none has a shape). */
+export declare function segmentsToSelection(width: number, height: number, results: readonly DecodedSegmentationResult[]): SelectionMask | null;

package/dist/image-editor/selection/segmentation.d.ts

@@ -0,0 +1,49 @@
+import type { CropBox, ImagePoint } from '../types.js';
+import { type PixelBuffer, type SelectionMask } from './mask.js';
+/** What a segmentation provider is asked for (see `SegmentationProvider`). */
+export interface SegmentationRequest {
+    /** The flattened image at its natural size, as a PNG data URL. */
+    image: string;
+    /** The image's natural size in pixels. */
+    width: number;
+    height: number;
+    /** What to find, in words ("the dog", "the sky", "every separate object"). */
+    target: string;
+    /** Return at most this many objects (the provider's default when absent). */
+    maxSegments?: number;
+    signal?: AbortSignal;
+}
+/**
+ * One object a provider found, in natural image pixels: an outline `polygon` (or several, for
+ * an object in pieces), or a `mask` covering the whole image (white = the object, any size;
+ * it is resampled), or both, which are combined.
+ */
+export interface SegmentationResult {
+    label?: string;
+    polygon?: ImagePoint[] | ImagePoint[][];
+    mask?: string | PixelBuffer;
+    /** The object's bounding box, when the provider knows it. */
+    box?: CropBox;
+    /** The provider's confidence, 0..1. */
+    score?: number;
+}
+/**
+ * Finds objects in an image by a description and returns their shapes: an AI segmentation
+ * model (SAM, Gemini, a background remover…) behind `ImageEditorController.selectSubject`.
+ */
+export interface SegmentationProvider {
+    /** Shown in the UI and in tool results (e.g. "Gemini"). */
+    readonly name?: string;
+    segment(request: SegmentationRequest): Promise<SegmentationResult[]>;
+}
+/** A segmentation result with its `mask` already decoded to pixels (strings are decoded by the controller). */
+export type DecodedSegmentationResult = Omit<SegmentationResult, 'mask'> & {
+    mask?: PixelBuffer;
+};
+/**
+ * One segmentation result as a selection over a `width × height` image, or null when it has no
+ * usable shape. Polygons are anti-aliased; several polygons and a mask are added together.
+ */
+export declare function segmentToSelection(width: number, height: number, result: DecodedSegmentationResult): SelectionMask | null;
+/** Every usable result added into one selection (null when none has a shape). */
+export declare function segmentsToSelection(width: number, height: number, results: readonly DecodedSegmentationResult[]): SelectionMask | null;

package/dist/image-editor/selection-tools.d.cts

@@ -1,4 +1,4 @@
-import type { AgentTool } from '../tools/registry.cjs';
+import type { AgentTool, AgentToolProvider } from '../tools/registry.cjs';
 import type { ImageEditorController } from './controller.cjs';
 /**
  * Agent tools for selections and masks (part of `createImageEditorToolProvider`):
@@ -6,3 +6,20 @@ import type { ImageEditorController } from './controller.cjs';
  * and `apply_selection_as_mask`. Coordinates are natural pixels of the source image.
  */
 export declare function createImageSelectionTools(controller: ImageEditorController): AgentTool[];
+/** Options for `createImageSegmentationToolProvider`. */
+export interface ImageSegmentationToolProviderOptions {
+    /** Provider id (default `image-segmentation`). */
+    id?: string;
+    /** Tool-name prefix (`<namespace>__<name>`). */
+    namespace?: string;
+    /** Agent playback surface (default the image editor's). */
+    playbackSurface?: string;
+}
+/** The `select_subject` tool: Select › Subject through the controller's segmentation provider. */
+export declare function createImageSubjectSelectionTool(controller: ImageEditorController): AgentTool;
+/**
+ * Agent tools that need the controller's segmentation provider (`select_subject`). The list is
+ * empty while no provider is set, so a host can register this once and the tool appears when a
+ * model is available.
+ */
+export declare function createImageSegmentationToolProvider(controller: ImageEditorController, options?: ImageSegmentationToolProviderOptions): AgentToolProvider;

package/dist/image-editor/selection-tools.d.ts

@@ -1,4 +1,4 @@
-import type { AgentTool } from '../tools/registry.js';
+import type { AgentTool, AgentToolProvider } from '../tools/registry.js';
 import type { ImageEditorController } from './controller.js';
 /**
  * Agent tools for selections and masks (part of `createImageEditorToolProvider`):
@@ -6,3 +6,20 @@ import type { ImageEditorController } from './controller.js';
  * and `apply_selection_as_mask`. Coordinates are natural pixels of the source image.
  */
 export declare function createImageSelectionTools(controller: ImageEditorController): AgentTool[];
+/** Options for `createImageSegmentationToolProvider`. */
+export interface ImageSegmentationToolProviderOptions {
+    /** Provider id (default `image-segmentation`). */
+    id?: string;
+    /** Tool-name prefix (`<namespace>__<name>`). */
+    namespace?: string;
+    /** Agent playback surface (default the image editor's). */
+    playbackSurface?: string;
+}
+/** The `select_subject` tool: Select › Subject through the controller's segmentation provider. */
+export declare function createImageSubjectSelectionTool(controller: ImageEditorController): AgentTool;
+/**
+ * Agent tools that need the controller's segmentation provider (`select_subject`). The list is
+ * empty while no provider is set, so a host can register this once and the tool appears when a
+ * model is available.
+ */
+export declare function createImageSegmentationToolProvider(controller: ImageEditorController, options?: ImageSegmentationToolProviderOptions): AgentToolProvider;

package/dist/image-editor/types.d.cts

@@ -395,6 +395,7 @@ export interface ExportOptions {
     layerIds?: string[];
 }
 import type { EffectProvider } from '../effects/types.cjs';
+import type { SegmentationProvider } from './selection/segmentation.cjs';
 export interface ImageEditorOptions {
     /**
      * What lays out and draws type layers. Default: the browser's fonts through Canvas 2D
@@ -421,6 +422,12 @@ export interface ImageEditorOptions {
      * with `setEffectsProvider`.
      */
     effects?: EffectProvider | null;
+    /**
+     * A segmentation provider (an AI model that finds objects by description) for Select ›
+     * Subject (`selectSubject`, the `select_subject` tool). Optional; change it later with
+     * `setSegmentationProvider`.
+     */
+    segmentation?: SegmentationProvider | null;
 }
 export interface InpaintOptions {
     prompt: string;

package/dist/image-editor/types.d.ts

@@ -395,6 +395,7 @@ export interface ExportOptions {
     layerIds?: string[];
 }
 import type { EffectProvider } from '../effects/types.js';
+import type { SegmentationProvider } from './selection/segmentation.js';
 export interface ImageEditorOptions {
     /**
      * What lays out and draws type layers. Default: the browser's fonts through Canvas 2D
@@ -421,6 +422,12 @@ export interface ImageEditorOptions {
      * with `setEffectsProvider`.
      */
     effects?: EffectProvider | null;
+    /**
+     * A segmentation provider (an AI model that finds objects by description) for Select ›
+     * Subject (`selectSubject`, the `select_subject` tool). Optional; change it later with
+     * `setSegmentationProvider`.
+     */
+    segmentation?: SegmentationProvider | null;
 }
 export interface InpaintOptions {
     prompt: string;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pixerate/editor",
-  "version": "0.179.0",
+  "version": "0.180.0",
   "description": "UI-agnostic rich text and media editing core library",
   "author": "Pixerate",
   "license": "MIT",