@pixerate/editor 0.202.0 → 0.203.0

package/dist/image-editor/index.js

@@ -10071,6 +10071,46 @@ var ImageEditorController = class {
     this.setSelection(sel, "replace");
     return this.state.selection ?? null;
   }
+  /**
+   * Asks the segmentation provider (which must be set) for `request.target` in the flattened image
+   * and returns its results with masks decoded to pixels.
+   */
+  async modelSegments(request) {
+    const provider = this.segmentation;
+    const { width, height } = this.naturalSize;
+    const image = await this.toDataURL({ format: "image/png" });
+    const results = await provider.segment({ ...request, image, width, height });
+    request.signal?.throwIfAborted();
+    const decoded = [];
+    for (const r of results ?? []) decoded.push({ ...r, mask: r.mask !== void 0 ? await this.decodeMaskImage(r.mask) : void 0 });
+    return decoded;
+  }
+  /**
+   * The model's object for Object Selection: of what it returns, the pieces mostly inside the box
+   * (clipped to the box grown by a tenth) or those under the point. `null` when none fits.
+   */
+  async modelObject(target, signal) {
+    const { width, height } = this.naturalSize;
+    const decoded = await this.modelSegments(
+      "rect" in target ? { target: "the main object inside the box", box: target.rect, maxSegments: 4, signal } : { target: "the object at the point", point: target.point, maxSegments: 4, signal }
+    );
+    let union = null;
+    for (const r of decoded) {
+      const one = segmentsToSelection(width, height, [r]);
+      if (!one?.bounds) continue;
+      if ("rect" in target) {
+        const r0 = target.rect;
+        const grow = 0.1 * Math.max(r0.width, r0.height);
+        const box = rectSelection(width, height, { x: r0.x - grow, y: r0.y - grow, width: r0.width + 2 * grow, height: r0.height + 2 * grow }, { antiAlias: false });
+        const inside = combineSelections(one, box, "intersect");
+        if (!inside || selectionArea(inside) < selectionArea(one) * 0.5) continue;
+        union = combineSelections(union, inside, "add");
+      } else if (selectionValueAt(one, target.point.x, target.point.y) >= 0.5) {
+        union = combineSelections(union, one, "add");
+      }
+    }
+    return union;
+  }
   /**
    * Object Selection: the object inside `target.rect` (natural pixels; what is outside the box is
    * background) or under `target.point`, found by GrabCut (colour models of object and background
@@ -10088,10 +10128,17 @@ var ImageEditorController = class {
     } else if (!(target.point.x >= 0 && target.point.y >= 0 && target.point.x < width && target.point.y < height)) {
       throw new Error(`(${target.point.x}, ${target.point.y}) is outside the image (0..${width - 1}, 0..${height - 1}); click on the object.`);
     }
+    if (this.segmentation && options.useModel !== false) {
+      const found = await this.modelObject(target, options.signal);
+      if (found) {
+        this.setSelection(found, options.mode ?? "replace");
+        return { selection: this.state.selection ?? null, method: "model" };
+      }
+    }
     const pixels = await this.requirePixels({ sample: options.sampleAllLayers === false ? "active-layer" : "composite" });
     const region = selectObjectIn(pixels, target, { enhanceEdge: options.enhanceEdge });
     this.setSelection(region, options.mode ?? "replace");
-    return this.state.selection ?? null;
+    return { selection: this.state.selection ?? null, method: "grabcut" };
   }
   /**
    * Select › Focus Area: selects the parts of the image that are in focus (fine detail above the
@@ -10111,11 +10158,18 @@ var ImageEditorController = class {
    * (the selection is then unchanged), why.
    */
   async selectSky(options = {}) {
+    if (this.segmentation && options.useModel !== false) {
+      const { width, height } = this.naturalSize;
+      const sky = segmentsToSelection(width, height, await this.modelSegments({ target: "the sky", signal: options.signal }));
+      if (!sky?.bounds) return { selection: this.state.selection ?? null, method: "model", note: "The segmentation model found no sky." };
+      this.setSelection(sky, options.mode ?? "replace");
+      return { selection: this.state.selection ?? null, method: "model" };
+    }
     const pixels = await this.requirePixels({ sample: options.sampleAllLayers === false ? "active-layer" : "composite" });
     const { selection, reason } = findSky(pixels, options);
-    if (!selection.bounds) return { selection: this.state.selection ?? null, note: reason };
+    if (!selection.bounds) return { selection: this.state.selection ?? null, method: "classical", note: reason };
     this.setSelection(selection, options.mode ?? "replace");
-    return { selection: this.state.selection ?? null };
+    return { selection: this.state.selection ?? null, method: "classical" };
   }
   /**
    * Supplies the pixels the selection tools sample (natural size RGBA), e.g. decoded in a worker
@@ -10306,13 +10360,7 @@ var ImageEditorController = class {
     const description = target.trim();
     if (!description) throw new Error("Describe the subject to select.");
     const { width, height } = this.naturalSize;
-    const image = await this.toDataURL({ format: "image/png" });
-    const results = await provider.segment({ image, width, height, target: description, maxSegments: options.maxSegments, signal: options.signal });
-    options.signal?.throwIfAborted();
-    const decoded = [];
-    for (const r of results ?? []) {
-      decoded.push({ ...r, mask: r.mask !== void 0 ? await this.decodeMaskImage(r.mask) : void 0 });
-    }
+    const decoded = await this.modelSegments({ target: description, maxSegments: options.maxSegments, signal: options.signal });
     const segments = decoded.map((r) => {
       const one = segmentsToSelection(width, height, [r]);
       return { label: r.label || description, bounds: one?.bounds ?? null, ...r.score !== void 0 ? { score: r.score } : {} };
@@ -13624,7 +13672,7 @@ function createImageSelectionTools(controller) {
   const selectObjectTool = {
     name: "select_object",
     title: "Object Selection",
-    description: "Object Selection: selects the object inside rect (a box drawn loosely around it, natural image pixels; what lies outside the box is treated as background) or the object under point, found from the colours of object and background and the edges between them (GrabCut). Combined with the selection by mode. A box is more reliable than a point: make it a little larger than the object on every side. Check with get_selection and capture_view, then fix with quick_select strokes or select_object with mode add / subtract. One undoable step. Returns the selection.",
+    description: 'Object Selection: selects the object inside rect (a box drawn loosely around it, natural image pixels; what lies outside the box is treated as background) or the object under point. With a host segmentation model it asks the model (falling back if nothing it returns fits); otherwise it finds the object from the colours of object and background and the edges between them (GrabCut). Combined with the selection by mode. A box is more reliable than a point: make it a little larger than the object on every side. Check with get_selection and capture_view, then fix with quick_select strokes or select_object with mode add / subtract. One undoable step. Returns the selection and method ("model" or "grabcut").',
     inputSchema: {
       type: "object",
       properties: {
@@ -13638,20 +13686,25 @@ function createImageSelectionTools(controller) {
         point: { ...pointSchema, description: "A point on the object (natural image pixels). Pass this or rect." },
         mode: modeSchema,
         sampleAllLayers: { type: "boolean", description: "Read every layer (default true); false reads only the active pixel layer." },
-        enhanceEdge: { type: "boolean", description: "Smoother, less blocky edges (Auto-Enhance; default false)." }
+        enhanceEdge: { type: "boolean", description: "Smoother, less blocky edges (Auto-Enhance; default false)." },
+        useModel: { type: "boolean", description: "Use the host segmentation model when there is one (default true); false always uses GrabCut." }
       },
       additionalProperties: false
     },
-    annotations: writeAnnotations("Object Selection"),
+    annotations: { ...writeAnnotations("Object Selection"), openWorldHint: true },
     handler: async (args) => {
       const rect = args.rect;
       const point = args.point;
       if (!rect && !point) throw new Error("Pass rect (a box around the object) or point (a point on it).");
       if (rect && point) throw new Error("Pass rect or point, not both.");
-      const options = { mode: parseSelectionMode(args.mode), sampleAllLayers: args.sampleAllLayers, enhanceEdge: args.enhanceEdge };
-      if (rect) await controller.selectObject({ rect: { x: num2(rect.x, NaN), y: num2(rect.y, NaN), width: num2(rect.width, NaN), height: num2(rect.height, NaN) } }, options);
-      else await controller.selectObject({ point: { x: num2(point.x, NaN), y: num2(point.y, NaN) } }, options);
-      return summary(controller);
+      const options = {
+        mode: parseSelectionMode(args.mode),
+        sampleAllLayers: args.sampleAllLayers,
+        enhanceEdge: args.enhanceEdge,
+        useModel: args.useModel
+      };
+      const { method } = rect ? await controller.selectObject({ rect: { x: num2(rect.x, NaN), y: num2(rect.y, NaN), width: num2(rect.width, NaN), height: num2(rect.height, NaN) } }, options) : await controller.selectObject({ point: { x: num2(point.x, NaN), y: num2(point.y, NaN) } }, options);
+      return { ...summary(controller), method };
     }
   };
   const selectFocusAreaTool = {
@@ -13684,26 +13737,28 @@ function createImageSelectionTools(controller) {
   const selectSkyTool = {
     name: "select_sky",
     title: "Select the sky",
-    description: "Select \u203A Sky: selects the sky (blue, or bright and grey when overcast, smooth, reaching down from the top of the image, including patches seen through branches or gaps), with the edge fitted to the skyline, combined with the selection by mode. Use it to replace or recolour a sky or to adjust the land without it (invert_selection). Raise threshold to take in haze or a sunset glow, lower it if bright buildings come in. One undoable step. Returns the selection, and a note when no sky was found (the selection is then unchanged).",
+    description: 'Select \u203A Sky: selects the sky, combined with the selection by mode. With a host segmentation model it asks the model; otherwise it finds it classically (blue, or bright and grey when overcast, smooth, reaching down from the top of the image, including patches seen through branches or gaps) with the edge fitted to the skyline. Use it to replace or recolour a sky or to adjust the land without it (invert_selection). Raise threshold to take in haze or a sunset glow, lower it if bright buildings come in. One undoable step. Returns the selection, method ("model" or "classical"; threshold and softness apply to classical), and a note when no sky was found (the selection is then unchanged).',
     inputSchema: {
       type: "object",
       properties: {
         threshold: { type: "number", minimum: 0, maximum: 100, description: "How sky-like a pixel must be, 0..100 (default 50): higher takes in less sky-like pixels." },
         softness: { type: "number", minimum: 0, maximum: 100, description: "Soften the edge over this many natural pixels (default 4; 0 for a hard edge)." },
         mode: modeSchema,
-        sampleAllLayers: { type: "boolean", description: "Read every layer (default true); false reads only the active pixel layer." }
+        sampleAllLayers: { type: "boolean", description: "Read every layer (default true); false reads only the active pixel layer." },
+        useModel: { type: "boolean", description: "Use the host segmentation model when there is one (default true); false finds the sky classically." }
       },
       additionalProperties: false
     },
-    annotations: writeAnnotations("Select the sky", true),
+    annotations: { ...writeAnnotations("Select the sky", true), openWorldHint: true },
     handler: async (args) => {
-      const { note } = await controller.selectSky({
+      const { note, method } = await controller.selectSky({
         threshold: args.threshold === void 0 ? void 0 : num2(args.threshold, 50),
         softness: args.softness === void 0 ? void 0 : num2(args.softness, 4),
         mode: parseSelectionMode(args.mode),
-        sampleAllLayers: args.sampleAllLayers
+        sampleAllLayers: args.sampleAllLayers,
+        useModel: args.useModel
       });
-      return { ...summary(controller), ...note ? { note } : {} };
+      return { ...summary(controller), method, ...note ? { note } : {} };
     }
   };
   const magicWandTool = {

package/dist/image-editor/selection/segmentation.d.cts

@@ -11,6 +11,13 @@ export interface SegmentationRequest {
     target: string;
     /** Return at most this many objects (the provider's default when absent). */
     maxSegments?: number;
+    /**
+     * Object Selection: the box (natural pixels) the wanted object is in. A box-prompted model (SAM)
+     * should segment the object inside it; others can read `target`, which describes it too.
+     */
+    box?: CropBox;
+    /** Object Selection: a point (natural pixels) on the wanted object, for a point-prompted model. */
+    point?: ImagePoint;
     signal?: AbortSignal;
 }
 /**

package/dist/image-editor/selection/segmentation.d.ts

@@ -11,6 +11,13 @@ export interface SegmentationRequest {
     target: string;
     /** Return at most this many objects (the provider's default when absent). */
     maxSegments?: number;
+    /**
+     * Object Selection: the box (natural pixels) the wanted object is in. A box-prompted model (SAM)
+     * should segment the object inside it; others can read `target`, which describes it too.
+     */
+    box?: CropBox;
+    /** Object Selection: a point (natural pixels) on the wanted object, for a point-prompted model. */
+    point?: ImagePoint;
     signal?: AbortSignal;
 }
 /**

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pixerate/editor",
-  "version": "0.202.0",
+  "version": "0.203.0",
   "description": "UI-agnostic rich text and media editing core library",
   "author": "Pixerate",
   "license": "MIT",

package/skills/pixerate-image-editor/SKILL.md

@@ -253,9 +253,9 @@ in the source frame at the document's bit depth, colour mode and profile. Read
 | `select_ellipse` | Elliptical marquee inscribed in `x`, `y`, `width`, `height`; same options. A feathered ellipse makes a vignette. Returns the summary. |
 | `select_line` | Single row or column marquee: `orientation` `row` (at y) or `column` (at x), `position` in natural px, `mode`. A 1 px line across the image. Returns the summary. |
 | `quick_select` | Quick Selection: `strokes` (brush paths of `{x, y}`), `size` (diameter, default 30), `mode`, `sampleAllLayers`, `enhanceEdge`. Each stroke grows out to its region's edges; with `replace` the first replaces and the rest add. Paint inside the area, then check and fix with `add` / `subtract` strokes. Returns the summary. |
-| `select_object` | Object Selection (GrabCut): `rect` (a box around the object) or `point` (on it), `mode`, `sampleAllLayers`, `enhanceEdge`. Outside the box is background. Make the box a little larger than the object. Returns the summary. |
+| `select_object` | Object Selection: `rect` (a box around the object) or `point` (on it), `mode`, `sampleAllLayers`, `enhanceEdge`, `useModel`. Uses the host model when set (else, or when nothing fits, GrabCut; outside the box is background). Make the box a little larger than the object. Returns the summary and `method`. |
 | `select_focus_area` | Select › Focus Area: the in-focus (sharp) parts, edge fitted to the image. `range` 0..100 (higher takes in softer areas), `noise` 0..100 (raise when grain is selected), `softness` px, `mode`. Returns the summary and `noiseLevel`. |
-| `select_sky` | Select › Sky: the sky (blue or overcast, from the top down, with patches through gaps), edge fitted to the skyline. `threshold` 0..100 (higher takes in haze), `softness` px, `mode`. Returns the summary, and `note` when none was found. |
+| `select_sky` | Select › Sky: the sky, from the host model when set, else classically (blue or overcast, from the top down, with patches through gaps). `threshold` 0..100 (higher takes in haze) and `softness` px for classical, `mode`, `useModel`. Returns the summary, `method`, and `note` when none was found. |
 | `select_polygon` | The lasso: `points` (at least 3 `{x, y}` along the outline, closed automatically), `mode`, `feather`, `antiAlias`; `snapToEdges: {width, contrast}` makes it the Magnetic Lasso (points snap to edges, the outline follows them). Trace a subject's contour, not its box. Returns the summary (plus `outlinePoints` when snapped). |
 | `magic_wand` | Select similar colour from `x`, `y`: `tolerance` 0..255 (default 32), `contiguous` (default true), `distance` `rgb` or `perceptual`, `sampleMerged`, `sample` (`composite` or `active-layer`), `mode`, `feather`. For flat backdrops and solid colours. Returns the summary. |
 | `select_color_range` | Color Range, soft: `samples` (`[{color, x?, y?}]`) or a `preset` (reds … magentas, highlights, midtones, shadows, skinTones); `fuzziness` (samples 0..200, default 40; tone presets' falloff, default 20), `localized` px radius around positioned samples, `invert`, `sample`, `mode`, `feather`. Returns the summary. |