@pixerate/editor 0.201.0 → 0.203.0

package/dist/image-editor/index.js

@@ -10071,6 +10071,46 @@ var ImageEditorController = class {
     this.setSelection(sel, "replace");
     return this.state.selection ?? null;
   }
+  /**
+   * Asks the segmentation provider (which must be set) for `request.target` in the flattened image
+   * and returns its results with masks decoded to pixels.
+   */
+  async modelSegments(request) {
+    const provider = this.segmentation;
+    const { width, height } = this.naturalSize;
+    const image = await this.toDataURL({ format: "image/png" });
+    const results = await provider.segment({ ...request, image, width, height });
+    request.signal?.throwIfAborted();
+    const decoded = [];
+    for (const r of results ?? []) decoded.push({ ...r, mask: r.mask !== void 0 ? await this.decodeMaskImage(r.mask) : void 0 });
+    return decoded;
+  }
+  /**
+   * The model's object for Object Selection: of what it returns, the pieces mostly inside the box
+   * (clipped to the box grown by a tenth) or those under the point. `null` when none fits.
+   */
+  async modelObject(target, signal) {
+    const { width, height } = this.naturalSize;
+    const decoded = await this.modelSegments(
+      "rect" in target ? { target: "the main object inside the box", box: target.rect, maxSegments: 4, signal } : { target: "the object at the point", point: target.point, maxSegments: 4, signal }
+    );
+    let union = null;
+    for (const r of decoded) {
+      const one = segmentsToSelection(width, height, [r]);
+      if (!one?.bounds) continue;
+      if ("rect" in target) {
+        const r0 = target.rect;
+        const grow = 0.1 * Math.max(r0.width, r0.height);
+        const box = rectSelection(width, height, { x: r0.x - grow, y: r0.y - grow, width: r0.width + 2 * grow, height: r0.height + 2 * grow }, { antiAlias: false });
+        const inside = combineSelections(one, box, "intersect");
+        if (!inside || selectionArea(inside) < selectionArea(one) * 0.5) continue;
+        union = combineSelections(union, inside, "add");
+      } else if (selectionValueAt(one, target.point.x, target.point.y) >= 0.5) {
+        union = combineSelections(union, one, "add");
+      }
+    }
+    return union;
+  }
   /**
    * Object Selection: the object inside `target.rect` (natural pixels; what is outside the box is
    * background) or under `target.point`, found by GrabCut (colour models of object and background
@@ -10088,10 +10128,17 @@ var ImageEditorController = class {
     } else if (!(target.point.x >= 0 && target.point.y >= 0 && target.point.x < width && target.point.y < height)) {
       throw new Error(`(${target.point.x}, ${target.point.y}) is outside the image (0..${width - 1}, 0..${height - 1}); click on the object.`);
     }
+    if (this.segmentation && options.useModel !== false) {
+      const found = await this.modelObject(target, options.signal);
+      if (found) {
+        this.setSelection(found, options.mode ?? "replace");
+        return { selection: this.state.selection ?? null, method: "model" };
+      }
+    }
     const pixels = await this.requirePixels({ sample: options.sampleAllLayers === false ? "active-layer" : "composite" });
     const region = selectObjectIn(pixels, target, { enhanceEdge: options.enhanceEdge });
     this.setSelection(region, options.mode ?? "replace");
-    return this.state.selection ?? null;
+    return { selection: this.state.selection ?? null, method: "grabcut" };
   }
   /**
    * Select › Focus Area: selects the parts of the image that are in focus (fine detail above the
@@ -10111,11 +10158,18 @@ var ImageEditorController = class {
    * (the selection is then unchanged), why.
    */
   async selectSky(options = {}) {
+    if (this.segmentation && options.useModel !== false) {
+      const { width, height } = this.naturalSize;
+      const sky = segmentsToSelection(width, height, await this.modelSegments({ target: "the sky", signal: options.signal }));
+      if (!sky?.bounds) return { selection: this.state.selection ?? null, method: "model", note: "The segmentation model found no sky." };
+      this.setSelection(sky, options.mode ?? "replace");
+      return { selection: this.state.selection ?? null, method: "model" };
+    }
     const pixels = await this.requirePixels({ sample: options.sampleAllLayers === false ? "active-layer" : "composite" });
     const { selection, reason } = findSky(pixels, options);
-    if (!selection.bounds) return { selection: this.state.selection ?? null, note: reason };
+    if (!selection.bounds) return { selection: this.state.selection ?? null, method: "classical", note: reason };
     this.setSelection(selection, options.mode ?? "replace");
-    return { selection: this.state.selection ?? null };
+    return { selection: this.state.selection ?? null, method: "classical" };
   }
   /**
    * Supplies the pixels the selection tools sample (natural size RGBA), e.g. decoded in a worker
@@ -10306,13 +10360,7 @@ var ImageEditorController = class {
     const description = target.trim();
     if (!description) throw new Error("Describe the subject to select.");
     const { width, height } = this.naturalSize;
-    const image = await this.toDataURL({ format: "image/png" });
-    const results = await provider.segment({ image, width, height, target: description, maxSegments: options.maxSegments, signal: options.signal });
-    options.signal?.throwIfAborted();
-    const decoded = [];
-    for (const r of results ?? []) {
-      decoded.push({ ...r, mask: r.mask !== void 0 ? await this.decodeMaskImage(r.mask) : void 0 });
-    }
+    const decoded = await this.modelSegments({ target: description, maxSegments: options.maxSegments, signal: options.signal });
     const segments = decoded.map((r) => {
       const one = segmentsToSelection(width, height, [r]);
       return { label: r.label || description, bounds: one?.bounds ?? null, ...r.score !== void 0 ? { score: r.score } : {} };
@@ -13624,7 +13672,7 @@ function createImageSelectionTools(controller) {
   const selectObjectTool = {
     name: "select_object",
     title: "Object Selection",
-    description: "Object Selection: selects the object inside rect (a box drawn loosely around it, natural image pixels; what lies outside the box is treated as background) or the object under point, found from the colours of object and background and the edges between them (GrabCut). Combined with the selection by mode. A box is more reliable than a point: make it a little larger than the object on every side. Check with get_selection and capture_view, then fix with quick_select strokes or select_object with mode add / subtract. One undoable step. Returns the selection.",
+    description: 'Object Selection: selects the object inside rect (a box drawn loosely around it, natural image pixels; what lies outside the box is treated as background) or the object under point. With a host segmentation model it asks the model (falling back if nothing it returns fits); otherwise it finds the object from the colours of object and background and the edges between them (GrabCut). Combined with the selection by mode. A box is more reliable than a point: make it a little larger than the object on every side. Check with get_selection and capture_view, then fix with quick_select strokes or select_object with mode add / subtract. One undoable step. Returns the selection and method ("model" or "grabcut").',
     inputSchema: {
       type: "object",
       properties: {
@@ -13638,20 +13686,25 @@ function createImageSelectionTools(controller) {
         point: { ...pointSchema, description: "A point on the object (natural image pixels). Pass this or rect." },
         mode: modeSchema,
         sampleAllLayers: { type: "boolean", description: "Read every layer (default true); false reads only the active pixel layer." },
-        enhanceEdge: { type: "boolean", description: "Smoother, less blocky edges (Auto-Enhance; default false)." }
+        enhanceEdge: { type: "boolean", description: "Smoother, less blocky edges (Auto-Enhance; default false)." },
+        useModel: { type: "boolean", description: "Use the host segmentation model when there is one (default true); false always uses GrabCut." }
       },
       additionalProperties: false
     },
-    annotations: writeAnnotations("Object Selection"),
+    annotations: { ...writeAnnotations("Object Selection"), openWorldHint: true },
     handler: async (args) => {
       const rect = args.rect;
       const point = args.point;
       if (!rect && !point) throw new Error("Pass rect (a box around the object) or point (a point on it).");
       if (rect && point) throw new Error("Pass rect or point, not both.");
-      const options = { mode: parseSelectionMode(args.mode), sampleAllLayers: args.sampleAllLayers, enhanceEdge: args.enhanceEdge };
-      if (rect) await controller.selectObject({ rect: { x: num2(rect.x, NaN), y: num2(rect.y, NaN), width: num2(rect.width, NaN), height: num2(rect.height, NaN) } }, options);
-      else await controller.selectObject({ point: { x: num2(point.x, NaN), y: num2(point.y, NaN) } }, options);
-      return summary(controller);
+      const options = {
+        mode: parseSelectionMode(args.mode),
+        sampleAllLayers: args.sampleAllLayers,
+        enhanceEdge: args.enhanceEdge,
+        useModel: args.useModel
+      };
+      const { method } = rect ? await controller.selectObject({ rect: { x: num2(rect.x, NaN), y: num2(rect.y, NaN), width: num2(rect.width, NaN), height: num2(rect.height, NaN) } }, options) : await controller.selectObject({ point: { x: num2(point.x, NaN), y: num2(point.y, NaN) } }, options);
+      return { ...summary(controller), method };
     }
   };
   const selectFocusAreaTool = {
@@ -13684,26 +13737,28 @@ function createImageSelectionTools(controller) {
   const selectSkyTool = {
     name: "select_sky",
     title: "Select the sky",
-    description: "Select \u203A Sky: selects the sky (blue, or bright and grey when overcast, smooth, reaching down from the top of the image, including patches seen through branches or gaps), with the edge fitted to the skyline, combined with the selection by mode. Use it to replace or recolour a sky or to adjust the land without it (invert_selection). Raise threshold to take in haze or a sunset glow, lower it if bright buildings come in. One undoable step. Returns the selection, and a note when no sky was found (the selection is then unchanged).",
+    description: 'Select \u203A Sky: selects the sky, combined with the selection by mode. With a host segmentation model it asks the model; otherwise it finds it classically (blue, or bright and grey when overcast, smooth, reaching down from the top of the image, including patches seen through branches or gaps) with the edge fitted to the skyline. Use it to replace or recolour a sky or to adjust the land without it (invert_selection). Raise threshold to take in haze or a sunset glow, lower it if bright buildings come in. One undoable step. Returns the selection, method ("model" or "classical"; threshold and softness apply to classical), and a note when no sky was found (the selection is then unchanged).',
     inputSchema: {
       type: "object",
       properties: {
         threshold: { type: "number", minimum: 0, maximum: 100, description: "How sky-like a pixel must be, 0..100 (default 50): higher takes in less sky-like pixels." },
         softness: { type: "number", minimum: 0, maximum: 100, description: "Soften the edge over this many natural pixels (default 4; 0 for a hard edge)." },
         mode: modeSchema,
-        sampleAllLayers: { type: "boolean", description: "Read every layer (default true); false reads only the active pixel layer." }
+        sampleAllLayers: { type: "boolean", description: "Read every layer (default true); false reads only the active pixel layer." },
+        useModel: { type: "boolean", description: "Use the host segmentation model when there is one (default true); false finds the sky classically." }
       },
       additionalProperties: false
     },
-    annotations: writeAnnotations("Select the sky", true),
+    annotations: { ...writeAnnotations("Select the sky", true), openWorldHint: true },
     handler: async (args) => {
-      const { note } = await controller.selectSky({
+      const { note, method } = await controller.selectSky({
         threshold: args.threshold === void 0 ? void 0 : num2(args.threshold, 50),
         softness: args.softness === void 0 ? void 0 : num2(args.softness, 4),
         mode: parseSelectionMode(args.mode),
-        sampleAllLayers: args.sampleAllLayers
+        sampleAllLayers: args.sampleAllLayers,
+        useModel: args.useModel
       });
-      return { ...summary(controller), ...note ? { note } : {} };
+      return { ...summary(controller), method, ...note ? { note } : {} };
     }
   };
   const magicWandTool = {

package/dist/image-editor/selection/segmentation.d.cts

@@ -11,6 +11,13 @@ export interface SegmentationRequest {
     target: string;
     /** Return at most this many objects (the provider's default when absent). */
     maxSegments?: number;
+    /**
+     * Object Selection: the box (natural pixels) the wanted object is in. A box-prompted model (SAM)
+     * should segment the object inside it; others can read `target`, which describes it too.
+     */
+    box?: CropBox;
+    /** Object Selection: a point (natural pixels) on the wanted object, for a point-prompted model. */
+    point?: ImagePoint;
     signal?: AbortSignal;
 }
 /**

package/dist/image-editor/selection/segmentation.d.ts

@@ -11,6 +11,13 @@ export interface SegmentationRequest {
     target: string;
     /** Return at most this many objects (the provider's default when absent). */
     maxSegments?: number;
+    /**
+     * Object Selection: the box (natural pixels) the wanted object is in. A box-prompted model (SAM)
+     * should segment the object inside it; others can read `target`, which describes it too.
+     */
+    box?: CropBox;
+    /** Object Selection: a point (natural pixels) on the wanted object, for a point-prompted model. */
+    point?: ImagePoint;
     signal?: AbortSignal;
 }
 /**

package/dist/paint-editor/guide-tools.d.cts

@@ -3,6 +3,6 @@
  * images) and colour (harmonies, swatches, the mixing pad). Guides, swatches and the pad are
  * editor state saved with the document (the pad is not saved), not undo steps.
  */
-import type { AgentTool } from '../tools/registry.cjs';
+import type { PaintAgentTool } from './hosted.cjs';
 import type { PaintEditorController } from './controller.cjs';
-export declare function createPaintGuideTools(c: PaintEditorController): AgentTool[];
+export declare function createPaintGuideTools(c: PaintEditorController): PaintAgentTool[];

package/dist/paint-editor/guide-tools.d.ts

@@ -3,6 +3,6 @@
  * images) and colour (harmonies, swatches, the mixing pad). Guides, swatches and the pad are
  * editor state saved with the document (the pad is not saved), not undo steps.
  */
-import type { AgentTool } from '../tools/registry.js';
+import type { PaintAgentTool } from './hosted.js';
 import type { PaintEditorController } from './controller.js';
-export declare function createPaintGuideTools(c: PaintEditorController): AgentTool[];
+export declare function createPaintGuideTools(c: PaintEditorController): PaintAgentTool[];

package/dist/paint-editor/hosted.d.cts

@@ -0,0 +1,34 @@
+import type { AgentTool } from '../tools/registry.cjs';
+import type { PaintEditorController } from './controller.cjs';
+/**
+ * Position fields of a paint tool, as paths: keys joined by `.`, a key ending in `[]` maps over
+ * that array, `''` is the value itself. Each path names an object whose numeric `x` / `y` are a
+ * position (a point, or the top-left of a rect); missing or `null` values are skipped.
+ */
+export interface PaintToolCoordinates {
+    /** Argument fields, given in host coordinates and moved into document pixels. */
+    in?: readonly string[];
+    /** Result fields, in document pixels and moved out into host coordinates. */
+    out?: readonly string[];
+}
+/** A paint tool with its position fields. */
+export type PaintAgentTool = AgentTool & {
+    coordinates?: PaintToolCoordinates;
+};
+export interface PaintOrigin {
+    x: number;
+    y: number;
+}
+/** Host coordinates → document pixels, at the given position fields. */
+export declare const toDocument: <T>(value: T, paths: readonly string[], origin: PaintOrigin) => T;
+/** Document pixels → host coordinates, at the given position fields. */
+export declare const fromDocument: <T>(value: T, paths: readonly string[], origin: PaintOrigin) => T;
+/** Tool prose in host terms: positions are canvas coordinates, not document pixels. */
+export declare function hostedText(text: string): string;
+/** `hostedText` over every `description` in a JSON schema. */
+export declare function hostedSchema<T>(schema: T): T;
+/**
+ * A controller that resolves `get` on every property access, so tools built over it never touch
+ * the document until a handler runs (a host can create the document lazily).
+ */
+export declare function lazyController(get: () => PaintEditorController): PaintEditorController;

package/dist/paint-editor/hosted.d.ts

@@ -0,0 +1,34 @@
+import type { AgentTool } from '../tools/registry.js';
+import type { PaintEditorController } from './controller.js';
+/**
+ * Position fields of a paint tool, as paths: keys joined by `.`, a key ending in `[]` maps over
+ * that array, `''` is the value itself. Each path names an object whose numeric `x` / `y` are a
+ * position (a point, or the top-left of a rect); missing or `null` values are skipped.
+ */
+export interface PaintToolCoordinates {
+    /** Argument fields, given in host coordinates and moved into document pixels. */
+    in?: readonly string[];
+    /** Result fields, in document pixels and moved out into host coordinates. */
+    out?: readonly string[];
+}
+/** A paint tool with its position fields. */
+export type PaintAgentTool = AgentTool & {
+    coordinates?: PaintToolCoordinates;
+};
+export interface PaintOrigin {
+    x: number;
+    y: number;
+}
+/** Host coordinates → document pixels, at the given position fields. */
+export declare const toDocument: <T>(value: T, paths: readonly string[], origin: PaintOrigin) => T;
+/** Document pixels → host coordinates, at the given position fields. */
+export declare const fromDocument: <T>(value: T, paths: readonly string[], origin: PaintOrigin) => T;
+/** Tool prose in host terms: positions are canvas coordinates, not document pixels. */
+export declare function hostedText(text: string): string;
+/** `hostedText` over every `description` in a JSON schema. */
+export declare function hostedSchema<T>(schema: T): T;
+/**
+ * A controller that resolves `get` on every property access, so tools built over it never touch
+ * the document until a handler runs (a host can create the document lazily).
+ */
+export declare function lazyController(get: () => PaintEditorController): PaintEditorController;