@pixerate/editor 0.178.0 → 0.180.0

package/dist/image-editor/renderer.d.cts

@@ -68,6 +68,8 @@ export interface RenderOptions {
         outline: ImagePoint[];
         handles: ImagePoint[];
         reference: ImagePoint;
+        /** The rotate handle, on a stem from the top edge's midpoint (`outline[0]` to `outline[1]`). */
+        rotate?: ImagePoint;
         /** Warp mode: the mesh's grid lines as cubics, its anchors, and handles with their anchors. */
         warp?: {
             curves: ImagePoint[][];
@@ -174,7 +176,10 @@ export declare class ImageEditorRenderer {
     private outlineCache;
     /** Rubylith over the unselected area, drawn in natural image space. */
     private drawQuickMask;
-    /** Marching ants: a white line under a black dashed one, one screen pixel wide. */
+    /**
+     * Marching ants: a white line under a black dashed one, one screen pixel wide. `scale` is
+     * screen pixels per natural pixel (the projection scale times the host's CSS scale).
+     */
     private drawAnts;
     private drawDraftPath;
     private getSourceId;

package/dist/image-editor/renderer.d.ts

@@ -68,6 +68,8 @@ export interface RenderOptions {
         outline: ImagePoint[];
         handles: ImagePoint[];
         reference: ImagePoint;
+        /** The rotate handle, on a stem from the top edge's midpoint (`outline[0]` to `outline[1]`). */
+        rotate?: ImagePoint;
         /** Warp mode: the mesh's grid lines as cubics, its anchors, and handles with their anchors. */
         warp?: {
             curves: ImagePoint[][];
@@ -174,7 +176,10 @@ export declare class ImageEditorRenderer {
     private outlineCache;
     /** Rubylith over the unselected area, drawn in natural image space. */
     private drawQuickMask;
-    /** Marching ants: a white line under a black dashed one, one screen pixel wide. */
+    /**
+     * Marching ants: a white line under a black dashed one, one screen pixel wide. `scale` is
+     * screen pixels per natural pixel (the projection scale times the host's CSS scale).
+     */
     private drawAnts;
     private drawDraftPath;
     private getSourceId;

package/dist/image-editor/selection/index.d.cts

@@ -7,3 +7,4 @@ export * from './layer-mask.cjs';
 export * from './overlay.cjs';
 export * from './tool-options.cjs';
 export * from './gesture.cjs';
+export * from './segmentation.cjs';

package/dist/image-editor/selection/index.d.ts

@@ -7,3 +7,4 @@ export * from './layer-mask.js';
 export * from './overlay.js';
 export * from './tool-options.js';
 export * from './gesture.js';
+export * from './segmentation.js';

package/dist/image-editor/selection/overlay.d.cts

@@ -16,3 +16,11 @@ export interface QuickMaskStyle {
  * selection shows through clear, as Photoshop's rubylith overlay.
  */
 export declare function quickMaskPixels(mask: SelectionMask, style?: QuickMaskStyle): Uint8ClampedArray;
+/** How often marching ants step (ms); each step moves the dashes one screen pixel. */
+export declare const MARCHING_ANTS_STEP_MS = 100;
+/**
+ * Marches the ants: calls `tick` with the next `RenderOptions.antsOffset` every `stepMs` until the
+ * returned stop function is called. Hosts run it while a selection is shown and render on each
+ * tick. With reduced motion requested the ants stand still and `tick` is never called.
+ */
+export declare function startMarchingAnts(tick: (offset: number) => void, stepMs?: number): () => void;

package/dist/image-editor/selection/overlay.d.ts

@@ -16,3 +16,11 @@ export interface QuickMaskStyle {
  * selection shows through clear, as Photoshop's rubylith overlay.
  */
 export declare function quickMaskPixels(mask: SelectionMask, style?: QuickMaskStyle): Uint8ClampedArray;
+/** How often marching ants step (ms); each step moves the dashes one screen pixel. */
+export declare const MARCHING_ANTS_STEP_MS = 100;
+/**
+ * Marches the ants: calls `tick` with the next `RenderOptions.antsOffset` every `stepMs` until the
+ * returned stop function is called. Hosts run it while a selection is shown and render on each
+ * tick. With reduced motion requested the ants stand still and `tick` is never called.
+ */
+export declare function startMarchingAnts(tick: (offset: number) => void, stepMs?: number): () => void;

package/dist/image-editor/selection/segmentation.d.cts

@@ -0,0 +1,49 @@
+import type { CropBox, ImagePoint } from '../types.cjs';
+import { type PixelBuffer, type SelectionMask } from './mask.cjs';
+/** What a segmentation provider is asked for (see `SegmentationProvider`). */
+export interface SegmentationRequest {
+    /** The flattened image at its natural size, as a PNG data URL. */
+    image: string;
+    /** The image's natural size in pixels. */
+    width: number;
+    height: number;
+    /** What to find, in words ("the dog", "the sky", "every separate object"). */
+    target: string;
+    /** Return at most this many objects (the provider's default when absent). */
+    maxSegments?: number;
+    signal?: AbortSignal;
+}
+/**
+ * One object a provider found, in natural image pixels: an outline `polygon` (or several, for
+ * an object in pieces), or a `mask` covering the whole image (white = the object, any size;
+ * it is resampled), or both, which are combined.
+ */
+export interface SegmentationResult {
+    label?: string;
+    polygon?: ImagePoint[] | ImagePoint[][];
+    mask?: string | PixelBuffer;
+    /** The object's bounding box, when the provider knows it. */
+    box?: CropBox;
+    /** The provider's confidence, 0..1. */
+    score?: number;
+}
+/**
+ * Finds objects in an image by a description and returns their shapes: an AI segmentation
+ * model (SAM, Gemini, a background remover…) behind `ImageEditorController.selectSubject`.
+ */
+export interface SegmentationProvider {
+    /** Shown in the UI and in tool results (e.g. "Gemini"). */
+    readonly name?: string;
+    segment(request: SegmentationRequest): Promise<SegmentationResult[]>;
+}
+/** A segmentation result with its `mask` already decoded to pixels (strings are decoded by the controller). */
+export type DecodedSegmentationResult = Omit<SegmentationResult, 'mask'> & {
+    mask?: PixelBuffer;
+};
+/**
+ * One segmentation result as a selection over a `width × height` image, or null when it has no
+ * usable shape. Polygons are anti-aliased; several polygons and a mask are added together.
+ */
+export declare function segmentToSelection(width: number, height: number, result: DecodedSegmentationResult): SelectionMask | null;
+/** Every usable result added into one selection (null when none has a shape). */
+export declare function segmentsToSelection(width: number, height: number, results: readonly DecodedSegmentationResult[]): SelectionMask | null;

package/dist/image-editor/selection/segmentation.d.ts

@@ -0,0 +1,49 @@
+import type { CropBox, ImagePoint } from '../types.js';
+import { type PixelBuffer, type SelectionMask } from './mask.js';
+/** What a segmentation provider is asked for (see `SegmentationProvider`). */
+export interface SegmentationRequest {
+    /** The flattened image at its natural size, as a PNG data URL. */
+    image: string;
+    /** The image's natural size in pixels. */
+    width: number;
+    height: number;
+    /** What to find, in words ("the dog", "the sky", "every separate object"). */
+    target: string;
+    /** Return at most this many objects (the provider's default when absent). */
+    maxSegments?: number;
+    signal?: AbortSignal;
+}
+/**
+ * One object a provider found, in natural image pixels: an outline `polygon` (or several, for
+ * an object in pieces), or a `mask` covering the whole image (white = the object, any size;
+ * it is resampled), or both, which are combined.
+ */
+export interface SegmentationResult {
+    label?: string;
+    polygon?: ImagePoint[] | ImagePoint[][];
+    mask?: string | PixelBuffer;
+    /** The object's bounding box, when the provider knows it. */
+    box?: CropBox;
+    /** The provider's confidence, 0..1. */
+    score?: number;
+}
+/**
+ * Finds objects in an image by a description and returns their shapes: an AI segmentation
+ * model (SAM, Gemini, a background remover…) behind `ImageEditorController.selectSubject`.
+ */
+export interface SegmentationProvider {
+    /** Shown in the UI and in tool results (e.g. "Gemini"). */
+    readonly name?: string;
+    segment(request: SegmentationRequest): Promise<SegmentationResult[]>;
+}
+/** A segmentation result with its `mask` already decoded to pixels (strings are decoded by the controller). */
+export type DecodedSegmentationResult = Omit<SegmentationResult, 'mask'> & {
+    mask?: PixelBuffer;
+};
+/**
+ * One segmentation result as a selection over a `width × height` image, or null when it has no
+ * usable shape. Polygons are anti-aliased; several polygons and a mask are added together.
+ */
+export declare function segmentToSelection(width: number, height: number, result: DecodedSegmentationResult): SelectionMask | null;
+/** Every usable result added into one selection (null when none has a shape). */
+export declare function segmentsToSelection(width: number, height: number, results: readonly DecodedSegmentationResult[]): SelectionMask | null;

package/dist/image-editor/selection-tools.d.cts

@@ -1,4 +1,4 @@
-import type { AgentTool } from '../tools/registry.cjs';
+import type { AgentTool, AgentToolProvider } from '../tools/registry.cjs';
 import type { ImageEditorController } from './controller.cjs';
 /**
  * Agent tools for selections and masks (part of `createImageEditorToolProvider`):
@@ -6,3 +6,20 @@ import type { ImageEditorController } from './controller.cjs';
  * and `apply_selection_as_mask`. Coordinates are natural pixels of the source image.
  */
 export declare function createImageSelectionTools(controller: ImageEditorController): AgentTool[];
+/** Options for `createImageSegmentationToolProvider`. */
+export interface ImageSegmentationToolProviderOptions {
+    /** Provider id (default `image-segmentation`). */
+    id?: string;
+    /** Tool-name prefix (`<namespace>__<name>`). */
+    namespace?: string;
+    /** Agent playback surface (default the image editor's). */
+    playbackSurface?: string;
+}
+/** The `select_subject` tool: Select › Subject through the controller's segmentation provider. */
+export declare function createImageSubjectSelectionTool(controller: ImageEditorController): AgentTool;
+/**
+ * Agent tools that need the controller's segmentation provider (`select_subject`). The list is
+ * empty while no provider is set, so a host can register this once and the tool appears when a
+ * model is available.
+ */
+export declare function createImageSegmentationToolProvider(controller: ImageEditorController, options?: ImageSegmentationToolProviderOptions): AgentToolProvider;

package/dist/image-editor/selection-tools.d.ts

@@ -1,4 +1,4 @@
-import type { AgentTool } from '../tools/registry.js';
+import type { AgentTool, AgentToolProvider } from '../tools/registry.js';
 import type { ImageEditorController } from './controller.js';
 /**
  * Agent tools for selections and masks (part of `createImageEditorToolProvider`):
@@ -6,3 +6,20 @@ import type { ImageEditorController } from './controller.js';
  * and `apply_selection_as_mask`. Coordinates are natural pixels of the source image.
  */
 export declare function createImageSelectionTools(controller: ImageEditorController): AgentTool[];
+/** Options for `createImageSegmentationToolProvider`. */
+export interface ImageSegmentationToolProviderOptions {
+    /** Provider id (default `image-segmentation`). */
+    id?: string;
+    /** Tool-name prefix (`<namespace>__<name>`). */
+    namespace?: string;
+    /** Agent playback surface (default the image editor's). */
+    playbackSurface?: string;
+}
+/** The `select_subject` tool: Select › Subject through the controller's segmentation provider. */
+export declare function createImageSubjectSelectionTool(controller: ImageEditorController): AgentTool;
+/**
+ * Agent tools that need the controller's segmentation provider (`select_subject`). The list is
+ * empty while no provider is set, so a host can register this once and the tool appears when a
+ * model is available.
+ */
+export declare function createImageSegmentationToolProvider(controller: ImageEditorController, options?: ImageSegmentationToolProviderOptions): AgentToolProvider;

package/dist/image-editor/types.d.cts

@@ -395,6 +395,7 @@ export interface ExportOptions {
     layerIds?: string[];
 }
 import type { EffectProvider } from '../effects/types.cjs';
+import type { SegmentationProvider } from './selection/segmentation.cjs';
 export interface ImageEditorOptions {
     /**
      * What lays out and draws type layers. Default: the browser's fonts through Canvas 2D
@@ -421,6 +422,12 @@ export interface ImageEditorOptions {
      * with `setEffectsProvider`.
      */
     effects?: EffectProvider | null;
+    /**
+     * A segmentation provider (an AI model that finds objects by description) for Select ›
+     * Subject (`selectSubject`, the `select_subject` tool). Optional; change it later with
+     * `setSegmentationProvider`.
+     */
+    segmentation?: SegmentationProvider | null;
 }
 export interface InpaintOptions {
     prompt: string;

package/dist/image-editor/types.d.ts

@@ -395,6 +395,7 @@ export interface ExportOptions {
     layerIds?: string[];
 }
 import type { EffectProvider } from '../effects/types.js';
+import type { SegmentationProvider } from './selection/segmentation.js';
 export interface ImageEditorOptions {
     /**
      * What lays out and draws type layers. Default: the browser's fonts through Canvas 2D
@@ -421,6 +422,12 @@ export interface ImageEditorOptions {
      * with `setEffectsProvider`.
      */
     effects?: EffectProvider | null;
+    /**
+     * A segmentation provider (an AI model that finds objects by description) for Select ›
+     * Subject (`selectSubject`, the `select_subject` tool). Optional; change it later with
+     * `setSegmentationProvider`.
+     */
+    segmentation?: SegmentationProvider | null;
 }
 export interface InpaintOptions {
     prompt: string;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pixerate/editor",
-  "version": "0.178.0",
+  "version": "0.180.0",
   "description": "UI-agnostic rich text and media editing core library",
   "author": "Pixerate",
   "license": "MIT",

package/skills/pixerate-image-editor/SKILL.md

@@ -47,7 +47,7 @@ batching.
 3. **Write in few steps.** Put every base change of one correction into one `set_adjustments`
    call (one undo step) with a `label`. Create layers with their `params` in the same
    `add_adjustment_layer` call rather than adding a neutral layer and then setting it.
-4. **Local edits:** select (`select_region`, `select_by_color`), refine (`modify_selection`,
+4. **Local edits:** select (`select_region`, `select_by_color`, `select_subject`), refine (`modify_selection`,
    `refine_selection`), check with `get_selection` (area, percent, bounds), then
    `apply_selection_as_mask` with a `kind` to create the masked layer in one step.
 5. **Verify.** `render_preview` after each group of corrections, not after each slider, then
@@ -62,7 +62,8 @@ batching.
    `set_adjustments` base controls edit the active pixel layer's `develop` filter.
 2. **New content goes on its own layer**: `add_pixel_layer` (empty, a colour fill or a PNG at
    `x`, `y`), `layer_via_copy` to lift a selection onto a layer, `duplicate_layer` before a
-   risky change. Name layers by intent.
+   risky change. Name layers by intent. Split out a subject by its outline, not its box
+   (`select_subject`, or a `select_region` polygon, then `refine_selection`), then `layer_via_copy`.
 3. **Shape layers non-destructively**: masks (`add_layer_mask` from the selection, or
    reveal-all / hide-all, softened with `density` / `feather`) instead of deleting pixels;
    `clipped: true` (via `set_layer_properties`) to limit a layer or adjustment to the layer below;
@@ -265,6 +266,7 @@ in the source frame at the document's bit depth, colour mode and profile. Read
 | `get_adjustments` | Read the current state: `values` by control id, `process` version (1 or 2), and `layers` bottom to top with id, kind, params, opacity, blend, enabled and mask. Call first, and to find layer ids. |
 | `list_controls` | Get every base control (id, label, section, min, max, step, default, `requiresProcess`), each layer kind's settable controls (plus `opacity`), the blend mode ids and the `layers.<layerId>.<controlId>` path format. Call once per session before writing. |
 | `get_selection` | Read the active selection: `empty`, `bounds` (natural pixels), `area` in pixels, `percent` of the image and a description. Check it after every select or modify step and before masking: a wand that covered 90% of the image is the wrong selection. |
+| `select_subject` | Select › Subject (only when the host has a segmentation model): `target` describes what to select ("the dog", "the sky"); an AI model finds the matching objects and selects their outlines, combined by `mode`. `maxSegments` 1 for a single subject. Returns the selection and `found` (label, bounds per object); nothing found leaves the selection unchanged. |
 | `list_layers` | Read the layer stack bottom to top as a tree: id, type (`pixel`, `adjustment`, `group`, `shape`, `annotation`, `smart`; smart objects add their `contents`, `smartFilters` and `filterMask`; shapes a `shape` summary of component ops and kinds and paint types, and `annotation` when migrated from one), name, document `bounds` of pixel layers, visibility, opacity, `fillOpacity`, blend, `clipped`, locks, a `mask` summary, an `effects` summary (master switch, scale, effect ids and kinds), pixel-layer `filters` (the Background's `develop` filter is the base adjustments), adjustment `kind` and `params`, group `children`, and which layer is `active`. Call it to understand a document with more than the Background before changing it; adjustment layer ids work with every adjustment-layer tool. |
 | `list_layer_effects` | Read one layer's effects: `enabled` (master switch), `scale` (Scale Effects), `margin` (px they reach beyond the shape), `fromPsd`, and every effect with `id`, `kind`, `label` and all its settings. Also returns the `globalLight`, the `patterns` pattern effects can use and the effect `kinds`. Call before changing effects you did not add. |
 | `get_image_info` | Read the geometry: source `image` size, `crop` box (source pixels, or null), `rotation` degrees, `flipH` / `flipV`, the `output` size after crop and rotation, `bitDepth` (8, 16 or 32), the layer count and `canUndo` / `canRedo`. Call before cropping or selecting so coordinates are right. |
@@ -289,7 +291,7 @@ in the source frame at the document's bit depth, colour mode and profile. Read
 | --- | --- |
 | `set_adjustments` | Set base controls and existing layer parameters in one undo step: `values` is a map of control id or `layers.<layerId>.<controlId>` path to number; `label` is echoed. Values are clamped to range; returns `applied` and `clamped` (and a note when `tint` is set on a process 1 edit). Unknown ids fail and change nothing. Cannot set curves points, booleans or colours. |
 | `add_adjustment_layer` | Add a layer above the other adjustment layers and below annotations (or directly above `aboveId`, or at the top of group `parentId`): `kind` (curves, levels, hueSaturation, vibrance, colorBalance, blackWhite, photoFilter, invert, posterize, threshold), optional `params` merged with neutral defaults, `opacity` 0..1 (default 1), `blend` (default normal), `name`. Returns the layer (with its new id) and its stack index. The only tool that sets curves points, `colorize`, `preserveLuminosity`, a black and white `tint` or a photo filter `color`. |
-| `select_region` | Select by `shape`: `rect` or `ellipse` (with `x`, `y`, `width`, `height` in natural pixels), `all`, `none` or `invert`. `mode` replace (default), add, subtract or intersect; `feather` px (default 0). Returns the selection summary. |
+| `select_region` | Select by `shape`: `rect` or `ellipse` (with `x`, `y`, `width`, `height` in natural pixels), `polygon` (the lasso: `points`, at least 3 `{x, y}` along the outline, closed automatically), `all`, `none` or `invert`. `mode` replace (default), add, subtract or intersect; `feather` px (default 0). Returns the selection summary. |
 | `select_by_color` | Magic wand from a seed (`x`, `y`, `tolerance` 0..255 default 32, `contiguous` default true) or Color Range for every similar pixel (`color` "#rrggbb", `fuzziness` 0..200 default 40). `sampleMerged: true` samples the adjusted image instead of the source; `sample: "active-layer"` samples only the active pixel layer (pick it with `set_active_layer`) instead of the whole image. `mode` as above. Use it for skies, backdrops and flat-coloured objects. |
 | `modify_selection` | Grow, shrink, feather, border or smooth the selection (`op`) by `radius` px (0..500). Use shrink then feather to keep an adjustment from bleeding over an edge; smooth to remove wand speckle. Fails when nothing is selected. |
 | `refine_selection` | Refine Edge: snap the selection edge to image edges within `radius` px (0..64, default 8), with `smartRadius`, `smooth` 0..100, `feather` px, `contrast` 0..100 and `shiftEdge` -100..100 %. Use on hair, fur, foliage or after loading a coarse AI mask. |