@meta-sam/graphics 0.1.9 → 0.1.11

package/README.md

@@ -272,9 +272,21 @@ are configurable:
 - Boxes use the same color at `globalAlpha = 1` and a source-coordinate line width
   of `2 / max(abs(scaleX), abs(scaleY))`.
 - Masks render before boxes.
-
-The renderer does not mutate records. Object colors, contour geometry, box styling,
-draw order, and source transforms are not configurable.
+- With `boxLabels: true`, each visible box gets a label at its top-left corner
+  after all boxes are drawn. The text is
+  `formatBoxLabel(boxLabel, objectId, confidence)`: the render's trimmed
+  `boxLabel` option, the box's `objectId`, then its parser `confidence` with
+  three decimals in parentheses, as `pillow 3 (0.945)`. The label and the
+  confidence appear only when present, so a box shows `pillow 3` without a
+  confidence and `3 (0.945)` without a label. The text is white 11px on a 16px-high fill in the object color.
+  Labels are sized in target CSS pixels, so they do not scale with the media. A
+  label sits above its box when the target has room and inside the box's top
+  edge otherwise, and it shifts left to stay inside the target. Hidden objects
+  and boxes from other video frames get no label. Labels are off by default.
+  Pass `boxLabel` to `render()` or `renderVideoFrame()`; it must be a string.
+
+The renderer does not mutate records. Object colors, contour geometry, box and
+label styling, draw order, and source transforms are not configurable.
 
 ## Lifecycle and transactional behavior
 
@@ -316,6 +328,7 @@ import {
 const options: SegmentationRendererOptions = {
   maskFillOpacity: 0.5,
   maskOutline: { width: 2, opacity: 0.9 },
+  boxLabels: true,
   maxCachedPaths: 128,
   maxCachedComplexity: 250_000,
   maxRecords: 20_000,
@@ -335,6 +348,7 @@ const renderer = new SegmentationRenderer(options);
 | `maskOutline`           | `true`                                     |
 | `maskOutline.width`     | `0.003 × min(source.width, source.height)` |
 | `maskOutline.opacity`   | `0.8`                                      |
+| `boxLabels`             | `false`                                    |
 | `maxCachedPaths`        | `128`                                      |
 | `maxCachedComplexity`   | `250_000`                                  |
 | `maxRecords`            | `20_000`                                   |
@@ -348,7 +362,8 @@ const renderer = new SegmentationRenderer(options);
 `maskFillOpacity` and `maskOutline.opacity` must be finite numbers in the inclusive
 range from `0` through `1`. An outline with opacity `0` remains enabled and is still
 stroked. `maskOutline` also accepts `true`, `false`, or `{ width }`; width is in source
-pixels and must be finite and greater than zero. Every resource override must be a
+pixels and must be finite and greater than zero. `boxLabels` must be a
+boolean. Every resource override must be a
 positive safe integer. Invalid constructor options throw `TypeError`. Constructor
 settings are resolved once, so later mutation of an options object has no effect.
 Cache limits evict least-recently-used paths. Other resource limits reject an update
@@ -364,6 +379,8 @@ Only the package root is public; deep imports are not supported.
 | -------------------------------- | ----------------------------------------------------------- |
 | `SegmentationRenderer`           | Retain parser views and render masks and boxes.             |
 | `objectColor`                    | The color assigned to an object ID, for legends and labels. |
+| `formatBoxLabel`                 | The text of one box label, for matching legends.            |
+| `formatConfidence`               | A confidence as box labels show it, for matching legends.   |
 | `SegmentationGraphicsError`      | Base class for package-specific errors.                     |
 | `UnsupportedMaskEncodingError`   | Reject a mask encoding other than `lossless` or `one_bit`.  |
 | `InvalidMaskPayloadError`        | Reject invalid mask data or a conflicting mask revision.    |
@@ -399,6 +416,8 @@ interface SegmentationRendererOptions {
   /** Mask fill opacity. Defaults to 0.35. */
   readonly maskFillOpacity?: number;
   readonly maskOutline?: boolean | MaskOutlineOptions;
+  /** Draw "<boxLabel> <objectId> (<confidence>)" box labels. Defaults to false. */
+  readonly boxLabels?: boolean;
   // Resource limit options are unchanged.
 }
 ```

package/dist/index.d.ts

@@ -1,3 +1,3 @@
 export { InvalidMaskPayloadError, InvalidRenderOptionsError, Path2DUnavailableError, RendererDisposedError, SegmentationGraphicsError, SegmentationResourceLimitError, UnsupportedMaskEncodingError, } from './errors.js';
-export { SegmentationRenderer, objectColor } from './renderer.js';
+export { SegmentationRenderer, formatBoxLabel, formatConfidence, objectColor, } from './renderer.js';
 export type { ImageRenderOptions, MaskOutlineOptions, Rectangle, SegmentationCanvasContext, SegmentationRendererOptions, SegmentationRenderOptions, SegmentationUpdateOptions, VideoFrameCompositionContext, VideoFrameCompositionOptions, VideoFrameFit, VideoRenderOptions, } from './renderer.js';

package/dist/index.js

@@ -2,4 +2,4 @@
  * Copyright (c) Meta Platforms, Inc. and affiliates. All Rights Reserved.
  */
 export { InvalidMaskPayloadError, InvalidRenderOptionsError, Path2DUnavailableError, RendererDisposedError, SegmentationGraphicsError, SegmentationResourceLimitError, UnsupportedMaskEncodingError, } from './errors.js';
-export { SegmentationRenderer, objectColor } from './renderer.js';
+export { SegmentationRenderer, formatBoxLabel, formatConfidence, objectColor, } from './renderer.js';

package/dist/renderer.d.ts

@@ -39,6 +39,8 @@ export interface VideoFrameCompositionOptions {
     readonly devicePixelRatio?: number | (() => number);
     /** Object IDs, not mask identities, to omit from this composition. */
     readonly hiddenIds?: ReadonlySet<string> | readonly string[];
+    /** Text shown before the object ID in box labels; see `boxLabels`. */
+    readonly boxLabel?: string;
 }
 export interface MaskOutlineOptions {
     /**
@@ -58,6 +60,13 @@ export interface SegmentationRendererOptions {
      * fill. Defaults to enabled; pass `false` for fill only.
      */
     readonly maskOutline?: boolean | MaskOutlineOptions;
+    /**
+     * Draw a label at each box's top-left corner: the render's `boxLabel`, the
+     * box's object ID, then its parser `confidence` in parentheses, as
+     * `pillow 3 (0.945)`. The label and the confidence appear only when present.
+     * Defaults to `false`.
+     */
+    readonly boxLabels?: boolean;
     /** Traced paths kept in the LRU cache. */
     readonly maxCachedPaths?: number;
     /** Total traced-path characters kept in the LRU cache. */
@@ -84,6 +93,8 @@ interface RenderOptionsBase {
     readonly source: Rectangle;
     readonly target: Rectangle;
     readonly hiddenIds?: ReadonlySet<string> | readonly string[];
+    /** Text shown before the object ID in box labels; see `boxLabels`. */
+    readonly boxLabel?: string;
 }
 export interface ImageRenderOptions extends RenderOptionsBase {
     readonly media: 'image';
@@ -99,6 +110,14 @@ type SegmentationView = SegmentationResult | SegmentationSnapshot;
  * Exposed so legends and inspectors can match the composited overlay exactly.
  */
 export declare function objectColor(objectId: string): string;
+/** A confidence as box labels show it: the value with three decimals. */
+export declare function formatConfidence(confidence: number): string;
+/**
+ * The text of one box label: the trimmed label, the object ID, then the
+ * confidence in parentheses, as `pillow 3 (0.945)`. A missing or blank label
+ * and a missing confidence are left out, so the object ID is always shown.
+ */
+export declare function formatBoxLabel(label: string | undefined, objectId: string, confidence: number | undefined): string;
 export declare class SegmentationRenderer {
     #private;
     constructor(options?: SegmentationRendererOptions);

package/dist/renderer.js

@@ -179,6 +179,32 @@ const DEFAULT_MASK_OUTLINE_OPACITY = 0.8;
 const MASK_OUTLINE_WIDTH_RATIO = 0.003;
 /** Contour width in target CSS pixels when the source size is unusable. */
 const FALLBACK_MASK_OUTLINE_WIDTH = 1.5;
+/** Box label geometry in target CSS pixels. */
+const BOX_LABEL_HEIGHT = 16;
+const BOX_LABEL_PADDING = 4;
+const BOX_LABEL_FONT = '600 11px system-ui, -apple-system, "Segoe UI", Roboto, sans-serif';
+const BOX_LABEL_TEXT_COLOR = '#ffffff';
+/** A confidence as box labels show it: the value with three decimals. */
+export function formatConfidence(confidence) {
+    return confidence.toFixed(3);
+}
+/**
+ * The text of one box label: the trimmed label, the object ID, then the
+ * confidence in parentheses, as `pillow 3 (0.945)`. A missing or blank label
+ * and a missing confidence are left out, so the object ID is always shown.
+ */
+export function formatBoxLabel(label, objectId, confidence) {
+    return [
+        ...(label === undefined || label.trim().length === 0 ? [] : [label.trim()]),
+        objectId,
+        ...(confidence === undefined ? [] : [`(${formatConfidence(confidence)})`]),
+    ].join(' ');
+}
+function validateBoxLabel(label) {
+    if (label !== undefined && typeof label !== 'string') {
+        throw new InvalidRenderOptionsError('boxLabel must be a string.');
+    }
+}
 function resolveMaskOutline(option) {
     if (option === false) {
         return { enabled: false, width: null, opacity: DEFAULT_MASK_OUTLINE_OPACITY };
@@ -206,6 +232,7 @@ export class SegmentationRenderer {
     #limits;
     #maskFillOpacity;
     #outline;
+    #boxLabels;
     #cache = new Map();
     #cacheComplexity = 0;
     #state;
@@ -215,6 +242,10 @@ export class SegmentationRenderer {
     constructor(options = {}) {
         this.#maskFillOpacity = unitInterval(options.maskFillOpacity, DEFAULT_MASK_FILL_OPACITY, 'maskFillOpacity');
         this.#outline = resolveMaskOutline(options.maskOutline);
+        if (options.boxLabels !== undefined && typeof options.boxLabels !== 'boolean') {
+            throw new TypeError('boxLabels must be a boolean.');
+        }
+        this.#boxLabels = options.boxLabels === true;
         this.#limits = {
             maxCachedPaths: positiveInteger(options.maxCachedPaths, 128, 'maxCachedPaths'),
             maxCachedComplexity: positiveInteger(options.maxCachedComplexity, 250_000, 'maxCachedComplexity'),
@@ -313,6 +344,7 @@ export class SegmentationRenderer {
                 source,
                 target,
                 ...(options.hiddenIds === undefined ? {} : { hiddenIds: options.hiddenIds }),
+                ...(options.boxLabel === undefined ? {} : { boxLabel: options.boxLabel }),
             });
         }
         finally {
@@ -327,6 +359,7 @@ export class SegmentationRenderer {
         validateRectangle(options.target, 'target');
         if (options.media === 'video')
             validateFrame(options.frameIndex);
+        validateBoxLabel(options.boxLabel);
         const state = this.#state;
         if (state === undefined)
             return;
@@ -400,6 +433,54 @@ export class SegmentationRenderer {
                     2 / Math.max(Math.abs(scaleX), Math.abs(scaleY), Number.MIN_VALUE);
                 context.strokeRect(box.left, box.top, box.right - box.left, box.bottom - box.top);
             }
+            if (this.#boxLabels) {
+                this.#paintBoxLabels(context, state, options, hidden, {
+                    scaleX,
+                    scaleY,
+                    offsetX,
+                    offsetY,
+                });
+            }
+        }
+        finally {
+            context.restore();
+        }
+    }
+    /**
+     * Draws each visible box's label in target CSS pixels, so the label keeps
+     * one size whatever the media scale. The label sits on the box's top
+     * edge, above the box when the target has room and inside it otherwise, and
+     * it is shifted left when it would extend past the target's right edge.
+     */
+    #paintBoxLabels(context, state, options, hidden, transform) {
+        const { scaleX, scaleY, offsetX, offsetY } = transform;
+        const { target } = options;
+        context.save();
+        try {
+            // Undo the source-to-target transform; any device-pixel scale stays.
+            context.transform(1 / scaleX, 0, 0, 1 / scaleY, -offsetX / scaleX, -offsetY / scaleY);
+            context.globalAlpha = 1;
+            context.font = BOX_LABEL_FONT;
+            context.textAlign = 'left';
+            context.textBaseline = 'middle';
+            for (const box of state.boxes.values()) {
+                if (hidden.has(box.objectId) ||
+                    (options.media === 'video' &&
+                        box.frameIndex !== undefined &&
+                        box.frameIndex !== options.frameIndex)) {
+                    continue;
+                }
+                const text = formatBoxLabel(options.boxLabel, box.objectId, box.confidence);
+                const width = context.measureText(text).width + 2 * BOX_LABEL_PADDING;
+                const left = offsetX + box.left * scaleX;
+                const top = offsetY + box.top * scaleY;
+                const x = Math.max(target.x, Math.min(left, target.x + target.width - width));
+                const y = top - BOX_LABEL_HEIGHT >= target.y ? top - BOX_LABEL_HEIGHT : top;
+                context.fillStyle = colorFor(box.objectId);
+                context.fillRect(x, y, width, BOX_LABEL_HEIGHT);
+                context.fillStyle = BOX_LABEL_TEXT_COLOR;
+                context.fillText(text, x + BOX_LABEL_PADDING, y + BOX_LABEL_HEIGHT / 2);
+            }
         }
         finally {
             context.restore();
@@ -471,6 +552,12 @@ export class SegmentationRenderer {
                     height < 0) {
                     throw new InvalidRenderOptionsError('Box coordinates are invalid.');
                 }
+                if (record.confidence !== undefined &&
+                    (!Number.isFinite(record.confidence) ||
+                        record.confidence < 0 ||
+                        record.confidence > 1)) {
+                    throw new InvalidRenderOptionsError('Box confidence must be a number from 0 through 1.');
+                }
                 const identity = boxIdentity(record);
                 boxes.set(identity, {
                     identity,
@@ -482,6 +569,7 @@ export class SegmentationRenderer {
                     top: record.top,
                     right: record.right,
                     bottom: record.bottom,
+                    ...(record.confidence === undefined ? {} : { confidence: record.confidence }),
                 });
             }
         }
@@ -496,6 +584,7 @@ export class SegmentationRenderer {
                 box.top,
                 box.right,
                 box.bottom,
+                box.confidence ?? null,
             ]).length;
             if (complexity > this.#limits.maxRetainedComplexity) {
                 throw new SegmentationResourceLimitError('maxRetainedComplexity');

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@meta-sam/graphics",
-  "version": "0.1.9",
+  "version": "0.1.11",
   "description": "Retained Canvas 2D renderer for SAM 3 segmentation",
   "license": "SEE LICENSE IN LICENSE",
   "homepage": "https://dev.meta.ai/",
@@ -33,7 +33,7 @@
     "node": "^20.17.0 || >=22.9.0"
   },
   "dependencies": {
-    "@meta-sam/parser": "0.0.12"
+    "@meta-sam/parser": "0.0.13"
   },
   "scripts": {
     "build": "tsc -b",