@elizaos/plugin-vision 2.0.0-beta.1 → 2.0.3-beta.5

package/dist/set-of-marks-provider.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"set-of-marks-provider.d.ts","sourceRoot":"","sources":["../src/set-of-marks-provider.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAEH,OAAO,EAAE,uBAAuB,EAAE,MAAM,sBAAsB,CAAC;AAC/D,OAAO,EAEL,KAAK,kBAAkB,EAGvB,KAAK,iBAAiB,EACtB,KAAK,OAAO,EACb,MAAM,UAAU,CAAC;AAElB,2DAA2D;AAC3D,MAAM,WAAW,mBAAmB;IAClC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,QAAQ,EAAE,UAAU,CAAC;IAC9B,QAAQ,CAAC,aAAa,CAAC,EAAE,OAAO,CAAC;CAClC;AAED,4DAA4D;AAC5D,MAAM,WAAW,oBAAoB;IACnC,QAAQ,CAAC,KAAK,EAAE,aAAa,CAAC,OAAO,CAAC,CAAC;IACvC,QAAQ,CAAC,gBAAgB,CAAC,EAAE,MAAM,CAAC;CACpC;AAED,8DAA8D;AAC9D,MAAM,WAAW,sBAAsB;IACrC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,KAAK,EAAE,mBAAmB,GAAG,OAAO,CAAC,oBAAoB,CAAC,CAAC;CACrE;AAED,MAAM,MAAM,0BAA0B,GAAG,CACvC,QAAQ,EAAE,sBAAsB,GAAG,IAAI,KACpC,IAAI,CAAC;AAEV,eAAO,MAAM,+BAA+B,+BAA+B,CAAC;AAE5E,MAAM,WAAW,sBAAsB;IACrC;;;OAGG;IACH,QAAQ,CAAC,WAAW,CAAC,EAAE,CACrB,QAAQ,EAAE,UAAU,KACjB,OAAO,CAAC,kBAAkB,EAAE,CAAC,CAAC;IACnC,4EAA4E;IAC5E,QAAQ,CAAC,UAAU,CAAC,EAAE,OAAO,uBAAuB,CAAC;IACrD,oDAAoD;IACpD,QAAQ,CAAC,OAAO,CAAC,EAAE,iBAAiB,CAAC;CACtC;AAED;;;;GAIG;AACH,wBAAgB,yBAAyB,IAAI,CAC3C,QAAQ,EAAE,UAAU,KACjB,OAAO,CAAC,kBAAkB,EAAE,CAAC,CA0BjC;AAED;;;GAGG;AACH,wBAAgB,6BAA6B,CAC3C,IAAI,GAAE,sBAA2B,GAChC,sBAAsB,CAkCxB;AAED;;;GAGG;AACH,wBAAgB,+BAA+B,CAC7C,QAAQ,EAAE,0BAA0B,EACpC,IAAI,CAAC,EAAE,sBAAsB,GAC5B,OAAO,CAGT"}

package/dist/som.d.ts

@@ -0,0 +1,135 @@
+/**
+ * Set-of-Marks (SoM) grounding — #9170 M9.
+ *
+ * Set-of-Marks is the grounding technique trycua/cua uses via OmniParser: take
+ * the icon detections (GGUF YOLO) and the OCR text boxes (the `CoordOcrProvider`
+ * seam plugin-vision already feeds), fuse them into ONE deduplicated set of
+ * candidate targets, draw a 1-indexed numbered box over each on the screenshot,
+ * and let the VLM pick a *number* instead of raw pixel coordinates. Numeric
+ * selection is far more reliable than free-floating coordinate regression.
+ *
+ * This module is split into:
+ *   - a PURE core (`buildSetOfMarks`) — icon-over-text suppression + NMS +
+ *     deterministic reading-order numbering. Dependency-free and structurally
+ *     typed so it unit-tests with zero environment, mirroring
+ *     `get-screen-elements.ts` and `computeruse-ocr-bridge.ts`.
+ *   - a renderer (`renderSetOfMarksOverlay`) — composites a numbered SVG overlay
+ *     onto the source PNG via `sharp`.
+ *
+ * The OmniParser fusion rules we reproduce:
+ *   1. Icon-over-text suppression — a text box mostly covered by an icon box is
+ *      dropped; the icon is the interactable, the text is its caption.
+ *   2. Non-max suppression — overlapping boxes collapse to the highest-priority
+ *      one (icons outrank text on ties), so each target is marked once.
+ *   3. 1-indexed reading-order numbering — top-to-bottom, then left-to-right,
+ *      with a row tolerance so a visual row isn't scrambled by sub-pixel y jitter.
+ */
+/** Display-local bounding box `[x, y, w, h]`. */
+export type SomBbox = readonly [number, number, number, number];
+/** Where a candidate mark came from. Icons outrank text during suppression. */
+export type SomSource = "icon" | "text";
+/** A raw candidate box fed into the SoM fusion. */
+export interface SomCandidate {
+    readonly bbox: SomBbox;
+    readonly source: SomSource;
+    /** Class name (icon) or recognized text (text). Optional. */
+    readonly label?: string;
+    /** Detector/OCR confidence in [0, 1]. Missing → treated as 0.5. */
+    readonly score?: number;
+}
+/** A finalized, numbered mark in the overlay. */
+export interface SomMark {
+    /** 1-indexed mark number shown in the overlay. */
+    readonly index: number;
+    readonly bbox: [number, number, number, number];
+    /** Box center `[x, y]` — the click target the VLM's number resolves to. */
+    readonly center: [number, number];
+    readonly source: SomSource;
+    readonly label?: string;
+    readonly score: number;
+}
+export interface SetOfMarksOptions {
+    /**
+     * A text box is dropped when this fraction of its area is covered by an icon
+     * box (icon-over-text suppression). Default 0.7.
+     */
+    readonly iconOverTextCoverage?: number;
+    /** Boxes overlapping above this IoU collapse during NMS. Default 0.5. */
+    readonly nmsIouThreshold?: number;
+    /**
+     * Rows within this many pixels of vertical offset are treated as the same
+     * reading row (so left-to-right ordering holds across a row). Default 12.
+     */
+    readonly rowTolerance?: number;
+    /** Drop candidates with a smaller score before fusion. Default 0 (keep all). */
+    readonly minScore?: number;
+}
+/** Fraction of `inner`'s area covered by `outer` (containment ratio). Pure. */
+export declare function coverageRatio(inner: SomBbox, outer: SomBbox): number;
+/**
+ * Fuse icon + text candidates into a deduplicated, 1-indexed set of marks.
+ *
+ * Pure and deterministic: same inputs → identical numbering, regardless of
+ * input ordering. Degenerate boxes (non-finite / zero-area) are dropped.
+ */
+export declare function buildSetOfMarks(candidates: readonly SomCandidate[], options?: SetOfMarksOptions): SomMark[];
+export interface SomOverlayOptions extends SetOfMarksOptions {
+    /** Stroke / badge color for icon marks. Default "#FF8C00" (orange). */
+    readonly iconColor?: string;
+    /** Stroke / badge color for text marks. Default "#1FA9FF". */
+    readonly textColor?: string;
+    /** Box stroke width in px. Default 2. */
+    readonly strokeWidth?: number;
+    /** Badge font size in px. Default 13. */
+    readonly fontSize?: number;
+}
+/**
+ * Build the SVG overlay markup for a set of marks over a `width × height`
+ * canvas. Pure (no I/O) — separated from the raster composite so it is
+ * unit-testable and reusable by non-sharp consumers (e.g. a browser overlay).
+ */
+export declare function buildSetOfMarksSvg(marks: readonly SomMark[], width: number, height: number, options?: SomOverlayOptions): string;
+/** Structural shape of a `DetectedObject` (YOLO) — `{x,y,width,height}` box. */
+export interface DetectedObjectLike {
+    readonly boundingBox: {
+        readonly x: number;
+        readonly y: number;
+        readonly width: number;
+        readonly height: number;
+    };
+    readonly type?: string;
+    readonly confidence?: number;
+}
+/** Structural shape of an OCR block (`OcrWithCoordsBlock`). */
+export interface OcrBlockLike {
+    readonly text?: string;
+    readonly confidence?: number;
+    readonly bbox: {
+        readonly x: number;
+        readonly y: number;
+        readonly width: number;
+        readonly height: number;
+    };
+}
+/** Adapt GGUF YOLO detections into icon-source SoM candidates. */
+export declare function somCandidatesFromDetections(objects: readonly DetectedObjectLike[]): SomCandidate[];
+/** Adapt `CoordOcrProvider` text blocks into text-source SoM candidates. */
+export declare function somCandidatesFromOcr(blocks: readonly OcrBlockLike[]): SomCandidate[];
+/**
+ * Convenience: fuse YOLO detections + OCR blocks straight into a numbered mark
+ * set. The seam `detect_elements`/grounding calls — pass whatever icon and text
+ * boxes the scene already has.
+ */
+export declare function buildSceneSetOfMarks(args: {
+    readonly detections?: readonly DetectedObjectLike[];
+    readonly ocrBlocks?: readonly OcrBlockLike[];
+}, options?: SetOfMarksOptions): SomMark[];
+/**
+ * Composite a numbered Set-of-Marks overlay onto a source PNG.
+ *
+ * Returns PNG bytes the same size as the input. `sharp` is loaded lazily so the
+ * pure core (`buildSetOfMarks`) carries no native dependency for consumers that
+ * only need coordinates.
+ */
+export declare function renderSetOfMarksOverlay(pngBytes: Uint8Array, marks: readonly SomMark[], options?: SomOverlayOptions): Promise<Buffer>;
+//# sourceMappingURL=som.d.ts.map

package/dist/som.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"som.d.ts","sourceRoot":"","sources":["../src/som.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AAIH,iDAAiD;AACjD,MAAM,MAAM,OAAO,GAAG,SAAS,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;AAEhE,+EAA+E;AAC/E,MAAM,MAAM,SAAS,GAAG,MAAM,GAAG,MAAM,CAAC;AAExC,mDAAmD;AACnD,MAAM,WAAW,YAAY;IAC3B,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC;IACvB,QAAQ,CAAC,MAAM,EAAE,SAAS,CAAC;IAC3B,6DAA6D;IAC7D,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,CAAC;IACxB,mEAAmE;IACnE,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,CAAC;CACzB;AAED,iDAAiD;AACjD,MAAM,WAAW,OAAO;IACtB,kDAAkD;IAClD,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,IAAI,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;IAChD,2EAA2E;IAC3E,QAAQ,CAAC,MAAM,EAAE,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAClC,QAAQ,CAAC,MAAM,EAAE,SAAS,CAAC;IAC3B,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;CACxB;AAED,MAAM,WAAW,iBAAiB;IAChC;;;OAGG;IACH,QAAQ,CAAC,oBAAoB,CAAC,EAAE,MAAM,CAAC;IACvC,yEAAyE;IACzE,QAAQ,CAAC,eAAe,CAAC,EAAE,MAAM,CAAC;IAClC;;;OAGG;IACH,QAAQ,CAAC,YAAY,CAAC,EAAE,MAAM,CAAC;IAC/B,gFAAgF;IAChF,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAC;CAC5B;AAuBD,+EAA+E;AAC/E,wBAAgB,aAAa,CAAC,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,GAAG,MAAM,CAIpE;AAaD;;;;;GAKG;AACH,wBAAgB,eAAe,CAC7B,UAAU,EAAE,SAAS,YAAY,EAAE,EACnC,OAAO,GAAE,iBAAsB,GAC9B,OAAO,EAAE,CAiEX;AAYD,MAAM,WAAW,iBAAkB,SAAQ,iBAAiB;IAC1D,uEAAuE;IACvE,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC;IAC5B,8DAA8D;IAC9D,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC;IAC5B,yCAAyC;IACzC,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAC;IAC9B,yCAAyC;IACzC,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAC;CAC5B;AAED;;;;GAIG;AACH,wBAAgB,kBAAkB,CAChC,KAAK,EAAE,SAAS,OAAO,EAAE,EACzB,KAAK,EAAE,MAAM,EACb,MAAM,EAAE,MAAM,EACd,OAAO,GAAE,iBAAsB,GAC9B,MAAM,CA+BR;AASD,gFAAgF;AAChF,MAAM,WAAW,kBAAkB;IACjC,QAAQ,CAAC,WAAW,EAAE;QACpB,QAAQ,CAAC,CAAC,EAAE,MAAM,CAAC;QACnB,QAAQ,CAAC,CAAC,EAAE,MAAM,CAAC;QACnB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;QACvB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;KACzB,CAAC;IACF,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,CAAC;CAC9B;AAED,+DAA+D;AAC/D,MAAM,WAAW,YAAY;IAC3B,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,IAAI,EAAE;QACb,QAAQ,CAAC,CAAC,EAAE,MAAM,CAAC;QACnB,QAAQ,CAAC,CAAC,EAAE,MAAM,CAAC;QACnB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;QACvB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;KACzB,CAAC;CACH;AAED,kEAAkE;AAClE,wBAAgB,2BAA2B,CACzC,OAAO,EAAE,SAAS,kBAAkB,EAAE,GACrC,YAAY,EAAE,CAehB;AAED,4EAA4E;AAC5E,wBAAgB,oBAAoB,CAClC,MAAM,EAAE,SAAS,YAAY,EAAE,GAC9B,YAAY,EAAE,CAUhB;AAED;;;;GAIG;AACH,wBAAgB,oBAAoB,CAClC,IAAI,EAAE;IACJ,QAAQ,CAAC,UAAU,CAAC,EAAE,SAAS,kBAAkB,EAAE,CAAC;IACpD,QAAQ,CAAC,SAAS,CAAC,EAAE,SAAS,YAAY,EAAE,CAAC;CAC9C,EACD,OAAO,CAAC,EAAE,iBAAiB,GAC1B,OAAO,EAAE,CAQX;AAED;;;;;;GAMG;AACH,wBAAsB,uBAAuB,CAC3C,QAAQ,EAAE,UAAU,EACpB,KAAK,EAAE,SAAS,OAAO,EAAE,EACzB,OAAO,GAAE,iBAAsB,GAC9B,OAAO,CAAC,MAAM,CAAC,CAgBjB"}

package/dist/som.js

@@ -0,0 +1,184 @@
+import { createRequire } from "node:module";
+var __defProp = Object.defineProperty;
+var __returnValue = (v) => v;
+function __exportSetter(name, newValue) {
+  this[name] = __returnValue.bind(null, newValue);
+}
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, {
+      get: all[name],
+      enumerable: true,
+      configurable: true,
+      set: __exportSetter.bind(all, name)
+    });
+};
+var __esm = (fn, res) => () => (fn && (res = fn(fn = 0)), res);
+var __require = /* @__PURE__ */ createRequire(import.meta.url);
+
+// src/get-screen-elements.ts
+function bboxIou(a, b) {
+  const [ax, ay, aw, ah] = a;
+  const [bx, by, bw, bh] = b;
+  if (aw <= 0 || ah <= 0 || bw <= 0 || bh <= 0)
+    return 0;
+  const ix = Math.max(ax, bx);
+  const iy = Math.max(ay, by);
+  const ix2 = Math.min(ax + aw, bx + bw);
+  const iy2 = Math.min(ay + ah, by + bh);
+  const iw = Math.max(0, ix2 - ix);
+  const ih = Math.max(0, iy2 - iy);
+  const inter = iw * ih;
+  if (inter <= 0)
+    return 0;
+  const union = aw * ah + bw * bh - inter;
+  return union > 0 ? inter / union : 0;
+}
+
+// src/som.ts
+var DEFAULT_ICON_OVER_TEXT_COVERAGE = 0.7;
+var DEFAULT_NMS_IOU = 0.5;
+var DEFAULT_ROW_TOLERANCE = 12;
+var DEFAULT_SCORE = 0.5;
+function intersectionArea(a, b) {
+  const ix = Math.max(a[0], b[0]);
+  const iy = Math.max(a[1], b[1]);
+  const ix2 = Math.min(a[0] + a[2], b[0] + b[2]);
+  const iy2 = Math.min(a[1] + a[3], b[1] + b[3]);
+  return Math.max(0, ix2 - ix) * Math.max(0, iy2 - iy);
+}
+function coverageRatio(inner, outer) {
+  const area = inner[2] * inner[3];
+  if (area <= 0)
+    return 0;
+  return intersectionArea(inner, outer) / area;
+}
+function isValidBox(b) {
+  return Number.isFinite(b[0]) && Number.isFinite(b[1]) && Number.isFinite(b[2]) && Number.isFinite(b[3]) && b[2] > 0 && b[3] > 0;
+}
+function buildSetOfMarks(candidates, options = {}) {
+  const iconOverText = options.iconOverTextCoverage ?? DEFAULT_ICON_OVER_TEXT_COVERAGE;
+  const nmsIou = options.nmsIouThreshold ?? DEFAULT_NMS_IOU;
+  const rowTolerance = options.rowTolerance ?? DEFAULT_ROW_TOLERANCE;
+  const minScore = options.minScore ?? 0;
+  const norm = candidates.filter((c) => isValidBox(c.bbox)).map((c) => ({
+    bbox: [c.bbox[0], c.bbox[1], c.bbox[2], c.bbox[3]],
+    source: c.source,
+    label: c.label,
+    score: c.score ?? DEFAULT_SCORE
+  })).filter((c) => c.score >= minScore);
+  const icons = norm.filter((c) => c.source === "icon");
+  const texts = norm.filter((c) => c.source === "text");
+  const keptTexts = texts.filter((t) => !icons.some((icon) => coverageRatio(t.bbox, icon.bbox) >= iconOverText));
+  const pool = [...icons, ...keptTexts].sort((a, b) => {
+    if (a.source !== b.source)
+      return a.source === "icon" ? -1 : 1;
+    if (b.score !== a.score)
+      return b.score - a.score;
+    return a.bbox[1] - b.bbox[1] || a.bbox[0] - b.bbox[0];
+  });
+  const kept = [];
+  for (const cand of pool) {
+    const overlaps = kept.some((k) => bboxIou(cand.bbox, k.bbox) >= nmsIou);
+    if (!overlaps)
+      kept.push(cand);
+  }
+  const ordered = [...kept].sort((a, b) => {
+    const sameRow = Math.abs(a.bbox[1] - b.bbox[1]) <= rowTolerance;
+    if (sameRow)
+      return a.bbox[0] - b.bbox[0] || a.bbox[1] - b.bbox[1];
+    return a.bbox[1] - b.bbox[1];
+  });
+  return ordered.map((c, i) => {
+    const [x, y, w, h] = c.bbox;
+    const mark = {
+      index: i + 1,
+      bbox: [x, y, w, h],
+      center: [Math.round(x + w / 2), Math.round(y + h / 2)],
+      source: c.source,
+      score: c.score
+    };
+    if (c.label !== undefined)
+      mark.label = c.label;
+    return mark;
+  });
+}
+function escapeXml(s) {
+  return s.replace(/&/g, "&amp;").replace(/</g, "&lt;").replace(/>/g, "&gt;").replace(/"/g, "&quot;").replace(/'/g, "&apos;");
+}
+function buildSetOfMarksSvg(marks, width, height, options = {}) {
+  const iconColor = options.iconColor ?? "#FF8C00";
+  const textColor = options.textColor ?? "#1FA9FF";
+  const strokeWidth = options.strokeWidth ?? 2;
+  const fontSize = options.fontSize ?? 13;
+  const badgeW = Math.round(fontSize * 1.4);
+  const badgeH = Math.round(fontSize * 1.3);
+  const parts = [];
+  for (const mark of marks) {
+    const [x, y, w, h] = mark.bbox;
+    const color = mark.source === "icon" ? iconColor : textColor;
+    const label = String(mark.index);
+    const bx = Math.max(0, Math.min(x, width - badgeW));
+    const by = Math.max(0, Math.min(y, height - badgeH));
+    parts.push(`<rect x="${x}" y="${y}" width="${w}" height="${h}" fill="none" ` + `stroke="${color}" stroke-width="${strokeWidth}" />`, `<rect x="${bx}" y="${by}" width="${badgeW}" height="${badgeH}" ` + `fill="${color}" />`, `<text x="${bx + badgeW / 2}" y="${by + badgeH / 2}" ` + `font-family="monospace" font-size="${fontSize}" font-weight="bold" ` + `fill="#000000" text-anchor="middle" dominant-baseline="central">` + `${escapeXml(label)}</text>`);
+  }
+  return `<svg xmlns="http://www.w3.org/2000/svg" width="${width}" ` + `height="${height}">${parts.join("")}</svg>`;
+}
+function somCandidatesFromDetections(objects) {
+  return objects.map((o) => {
+    const c = {
+      bbox: [
+        o.boundingBox.x,
+        o.boundingBox.y,
+        o.boundingBox.width,
+        o.boundingBox.height
+      ],
+      source: "icon",
+      score: o.confidence ?? DEFAULT_SCORE
+    };
+    if (o.type !== undefined)
+      c.label = o.type;
+    return c;
+  });
+}
+function somCandidatesFromOcr(blocks) {
+  return blocks.map((b) => {
+    const c = {
+      bbox: [b.bbox.x, b.bbox.y, b.bbox.width, b.bbox.height],
+      source: "text",
+      score: b.confidence ?? DEFAULT_SCORE
+    };
+    if (b.text !== undefined)
+      c.label = b.text;
+    return c;
+  });
+}
+function buildSceneSetOfMarks(args, options) {
+  return buildSetOfMarks([
+    ...somCandidatesFromDetections(args.detections ?? []),
+    ...somCandidatesFromOcr(args.ocrBlocks ?? [])
+  ], options);
+}
+async function renderSetOfMarksOverlay(pngBytes, marks, options = {}) {
+  const { default: sharp } = await import("sharp");
+  const base = sharp(Buffer.from(pngBytes));
+  const meta = await base.metadata();
+  const width = meta.width ?? 0;
+  const height = meta.height ?? 0;
+  if (width <= 0 || height <= 0) {
+    throw new Error(`[vision/som] cannot overlay marks: source image has no dimensions`);
+  }
+  const svg = buildSetOfMarksSvg(marks, width, height, options);
+  return base.composite([{ input: Buffer.from(svg), top: 0, left: 0 }]).png().toBuffer();
+}
+export {
+  somCandidatesFromOcr,
+  somCandidatesFromDetections,
+  renderSetOfMarksOverlay,
+  coverageRatio,
+  buildSetOfMarksSvg,
+  buildSetOfMarks,
+  buildSceneSetOfMarks
+};
+
+//# debugId=55C971527120024164756E2164756E21

package/dist/som.js.map

@@ -0,0 +1,11 @@
+{
+  "version": 3,
+  "sources": ["../src/get-screen-elements.ts", "../src/som.ts"],
+  "sourcesContent": [
+    "/**\n * Pure element-merge core for the GET_SCREEN action (#9105 Slice 2 / M2).\n *\n * GET_SCREEN returns a cheap, token-frugal list of grounded, clickable screen\n * elements unified from three sources: OCR text boxes, accessibility (AX)\n * clickables, and (optionally) VLM-detected elements. This module is the\n * deterministic heart of that envelope — it collapses the three sources into a\n * single deduplicated, stably-ordered element list, recording each element's\n * `groundingSources` provenance.\n *\n * Like the M1 OCR bridge (`computeruse-ocr-bridge.ts`), this is intentionally\n * dependency-free and pure: the source types live in `@elizaos/plugin-computeruse`\n * (`SceneOcrBox` / `SceneAxNode` / `SceneVlmElement`), but we describe their\n * shapes STRUCTURALLY here rather than importing them, to keep the no-hard-dep\n * rule. That also makes the merge engine fully unit-testable with zero\n * environment, decoupled from the runtime/native/model wiring (Slice 3).\n */\n\n/** Display-local bounding box `[x, y, w, h]`. */\nexport type Bbox = readonly [number, number, number, number];\n\n/** Structural shape of computeruse's `SceneOcrBox`. */\nexport interface OcrBoxLike {\n  readonly id: string;\n  readonly text: string;\n  readonly bbox: Bbox;\n  readonly conf?: number;\n  readonly displayId: number;\n}\n\n/** Structural shape of computeruse's `SceneAxNode`. */\nexport interface AxNodeLike {\n  readonly id: string;\n  readonly role: string;\n  readonly label?: string;\n  readonly bbox: Bbox;\n  readonly actions?: readonly string[];\n  readonly displayId: number;\n}\n\n/** Structural shape of computeruse's `SceneVlmElement`. */\nexport interface VlmElementLike {\n  readonly id: string;\n  readonly kind: string;\n  readonly desc: string;\n  readonly bbox: Bbox;\n  readonly displayId: number;\n}\n\nexport type GroundingSource = \"ocr\" | \"ax\" | \"vlm\";\n\n/** A single unified, grounded screen element in the GET_SCREEN envelope. */\nexport interface GetScreenElement {\n  /** Stable id, preferring the AX id, then OCR, then VLM. */\n  id: string;\n  /** Display-local bbox `[x, y, w, h]` of the representative (highest-priority) source. */\n  bbox: [number, number, number, number];\n  /** User-facing text/label: AX label, else OCR text, else VLM description. */\n  text: string;\n  /** Element kind/role when known: AX role, else VLM kind. */\n  kind?: string;\n  displayId: number;\n  /** AX actions when the element is accessibility-grounded. */\n  actions?: string[];\n  /** Provenance — every source that contributed to this element, in fixed\n   * `ocr < ax < vlm` order for stability. Always non-empty. */\n  groundingSources: GroundingSource[];\n}\n\nexport interface MergeScreenInput {\n  readonly ocr?: readonly OcrBoxLike[];\n  readonly ax?: readonly AxNodeLike[];\n  readonly vlm?: readonly VlmElementLike[];\n}\n\nexport interface MergeScreenOptions {\n  /** Boxes whose IoU exceeds this collapse into one element (default 0.6). */\n  readonly iouThreshold?: number;\n}\n\nconst DEFAULT_IOU_THRESHOLD = 0.6;\n\n/** Intersection-over-union of two `[x, y, w, h]` boxes. 0 when either is empty\n * or they don't overlap. */\nexport function bboxIou(a: Bbox, b: Bbox): number {\n  const [ax, ay, aw, ah] = a;\n  const [bx, by, bw, bh] = b;\n  if (aw <= 0 || ah <= 0 || bw <= 0 || bh <= 0) return 0;\n  const ix = Math.max(ax, bx);\n  const iy = Math.max(ay, by);\n  const ix2 = Math.min(ax + aw, bx + bw);\n  const iy2 = Math.min(ay + ah, by + bh);\n  const iw = Math.max(0, ix2 - ix);\n  const ih = Math.max(0, iy2 - iy);\n  const inter = iw * ih;\n  if (inter <= 0) return 0;\n  const union = aw * ah + bw * bh - inter;\n  return union > 0 ? inter / union : 0;\n}\n\ninterface Cluster {\n  displayId: number;\n  bbox: Bbox;\n  ax?: AxNodeLike;\n  ocr?: OcrBoxLike;\n  vlm?: VlmElementLike;\n}\n\n/** Stable sort key: top-to-bottom, then left-to-right, then by id for ties. */\nfunction byPosition(\n  a: { bbox: Bbox; id: string },\n  b: { bbox: Bbox; id: string },\n): number {\n  return (\n    a.bbox[1] - b.bbox[1] || a.bbox[0] - b.bbox[0] || a.id.localeCompare(b.id)\n  );\n}\n\n/**\n * Merge OCR boxes + AX clickables + VLM elements into one deduplicated,\n * deterministically-ordered element list.\n *\n * - Elements from different sources whose bboxes overlap above `iouThreshold`\n *   (and share a `displayId`) collapse into one element that records all\n *   contributing sources in `groundingSources`.\n * - Field precedence is AX > OCR > VLM (AX wins id/label/role; OCR text fills\n *   in when AX has no label; VLM desc is the last resort).\n * - Output order is top-to-bottom, then left-to-right, so the envelope is\n *   stable across turns regardless of input ordering.\n * - Degrades gracefully: any source may be absent/empty (e.g. accessibility off)\n *   and the function never throws.\n */\nexport function mergeScreenElements(\n  input: MergeScreenInput,\n  options: MergeScreenOptions = {},\n): GetScreenElement[] {\n  const threshold = options.iouThreshold ?? DEFAULT_IOU_THRESHOLD;\n  const clusters: Cluster[] = [];\n\n  const attach = (\n    displayId: number,\n    bbox: Bbox,\n    set: (c: Cluster) => void,\n  ): void => {\n    const match = clusters.find(\n      (c) => c.displayId === displayId && bboxIou(c.bbox, bbox) >= threshold,\n    );\n    if (match) {\n      set(match);\n      return;\n    }\n    const cluster: Cluster = { displayId, bbox };\n    set(cluster);\n    clusters.push(cluster);\n  };\n\n  // Process in precedence order so a cluster's representative bbox is set by its\n  // highest-priority contributing source, and pre-sort each source by position\n  // so cluster creation order is deterministic.\n  for (const ax of [...(input.ax ?? [])].sort(byPosition)) {\n    attach(ax.displayId, ax.bbox, (c) => {\n      if (!c.ax) c.ax = ax;\n    });\n  }\n  for (const ocr of [...(input.ocr ?? [])].sort(byPosition)) {\n    attach(ocr.displayId, ocr.bbox, (c) => {\n      if (!c.ocr) c.ocr = ocr;\n    });\n  }\n  for (const vlm of [...(input.vlm ?? [])].sort(byPosition)) {\n    attach(vlm.displayId, vlm.bbox, (c) => {\n      if (!c.vlm) c.vlm = vlm;\n    });\n  }\n\n  const elements = clusters.map((c): GetScreenElement => {\n    const groundingSources: GroundingSource[] = [];\n    if (c.ocr) groundingSources.push(\"ocr\");\n    if (c.ax) groundingSources.push(\"ax\");\n    if (c.vlm) groundingSources.push(\"vlm\");\n\n    const id = c.ax?.id ?? c.ocr?.id ?? c.vlm?.id ?? \"el\";\n    const text = c.ax?.label || c.ocr?.text || c.vlm?.desc || \"\";\n    const kind = c.ax?.role ?? c.vlm?.kind;\n    const [x, y, w, h] = c.bbox;\n\n    const element: GetScreenElement = {\n      id,\n      bbox: [x, y, w, h],\n      text,\n      displayId: c.displayId,\n      groundingSources,\n    };\n    if (kind !== undefined) element.kind = kind;\n    if (c.ax?.actions && c.ax.actions.length > 0) {\n      element.actions = [...c.ax.actions];\n    }\n    return element;\n  });\n\n  return elements.sort(byPosition);\n}\n",
+    "/**\n * Set-of-Marks (SoM) grounding — #9170 M9.\n *\n * Set-of-Marks is the grounding technique trycua/cua uses via OmniParser: take\n * the icon detections (GGUF YOLO) and the OCR text boxes (the `CoordOcrProvider`\n * seam plugin-vision already feeds), fuse them into ONE deduplicated set of\n * candidate targets, draw a 1-indexed numbered box over each on the screenshot,\n * and let the VLM pick a *number* instead of raw pixel coordinates. Numeric\n * selection is far more reliable than free-floating coordinate regression.\n *\n * This module is split into:\n *   - a PURE core (`buildSetOfMarks`) — icon-over-text suppression + NMS +\n *     deterministic reading-order numbering. Dependency-free and structurally\n *     typed so it unit-tests with zero environment, mirroring\n *     `get-screen-elements.ts` and `computeruse-ocr-bridge.ts`.\n *   - a renderer (`renderSetOfMarksOverlay`) — composites a numbered SVG overlay\n *     onto the source PNG via `sharp`.\n *\n * The OmniParser fusion rules we reproduce:\n *   1. Icon-over-text suppression — a text box mostly covered by an icon box is\n *      dropped; the icon is the interactable, the text is its caption.\n *   2. Non-max suppression — overlapping boxes collapse to the highest-priority\n *      one (icons outrank text on ties), so each target is marked once.\n *   3. 1-indexed reading-order numbering — top-to-bottom, then left-to-right,\n *      with a row tolerance so a visual row isn't scrambled by sub-pixel y jitter.\n */\n\nimport { bboxIou } from \"./get-screen-elements.js\";\n\n/** Display-local bounding box `[x, y, w, h]`. */\nexport type SomBbox = readonly [number, number, number, number];\n\n/** Where a candidate mark came from. Icons outrank text during suppression. */\nexport type SomSource = \"icon\" | \"text\";\n\n/** A raw candidate box fed into the SoM fusion. */\nexport interface SomCandidate {\n  readonly bbox: SomBbox;\n  readonly source: SomSource;\n  /** Class name (icon) or recognized text (text). Optional. */\n  readonly label?: string;\n  /** Detector/OCR confidence in [0, 1]. Missing → treated as 0.5. */\n  readonly score?: number;\n}\n\n/** A finalized, numbered mark in the overlay. */\nexport interface SomMark {\n  /** 1-indexed mark number shown in the overlay. */\n  readonly index: number;\n  readonly bbox: [number, number, number, number];\n  /** Box center `[x, y]` — the click target the VLM's number resolves to. */\n  readonly center: [number, number];\n  readonly source: SomSource;\n  readonly label?: string;\n  readonly score: number;\n}\n\nexport interface SetOfMarksOptions {\n  /**\n   * A text box is dropped when this fraction of its area is covered by an icon\n   * box (icon-over-text suppression). Default 0.7.\n   */\n  readonly iconOverTextCoverage?: number;\n  /** Boxes overlapping above this IoU collapse during NMS. Default 0.5. */\n  readonly nmsIouThreshold?: number;\n  /**\n   * Rows within this many pixels of vertical offset are treated as the same\n   * reading row (so left-to-right ordering holds across a row). Default 12.\n   */\n  readonly rowTolerance?: number;\n  /** Drop candidates with a smaller score before fusion. Default 0 (keep all). */\n  readonly minScore?: number;\n}\n\nconst DEFAULT_ICON_OVER_TEXT_COVERAGE = 0.7;\nconst DEFAULT_NMS_IOU = 0.5;\nconst DEFAULT_ROW_TOLERANCE = 12;\nconst DEFAULT_SCORE = 0.5;\n\ninterface NormCandidate {\n  bbox: [number, number, number, number];\n  source: SomSource;\n  label?: string;\n  score: number;\n}\n\n/** Area of intersection of two `[x, y, w, h]` boxes (0 when disjoint). Pure. */\nfunction intersectionArea(a: SomBbox, b: SomBbox): number {\n  const ix = Math.max(a[0], b[0]);\n  const iy = Math.max(a[1], b[1]);\n  const ix2 = Math.min(a[0] + a[2], b[0] + b[2]);\n  const iy2 = Math.min(a[1] + a[3], b[1] + b[3]);\n  return Math.max(0, ix2 - ix) * Math.max(0, iy2 - iy);\n}\n\n/** Fraction of `inner`'s area covered by `outer` (containment ratio). Pure. */\nexport function coverageRatio(inner: SomBbox, outer: SomBbox): number {\n  const area = inner[2] * inner[3];\n  if (area <= 0) return 0;\n  return intersectionArea(inner, outer) / area;\n}\n\nfunction isValidBox(b: SomBbox): boolean {\n  return (\n    Number.isFinite(b[0]) &&\n    Number.isFinite(b[1]) &&\n    Number.isFinite(b[2]) &&\n    Number.isFinite(b[3]) &&\n    b[2] > 0 &&\n    b[3] > 0\n  );\n}\n\n/**\n * Fuse icon + text candidates into a deduplicated, 1-indexed set of marks.\n *\n * Pure and deterministic: same inputs → identical numbering, regardless of\n * input ordering. Degenerate boxes (non-finite / zero-area) are dropped.\n */\nexport function buildSetOfMarks(\n  candidates: readonly SomCandidate[],\n  options: SetOfMarksOptions = {},\n): SomMark[] {\n  const iconOverText =\n    options.iconOverTextCoverage ?? DEFAULT_ICON_OVER_TEXT_COVERAGE;\n  const nmsIou = options.nmsIouThreshold ?? DEFAULT_NMS_IOU;\n  const rowTolerance = options.rowTolerance ?? DEFAULT_ROW_TOLERANCE;\n  const minScore = options.minScore ?? 0;\n\n  const norm: NormCandidate[] = candidates\n    .filter((c) => isValidBox(c.bbox))\n    .map((c) => ({\n      bbox: [c.bbox[0], c.bbox[1], c.bbox[2], c.bbox[3]] as [\n        number,\n        number,\n        number,\n        number,\n      ],\n      source: c.source,\n      label: c.label,\n      score: c.score ?? DEFAULT_SCORE,\n    }))\n    .filter((c) => c.score >= minScore);\n\n  const icons = norm.filter((c) => c.source === \"icon\");\n  const texts = norm.filter((c) => c.source === \"text\");\n\n  // 1. Icon-over-text suppression: drop text mostly covered by some icon.\n  const keptTexts = texts.filter(\n    (t) =>\n      !icons.some((icon) => coverageRatio(t.bbox, icon.bbox) >= iconOverText),\n  );\n\n  // 2. NMS over the survivors. Sort so icons outrank text, then by score desc,\n  //    then by a stable positional key — the greedy keep is deterministic.\n  const pool = [...icons, ...keptTexts].sort((a, b) => {\n    if (a.source !== b.source) return a.source === \"icon\" ? -1 : 1;\n    if (b.score !== a.score) return b.score - a.score;\n    return a.bbox[1] - b.bbox[1] || a.bbox[0] - b.bbox[0];\n  });\n\n  const kept: NormCandidate[] = [];\n  for (const cand of pool) {\n    const overlaps = kept.some((k) => bboxIou(cand.bbox, k.bbox) >= nmsIou);\n    if (!overlaps) kept.push(cand);\n  }\n\n  // 3. Reading-order numbering: bucket into rows (within rowTolerance), then\n  //    order rows top-to-bottom and members left-to-right.\n  const ordered = [...kept].sort((a, b) => {\n    const sameRow = Math.abs(a.bbox[1] - b.bbox[1]) <= rowTolerance;\n    if (sameRow) return a.bbox[0] - b.bbox[0] || a.bbox[1] - b.bbox[1];\n    return a.bbox[1] - b.bbox[1];\n  });\n\n  return ordered.map((c, i): SomMark => {\n    const [x, y, w, h] = c.bbox;\n    const mark: SomMark = {\n      index: i + 1,\n      bbox: [x, y, w, h],\n      center: [Math.round(x + w / 2), Math.round(y + h / 2)],\n      source: c.source,\n      score: c.score,\n    };\n    if (c.label !== undefined) (mark as { label?: string }).label = c.label;\n    return mark;\n  });\n}\n\n/** XML-escape a label so it is safe to embed in the SVG overlay. */\nfunction escapeXml(s: string): string {\n  return s\n    .replace(/&/g, \"&amp;\")\n    .replace(/</g, \"&lt;\")\n    .replace(/>/g, \"&gt;\")\n    .replace(/\"/g, \"&quot;\")\n    .replace(/'/g, \"&apos;\");\n}\n\nexport interface SomOverlayOptions extends SetOfMarksOptions {\n  /** Stroke / badge color for icon marks. Default \"#FF8C00\" (orange). */\n  readonly iconColor?: string;\n  /** Stroke / badge color for text marks. Default \"#1FA9FF\". */\n  readonly textColor?: string;\n  /** Box stroke width in px. Default 2. */\n  readonly strokeWidth?: number;\n  /** Badge font size in px. Default 13. */\n  readonly fontSize?: number;\n}\n\n/**\n * Build the SVG overlay markup for a set of marks over a `width × height`\n * canvas. Pure (no I/O) — separated from the raster composite so it is\n * unit-testable and reusable by non-sharp consumers (e.g. a browser overlay).\n */\nexport function buildSetOfMarksSvg(\n  marks: readonly SomMark[],\n  width: number,\n  height: number,\n  options: SomOverlayOptions = {},\n): string {\n  const iconColor = options.iconColor ?? \"#FF8C00\";\n  const textColor = options.textColor ?? \"#1FA9FF\";\n  const strokeWidth = options.strokeWidth ?? 2;\n  const fontSize = options.fontSize ?? 13;\n  const badgeW = Math.round(fontSize * 1.4);\n  const badgeH = Math.round(fontSize * 1.3);\n\n  const parts: string[] = [];\n  for (const mark of marks) {\n    const [x, y, w, h] = mark.bbox;\n    const color = mark.source === \"icon\" ? iconColor : textColor;\n    const label = String(mark.index);\n    // Badge sits just inside the top-left of the box, clamped to the canvas.\n    const bx = Math.max(0, Math.min(x, width - badgeW));\n    const by = Math.max(0, Math.min(y, height - badgeH));\n    parts.push(\n      `<rect x=\"${x}\" y=\"${y}\" width=\"${w}\" height=\"${h}\" fill=\"none\" ` +\n        `stroke=\"${color}\" stroke-width=\"${strokeWidth}\" />`,\n      `<rect x=\"${bx}\" y=\"${by}\" width=\"${badgeW}\" height=\"${badgeH}\" ` +\n        `fill=\"${color}\" />`,\n      `<text x=\"${bx + badgeW / 2}\" y=\"${by + badgeH / 2}\" ` +\n        `font-family=\"monospace\" font-size=\"${fontSize}\" font-weight=\"bold\" ` +\n        `fill=\"#000000\" text-anchor=\"middle\" dominant-baseline=\"central\">` +\n        `${escapeXml(label)}</text>`,\n    );\n  }\n  return (\n    `<svg xmlns=\"http://www.w3.org/2000/svg\" width=\"${width}\" ` +\n    `height=\"${height}\">${parts.join(\"\")}</svg>`\n  );\n}\n\n// ── Source adapters ─────────────────────────────────────────────────────────\n//\n// Turn the two grounding sources we already have — GGUF YOLO icon detections\n// and `CoordOcrProvider` text blocks — into SoM candidates. Both are described\n// structurally so this module keeps no hard dependency on the detector or OCR\n// implementations (same no-hard-dep rule as `get-screen-elements.ts`).\n\n/** Structural shape of a `DetectedObject` (YOLO) — `{x,y,width,height}` box. */\nexport interface DetectedObjectLike {\n  readonly boundingBox: {\n    readonly x: number;\n    readonly y: number;\n    readonly width: number;\n    readonly height: number;\n  };\n  readonly type?: string;\n  readonly confidence?: number;\n}\n\n/** Structural shape of an OCR block (`OcrWithCoordsBlock`). */\nexport interface OcrBlockLike {\n  readonly text?: string;\n  readonly confidence?: number;\n  readonly bbox: {\n    readonly x: number;\n    readonly y: number;\n    readonly width: number;\n    readonly height: number;\n  };\n}\n\n/** Adapt GGUF YOLO detections into icon-source SoM candidates. */\nexport function somCandidatesFromDetections(\n  objects: readonly DetectedObjectLike[],\n): SomCandidate[] {\n  return objects.map((o) => {\n    const c: SomCandidate = {\n      bbox: [\n        o.boundingBox.x,\n        o.boundingBox.y,\n        o.boundingBox.width,\n        o.boundingBox.height,\n      ],\n      source: \"icon\",\n      score: o.confidence ?? DEFAULT_SCORE,\n    };\n    if (o.type !== undefined) (c as { label?: string }).label = o.type;\n    return c;\n  });\n}\n\n/** Adapt `CoordOcrProvider` text blocks into text-source SoM candidates. */\nexport function somCandidatesFromOcr(\n  blocks: readonly OcrBlockLike[],\n): SomCandidate[] {\n  return blocks.map((b) => {\n    const c: SomCandidate = {\n      bbox: [b.bbox.x, b.bbox.y, b.bbox.width, b.bbox.height],\n      source: \"text\",\n      score: b.confidence ?? DEFAULT_SCORE,\n    };\n    if (b.text !== undefined) (c as { label?: string }).label = b.text;\n    return c;\n  });\n}\n\n/**\n * Convenience: fuse YOLO detections + OCR blocks straight into a numbered mark\n * set. The seam `detect_elements`/grounding calls — pass whatever icon and text\n * boxes the scene already has.\n */\nexport function buildSceneSetOfMarks(\n  args: {\n    readonly detections?: readonly DetectedObjectLike[];\n    readonly ocrBlocks?: readonly OcrBlockLike[];\n  },\n  options?: SetOfMarksOptions,\n): SomMark[] {\n  return buildSetOfMarks(\n    [\n      ...somCandidatesFromDetections(args.detections ?? []),\n      ...somCandidatesFromOcr(args.ocrBlocks ?? []),\n    ],\n    options,\n  );\n}\n\n/**\n * Composite a numbered Set-of-Marks overlay onto a source PNG.\n *\n * Returns PNG bytes the same size as the input. `sharp` is loaded lazily so the\n * pure core (`buildSetOfMarks`) carries no native dependency for consumers that\n * only need coordinates.\n */\nexport async function renderSetOfMarksOverlay(\n  pngBytes: Uint8Array,\n  marks: readonly SomMark[],\n  options: SomOverlayOptions = {},\n): Promise<Buffer> {\n  const { default: sharp } = await import(\"sharp\");\n  const base = sharp(Buffer.from(pngBytes));\n  const meta = await base.metadata();\n  const width = meta.width ?? 0;\n  const height = meta.height ?? 0;\n  if (width <= 0 || height <= 0) {\n    throw new Error(\n      `[vision/som] cannot overlay marks: source image has no dimensions`,\n    );\n  }\n  const svg = buildSetOfMarksSvg(marks, width, height, options);\n  return base\n    .composite([{ input: Buffer.from(svg), top: 0, left: 0 }])\n    .png()\n    .toBuffer();\n}\n"
+  ],
+  "mappings": ";;;;;;;;;;;;;;;;;;;AAoFO,SAAS,OAAO,CAAC,GAAS,GAAiB;AAAA,EAChD,OAAO,IAAI,IAAI,IAAI,MAAM;AAAA,EACzB,OAAO,IAAI,IAAI,IAAI,MAAM;AAAA,EACzB,IAAI,MAAM,KAAK,MAAM,KAAK,MAAM,KAAK,MAAM;AAAA,IAAG,OAAO;AAAA,EACrD,MAAM,KAAK,KAAK,IAAI,IAAI,EAAE;AAAA,EAC1B,MAAM,KAAK,KAAK,IAAI,IAAI,EAAE;AAAA,EAC1B,MAAM,MAAM,KAAK,IAAI,KAAK,IAAI,KAAK,EAAE;AAAA,EACrC,MAAM,MAAM,KAAK,IAAI,KAAK,IAAI,KAAK,EAAE;AAAA,EACrC,MAAM,KAAK,KAAK,IAAI,GAAG,MAAM,EAAE;AAAA,EAC/B,MAAM,KAAK,KAAK,IAAI,GAAG,MAAM,EAAE;AAAA,EAC/B,MAAM,QAAQ,KAAK;AAAA,EACnB,IAAI,SAAS;AAAA,IAAG,OAAO;AAAA,EACvB,MAAM,QAAQ,KAAK,KAAK,KAAK,KAAK;AAAA,EAClC,OAAO,QAAQ,IAAI,QAAQ,QAAQ;AAAA;;;ACvBrC,IAAM,kCAAkC;AACxC,IAAM,kBAAkB;AACxB,IAAM,wBAAwB;AAC9B,IAAM,gBAAgB;AAUtB,SAAS,gBAAgB,CAAC,GAAY,GAAoB;AAAA,EACxD,MAAM,KAAK,KAAK,IAAI,EAAE,IAAI,EAAE,EAAE;AAAA,EAC9B,MAAM,KAAK,KAAK,IAAI,EAAE,IAAI,EAAE,EAAE;AAAA,EAC9B,MAAM,MAAM,KAAK,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE;AAAA,EAC7C,MAAM,MAAM,KAAK,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE;AAAA,EAC7C,OAAO,KAAK,IAAI,GAAG,MAAM,EAAE,IAAI,KAAK,IAAI,GAAG,MAAM,EAAE;AAAA;AAI9C,SAAS,aAAa,CAAC,OAAgB,OAAwB;AAAA,EACpE,MAAM,OAAO,MAAM,KAAK,MAAM;AAAA,EAC9B,IAAI,QAAQ;AAAA,IAAG,OAAO;AAAA,EACtB,OAAO,iBAAiB,OAAO,KAAK,IAAI;AAAA;AAG1C,SAAS,UAAU,CAAC,GAAqB;AAAA,EACvC,OACE,OAAO,SAAS,EAAE,EAAE,KACpB,OAAO,SAAS,EAAE,EAAE,KACpB,OAAO,SAAS,EAAE,EAAE,KACpB,OAAO,SAAS,EAAE,EAAE,KACpB,EAAE,KAAK,KACP,EAAE,KAAK;AAAA;AAUJ,SAAS,eAAe,CAC7B,YACA,UAA6B,CAAC,GACnB;AAAA,EACX,MAAM,eACJ,QAAQ,wBAAwB;AAAA,EAClC,MAAM,SAAS,QAAQ,mBAAmB;AAAA,EAC1C,MAAM,eAAe,QAAQ,gBAAgB;AAAA,EAC7C,MAAM,WAAW,QAAQ,YAAY;AAAA,EAErC,MAAM,OAAwB,WAC3B,OAAO,CAAC,MAAM,WAAW,EAAE,IAAI,CAAC,EAChC,IAAI,CAAC,OAAO;AAAA,IACX,MAAM,CAAC,EAAE,KAAK,IAAI,EAAE,KAAK,IAAI,EAAE,KAAK,IAAI,EAAE,KAAK,EAAE;AAAA,IAMjD,QAAQ,EAAE;AAAA,IACV,OAAO,EAAE;AAAA,IACT,OAAO,EAAE,SAAS;AAAA,EACpB,EAAE,EACD,OAAO,CAAC,MAAM,EAAE,SAAS,QAAQ;AAAA,EAEpC,MAAM,QAAQ,KAAK,OAAO,CAAC,MAAM,EAAE,WAAW,MAAM;AAAA,EACpD,MAAM,QAAQ,KAAK,OAAO,CAAC,MAAM,EAAE,WAAW,MAAM;AAAA,EAGpD,MAAM,YAAY,MAAM,OACtB,CAAC,MACC,CAAC,MAAM,KAAK,CAAC,SAAS,cAAc,EAAE,MAAM,KAAK,IAAI,KAAK,YAAY,CAC1E;AAAA,EAIA,MAAM,OAAO,CAAC,GAAG,OAAO,GAAG,SAAS,EAAE,KAAK,CAAC,GAAG,MAAM;AAAA,IACnD,IAAI,EAAE,WAAW,EAAE;AAAA,MAAQ,OAAO,EAAE,WAAW,SAAS,KAAK;AAAA,IAC7D,IAAI,EAAE,UAAU,EAAE;AAAA,MAAO,OAAO,EAAE,QAAQ,EAAE;AAAA,IAC5C,OAAO,EAAE,KAAK,KAAK,EAAE,KAAK,MAAM,EAAE,KAAK,KAAK,EAAE,KAAK;AAAA,GACpD;AAAA,EAED,MAAM,OAAwB,CAAC;AAAA,EAC/B,WAAW,QAAQ,MAAM;AAAA,IACvB,MAAM,WAAW,KAAK,KAAK,CAAC,MAAM,QAAQ,KAAK,MAAM,EAAE,IAAI,KAAK,MAAM;AAAA,IACtE,IAAI,CAAC;AAAA,MAAU,KAAK,KAAK,IAAI;AAAA,EAC/B;AAAA,EAIA,MAAM,UAAU,CAAC,GAAG,IAAI,EAAE,KAAK,CAAC,GAAG,MAAM;AAAA,IACvC,MAAM,UAAU,KAAK,IAAI,EAAE,KAAK,KAAK,EAAE,KAAK,EAAE,KAAK;AAAA,IACnD,IAAI;AAAA,MAAS,OAAO,EAAE,KAAK,KAAK,EAAE,KAAK,MAAM,EAAE,KAAK,KAAK,EAAE,KAAK;AAAA,IAChE,OAAO,EAAE,KAAK,KAAK,EAAE,KAAK;AAAA,GAC3B;AAAA,EAED,OAAO,QAAQ,IAAI,CAAC,GAAG,MAAe;AAAA,IACpC,OAAO,GAAG,GAAG,GAAG,KAAK,EAAE;AAAA,IACvB,MAAM,OAAgB;AAAA,MACpB,OAAO,IAAI;AAAA,MACX,MAAM,CAAC,GAAG,GAAG,GAAG,CAAC;AAAA,MACjB,QAAQ,CAAC,KAAK,MAAM,IAAI,IAAI,CAAC,GAAG,KAAK,MAAM,IAAI,IAAI,CAAC,CAAC;AAAA,MACrD,QAAQ,EAAE;AAAA,MACV,OAAO,EAAE;AAAA,IACX;AAAA,IACA,IAAI,EAAE,UAAU;AAAA,MAAY,KAA4B,QAAQ,EAAE;AAAA,IAClE,OAAO;AAAA,GACR;AAAA;AAIH,SAAS,SAAS,CAAC,GAAmB;AAAA,EACpC,OAAO,EACJ,QAAQ,MAAM,OAAO,EACrB,QAAQ,MAAM,MAAM,EACpB,QAAQ,MAAM,MAAM,EACpB,QAAQ,MAAM,QAAQ,EACtB,QAAQ,MAAM,QAAQ;AAAA;AAmBpB,SAAS,kBAAkB,CAChC,OACA,OACA,QACA,UAA6B,CAAC,GACtB;AAAA,EACR,MAAM,YAAY,QAAQ,aAAa;AAAA,EACvC,MAAM,YAAY,QAAQ,aAAa;AAAA,EACvC,MAAM,cAAc,QAAQ,eAAe;AAAA,EAC3C,MAAM,WAAW,QAAQ,YAAY;AAAA,EACrC,MAAM,SAAS,KAAK,MAAM,WAAW,GAAG;AAAA,EACxC,MAAM,SAAS,KAAK,MAAM,WAAW,GAAG;AAAA,EAExC,MAAM,QAAkB,CAAC;AAAA,EACzB,WAAW,QAAQ,OAAO;AAAA,IACxB,OAAO,GAAG,GAAG,GAAG,KAAK,KAAK;AAAA,IAC1B,MAAM,QAAQ,KAAK,WAAW,SAAS,YAAY;AAAA,IACnD,MAAM,QAAQ,OAAO,KAAK,KAAK;AAAA,IAE/B,MAAM,KAAK,KAAK,IAAI,GAAG,KAAK,IAAI,GAAG,QAAQ,MAAM,CAAC;AAAA,IAClD,MAAM,KAAK,KAAK,IAAI,GAAG,KAAK,IAAI,GAAG,SAAS,MAAM,CAAC;AAAA,IACnD,MAAM,KACJ,YAAY,SAAS,aAAa,cAAc,oBAC9C,WAAW,wBAAwB,mBACrC,YAAY,UAAU,cAAc,mBAAmB,aACrD,SAAS,aACX,YAAY,KAAK,SAAS,SAAS,KAAK,SAAS,QAC/C,sCAAsC,kCACtC,qEACA,GAAG,UAAU,KAAK,UACtB;AAAA,EACF;AAAA,EACA,OACE,kDAAkD,YAClD,WAAW,WAAW,MAAM,KAAK,EAAE;AAAA;AAoChC,SAAS,2BAA2B,CACzC,SACgB;AAAA,EAChB,OAAO,QAAQ,IAAI,CAAC,MAAM;AAAA,IACxB,MAAM,IAAkB;AAAA,MACtB,MAAM;AAAA,QACJ,EAAE,YAAY;AAAA,QACd,EAAE,YAAY;AAAA,QACd,EAAE,YAAY;AAAA,QACd,EAAE,YAAY;AAAA,MAChB;AAAA,MACA,QAAQ;AAAA,MACR,OAAO,EAAE,cAAc;AAAA,IACzB;AAAA,IACA,IAAI,EAAE,SAAS;AAAA,MAAY,EAAyB,QAAQ,EAAE;AAAA,IAC9D,OAAO;AAAA,GACR;AAAA;AAII,SAAS,oBAAoB,CAClC,QACgB;AAAA,EAChB,OAAO,OAAO,IAAI,CAAC,MAAM;AAAA,IACvB,MAAM,IAAkB;AAAA,MACtB,MAAM,CAAC,EAAE,KAAK,GAAG,EAAE,KAAK,GAAG,EAAE,KAAK,OAAO,EAAE,KAAK,MAAM;AAAA,MACtD,QAAQ;AAAA,MACR,OAAO,EAAE,cAAc;AAAA,IACzB;AAAA,IACA,IAAI,EAAE,SAAS;AAAA,MAAY,EAAyB,QAAQ,EAAE;AAAA,IAC9D,OAAO;AAAA,GACR;AAAA;AAQI,SAAS,oBAAoB,CAClC,MAIA,SACW;AAAA,EACX,OAAO,gBACL;AAAA,IACE,GAAG,4BAA4B,KAAK,cAAc,CAAC,CAAC;AAAA,IACpD,GAAG,qBAAqB,KAAK,aAAa,CAAC,CAAC;AAAA,EAC9C,GACA,OACF;AAAA;AAUF,eAAsB,uBAAuB,CAC3C,UACA,OACA,UAA6B,CAAC,GACb;AAAA,EACjB,QAAQ,SAAS,UAAU,MAAa;AAAA,EACxC,MAAM,OAAO,MAAM,OAAO,KAAK,QAAQ,CAAC;AAAA,EACxC,MAAM,OAAO,MAAM,KAAK,SAAS;AAAA,EACjC,MAAM,QAAQ,KAAK,SAAS;AAAA,EAC5B,MAAM,SAAS,KAAK,UAAU;AAAA,EAC9B,IAAI,SAAS,KAAK,UAAU,GAAG;AAAA,IAC7B,MAAM,IAAI,MACR,mEACF;AAAA,EACF;AAAA,EACA,MAAM,MAAM,mBAAmB,OAAO,OAAO,QAAQ,OAAO;AAAA,EAC5D,OAAO,KACJ,UAAU,CAAC,EAAE,OAAO,OAAO,KAAK,GAAG,GAAG,KAAK,GAAG,MAAM,EAAE,CAAC,CAAC,EACxD,IAAI,EACJ,SAAS;AAAA;",
+  "debugId": "55C971527120024164756E2164756E21",
+  "names": []
+}

package/dist/test-input.d.ts

@@ -0,0 +1,25 @@
+/**
+ * Test-input shim for plugin-vision.
+ *
+ * Lets tests inject a fixture image into the vision analysis pipeline
+ * without monkey-patching platform capture code.
+ *
+ * Selection: read `ELIZA_VISION_TEST_INPUT`:
+ *   - "image"  → return the fixture bytes (resolved via
+ *     `ELIZA_VISION_TEST_FIXTURE` env var, default the bundled
+ *     test/fixtures/sample-scene.png path).
+ *   - "camera" / "screen" / unset → return null, signaling that the caller
+ *     should use the existing platform capture path.
+ *
+ * Single-purpose helper: anything that wants test-injected pixels reads
+ * `getTestImage()`. When it returns a Buffer, use those bytes. When it
+ * returns null, fall through to live capture.
+ */
+export type TestInputMode = "image" | "camera" | "screen" | "unset";
+export declare function getTestInputMode(): TestInputMode;
+/**
+ * If `ELIZA_VISION_TEST_INPUT=image`, return fixture bytes (PNG buffer);
+ * otherwise return null. Result is cached per-fixture-path for the process.
+ */
+export declare function getTestImage(): Buffer | null;
+//# sourceMappingURL=test-input.d.ts.map

package/dist/test-input.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"test-input.d.ts","sourceRoot":"","sources":["../src/test-input.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAMH,MAAM,MAAM,aAAa,GAAG,OAAO,GAAG,QAAQ,GAAG,QAAQ,GAAG,OAAO,CAAC;AAEpE,wBAAgB,gBAAgB,IAAI,aAAa,CAMhD;AAcD;;;GAGG;AACH,wBAAgB,YAAY,IAAI,MAAM,GAAG,IAAI,CAa5C"}

package/dist/types.d.ts

@@ -0,0 +1,241 @@
+import type { DescribePauseReason } from "./describe-backpressure";
+export declare const VisionServiceType: {
+    VISION: "VISION";
+};
+declare module "@elizaos/core" {
+    interface ServiceTypeRegistry {
+        VISION: "VISION";
+    }
+}
+export interface CameraInfo {
+    id: string;
+    name: string;
+    connected: boolean;
+}
+export interface SceneDescription {
+    /** Freshest processed frame timestamp for object/person/change signals. */
+    timestamp: number;
+    /** Timestamp of the VLM prose in `description`; may be older than frame data. */
+    descriptionTimestamp?: number;
+    description: string;
+    objects: DetectedObject[];
+    people: PersonInfo[];
+    sceneChanged: boolean;
+    changePercentage: number;
+    /** True when the VLM prose was reused after a describe skip. */
+    descriptionStale?: boolean;
+    /** True when the VLM describe step is currently paused by backpressure. */
+    describePaused?: boolean;
+    describePauseReason?: Exclude<DescribePauseReason, null>;
+    audioTranscription?: string;
+}
+export interface DetectedObject {
+    id: string;
+    type: string;
+    confidence: number;
+    boundingBox: BoundingBox;
+}
+export interface PersonInfo {
+    id: string;
+    pose: "sitting" | "standing" | "lying" | "unknown";
+    facing: "camera" | "away" | "left" | "right" | "unknown";
+    confidence: number;
+    boundingBox: BoundingBox;
+    keypoints?: Array<{
+        part: string;
+        position: {
+            x: number;
+            y: number;
+        };
+        score: number;
+    }>;
+}
+export interface BoundingBox {
+    x: number;
+    y: number;
+    width: number;
+    height: number;
+}
+export interface VisionFrame {
+    timestamp: number;
+    width: number;
+    height: number;
+    data: Buffer;
+    format: "rgb" | "rgba" | "jpeg" | "png";
+}
+export declare enum VisionMode {
+    OFF = "OFF",
+    CAMERA = "CAMERA",
+    SCREEN = "SCREEN",
+    BOTH = "BOTH"
+}
+export interface ScreenCapture {
+    timestamp: number;
+    width: number;
+    height: number;
+    data: Buffer;
+    tiles: ScreenTile[];
+}
+export interface ScreenTile {
+    id: string;
+    row: number;
+    col: number;
+    /** Tile origin X within the source capture (display-local pixels). */
+    x: number;
+    /** Tile origin Y within the source capture (display-local pixels). */
+    y: number;
+    width: number;
+    height: number;
+    data?: Buffer;
+    analysis?: TileAnalysis;
+    /**
+     * Source display id, when the tile came from a per-display capture pass.
+     * Stringified so opaque platform ids (CGDirectDisplayID, sway output names)
+     * round-trip without lossy coercion.
+     */
+    displayId?: string;
+    /**
+     * Absolute pixel X of the tile origin in the source display's native space.
+     * Same value as `x` when `displayId` refers to a single-display capture; it
+     * becomes load-bearing once the capture pipeline composes per-display
+     * screenshots into a multi-monitor stream.
+     */
+    sourceX?: number;
+    /** Absolute pixel Y of the tile origin in the source display's native space. */
+    sourceY?: number;
+}
+export interface TileAnalysis {
+    timestamp: number;
+    ocr?: OCRResult;
+    objects?: DetectedObject[];
+    text?: string;
+    summary?: string;
+}
+export interface OCRResult {
+    text: string;
+    blocks: Array<{
+        text: string;
+        bbox: BoundingBox;
+        confidence: number;
+        words?: Array<{
+            text: string;
+            bbox: BoundingBox;
+            confidence: number;
+        }>;
+    }>;
+    fullText: string;
+}
+export interface EnhancedSceneDescription extends SceneDescription {
+    screenCapture?: ScreenCapture;
+    screenAnalysis?: {
+        fullScreenOCR?: string;
+        activeTile?: TileAnalysis;
+        gridSummary?: string;
+        focusedApp?: string;
+        uiElements?: Array<{
+            type: string;
+            text: string;
+            position: BoundingBox;
+        }>;
+    };
+}
+export interface VisionConfig {
+    cameraName?: string;
+    pixelChangeThreshold?: number;
+    updateInterval?: number;
+    enablePoseDetection?: boolean;
+    enableObjectDetection?: boolean;
+    tfUpdateInterval?: number;
+    vlmUpdateInterval?: number;
+    tfChangeThreshold?: number;
+    vlmChangeThreshold?: number;
+    visionMode?: VisionMode;
+    screenCaptureInterval?: number;
+    tileSize?: number;
+    tileProcessingOrder?: "sequential" | "priority" | "random";
+    ocrEnabled?: boolean;
+    screenRegion?: {
+        x: number;
+        y: number;
+        width: number;
+        height: number;
+    };
+    displayIndex?: number;
+    captureAllDisplays?: boolean;
+    targetScreenFPS?: number;
+    textRegions?: Array<{
+        x: number;
+        y: number;
+        width: number;
+        height: number;
+    }>;
+}
+export interface TrackedEntity {
+    id: string;
+    entityType: "person" | "object" | "pet";
+    firstSeen: number;
+    lastSeen: number;
+    lastPosition: BoundingBox;
+    appearances: EntityAppearance[];
+    attributes: EntityAttributes;
+    worldId?: string;
+    roomId?: string;
+}
+interface EntityAppearance {
+    timestamp: number;
+    boundingBox: BoundingBox;
+    confidence: number;
+    embedding?: number[];
+    keypoints?: Array<{
+        part: string;
+        position: {
+            x: number;
+            y: number;
+        };
+        score: number;
+    }>;
+}
+export interface EntityAttributes {
+    [key: string]: string | number | boolean | null | undefined | string[] | number[];
+    name?: string;
+    faceEmbedding?: number[];
+    faceId?: string;
+    clothing?: string[];
+    hairColor?: string;
+    accessories?: string[];
+    objectType?: string;
+    color?: string;
+    size?: "small" | "medium" | "large";
+    description?: string;
+    tags?: string[];
+}
+export interface FaceLibrary {
+    faces: Map<string, FaceProfile>;
+    embeddings: Map<string, number[][]>;
+}
+export interface FaceProfile {
+    id: string;
+    name?: string;
+    embeddings: number[][];
+    firstSeen: number;
+    lastSeen: number;
+    seenCount: number;
+    attributes?: {
+        age?: string;
+        gender?: string;
+        emotion?: string;
+    };
+}
+export interface WorldState {
+    worldId: string;
+    entities: Map<string, TrackedEntity>;
+    lastUpdate: number;
+    activeEntities: string[];
+    recentlyLeft: Array<{
+        entityId: string;
+        leftAt: number;
+        lastPosition: BoundingBox;
+    }>;
+}
+export {};
+//# sourceMappingURL=types.d.ts.map