@elizaos/plugin-vision 2.0.0-alpha.9 → 2.0.3-beta.5

package/dist/entity-tracker.d.ts

@@ -0,0 +1,33 @@
+import { type IAgentRuntime } from "@elizaos/core";
+import type { DetectedObject, PersonInfo, TrackedEntity, WorldState } from "./types";
+export declare class EntityTracker {
+    private worldState;
+    private readonly POSITION_THRESHOLD;
+    private readonly MISSING_THRESHOLD;
+    private readonly CLEANUP_THRESHOLD;
+    constructor(worldId: string);
+    updateEntities(detectedObjects: DetectedObject[], people: PersonInfo[], faceProfiles?: Map<string, string>, // Maps person ID to face profile ID
+    runtime?: IAgentRuntime): Promise<TrackedEntity[]>;
+    private trackPerson;
+    private trackObject;
+    private findMatchingEntity;
+    private calculateDistance;
+    private updateWorldState;
+    private syncWithRuntime;
+    getWorldState(): WorldState;
+    getActiveEntities(): TrackedEntity[];
+    getEntity(entityId: string): TrackedEntity | undefined;
+    getRecentlyLeft(): Array<{
+        entity: TrackedEntity;
+        leftAt: number;
+    }>;
+    assignNameToEntity(entityId: string, name: string): boolean;
+    getStatistics(): {
+        totalEntities: number;
+        activeEntities: number;
+        recentlyLeft: number;
+        people: number;
+        objects: number;
+    };
+}
+//# sourceMappingURL=entity-tracker.d.ts.map

package/dist/entity-tracker.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"entity-tracker.d.ts","sourceRoot":"","sources":["../src/entity-tracker.ts"],"names":[],"mappings":"AAAA,OAAO,EAAoB,KAAK,aAAa,EAAU,MAAM,eAAe,CAAC;AAC7E,OAAO,KAAK,EAEV,cAAc,EACd,UAAU,EACV,aAAa,EACb,UAAU,EACX,MAAM,SAAS,CAAC;AAEjB,qBAAa,aAAa;IACxB,OAAO,CAAC,UAAU,CAAa;IAC/B,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAAO;IAC1C,OAAO,CAAC,QAAQ,CAAC,iBAAiB,CAAQ;IAC1C,OAAO,CAAC,QAAQ,CAAC,iBAAiB,CAAS;gBAE/B,OAAO,EAAE,MAAM;IAUrB,cAAc,CAClB,eAAe,EAAE,cAAc,EAAE,EACjC,MAAM,EAAE,UAAU,EAAE,EACpB,YAAY,CAAC,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,oCAAoC;IACxE,OAAO,CAAC,EAAE,aAAa,GACtB,OAAO,CAAC,aAAa,EAAE,CAAC;YAoCb,WAAW;YA8DX,WAAW;IAsDzB,OAAO,CAAC,kBAAkB;IA2C1B,OAAO,CAAC,iBAAiB;IAezB,OAAO,CAAC,gBAAgB;YAuCV,eAAe;IAqD7B,aAAa,IAAI,UAAU;IAI3B,iBAAiB,IAAI,aAAa,EAAE;IAMpC,SAAS,CAAC,QAAQ,EAAE,MAAM,GAAG,aAAa,GAAG,SAAS;IAItD,eAAe,IAAI,KAAK,CAAC;QAAE,MAAM,EAAE,aAAa,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,CAAC;IAanE,kBAAkB,CAAC,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,OAAO;IAa3D,aAAa,IAAI;QACf,aAAa,EAAE,MAAM,CAAC;QACtB,cAAc,EAAE,MAAM,CAAC;QACvB,YAAY,EAAE,MAAM,CAAC;QACrB,MAAM,EAAE,MAAM,CAAC;QACf,OAAO,EAAE,MAAM,CAAC;KACjB;CAUF"}

package/dist/face-detector-ggml.d.ts

@@ -0,0 +1,60 @@
+import type { BoundingBox } from "./types";
+/**
+ * Same shape the removed ONNX detector exported. Kept identical so callers can
+ * select the native backend without reshaping results.
+ */
+export interface MediaPipeFaceConfig {
+    modelUrl?: string;
+    modelSha256?: string | null;
+    modelDir?: string;
+    scoreThreshold?: number;
+    trusted?: boolean;
+}
+export interface MediaPipeFaceDetection {
+    bbox: BoundingBox;
+    confidence: number;
+    /**
+     * BlazeFace's 6 keypoints in canonical order:
+     *   0: left eye   1: right eye   2: nose tip
+     *   3: mouth      4: left ear    5: right ear
+     * Coordinates are in source-image absolute pixels.
+     */
+    keypoints?: Array<{
+        x: number;
+        y: number;
+    }>;
+}
+/**
+ * ggml-backed BlazeFace face detector. Mirrors the
+ * `MediaPipeFaceDetector` compatibility surface — same constructor config,
+ * same `MediaPipeFaceDetection` output shape.
+ *
+ * Currently disabled (`isAvailable()` returns `false`) until the
+ * face-cpp model entries gain runtime implementations and a BlazeFace GGUF
+ * artifact lands.
+ */
+export declare class BlazeFaceGgmlDetector {
+    private readonly cfg;
+    private bindings;
+    private handle;
+    private initialized;
+    private initPromise;
+    constructor(config?: MediaPipeFaceConfig);
+    /**
+     * `true` only when both the native library AND the GGUF weights are
+     * on disk. Loading them happens lazily in `initialize()`.
+     */
+    static isAvailable(): Promise<boolean>;
+    isInitialized(): boolean;
+    initialize(): Promise<void>;
+    private _initialize;
+    /**
+     * Detect faces in the given image buffer. The buffer can be any
+     * sharp-supported format (PNG, JPEG, raw); we resize/letterbox to
+     * the BlazeFace 128x128 input via sharp, run the native detector,
+     * then return source-pixel bboxes + 6 keypoints.
+     */
+    detect(imageBuffer: Buffer): Promise<MediaPipeFaceDetection[]>;
+    dispose(): Promise<void>;
+}
+//# sourceMappingURL=face-detector-ggml.d.ts.map

package/dist/face-detector-ggml.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"face-detector-ggml.d.ts","sourceRoot":"","sources":["../src/face-detector-ggml.ts"],"names":[],"mappings":"AAwBA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,SAAS,CAAC;AAO3C;;;GAGG;AACH,MAAM,WAAW,mBAAmB;IAClC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,OAAO,CAAC,EAAE,OAAO,CAAC;CACnB;AAED,MAAM,WAAW,sBAAsB;IACrC,IAAI,EAAE,WAAW,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB;;;;;OAKG;IACH,SAAS,CAAC,EAAE,KAAK,CAAC;QAAE,CAAC,EAAE,MAAM,CAAC;QAAC,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;CAC7C;AA4MD;;;;;;;;GAQG;AACH,qBAAa,qBAAqB;IAChC,OAAO,CAAC,QAAQ,CAAC,GAAG,CAGlB;IACF,OAAO,CAAC,QAAQ,CAAmC;IACnD,OAAO,CAAC,MAAM,CAAiB;IAC/B,OAAO,CAAC,WAAW,CAAS;IAC5B,OAAO,CAAC,WAAW,CAA8B;gBAErC,MAAM,GAAE,mBAAwB;IAQ5C;;;OAGG;WACU,WAAW,IAAI,OAAO,CAAC,OAAO,CAAC;IAgB5C,aAAa,IAAI,OAAO;IAIlB,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;YAOnB,WAAW;IAsBzB;;;;;OAKG;IACG,MAAM,CAAC,WAAW,EAAE,MAAM,GAAG,OAAO,CAAC,sBAAsB,EAAE,CAAC;IAsD9D,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;CAS/B"}

package/dist/face-detector-mediapipe.d.ts

@@ -0,0 +1,25 @@
+import type { BoundingBox } from "./types";
+export interface MediaPipeFaceConfig {
+    modelUrl?: string;
+    modelSha256?: string | null;
+    modelDir?: string;
+    scoreThreshold?: number;
+    trusted?: boolean;
+}
+export interface MediaPipeFaceDetection {
+    bbox: BoundingBox;
+    confidence: number;
+    keypoints?: Array<{
+        x: number;
+        y: number;
+    }>;
+}
+export declare class MediaPipeFaceDetector {
+    constructor(_config?: MediaPipeFaceConfig);
+    static isAvailable(): Promise<boolean>;
+    isInitialized(): boolean;
+    initialize(): Promise<void>;
+    detect(_imageBuffer: Buffer): Promise<MediaPipeFaceDetection[]>;
+    dispose(): Promise<void>;
+}
+//# sourceMappingURL=face-detector-mediapipe.d.ts.map

package/dist/face-detector-mediapipe.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"face-detector-mediapipe.d.ts","sourceRoot":"","sources":["../src/face-detector-mediapipe.ts"],"names":[],"mappings":"AAWA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,SAAS,CAAC;AAE3C,MAAM,WAAW,mBAAmB;IAClC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,OAAO,CAAC,EAAE,OAAO,CAAC;CACnB;AAED,MAAM,WAAW,sBAAsB;IACrC,IAAI,EAAE,WAAW,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,SAAS,CAAC,EAAE,KAAK,CAAC;QAAE,CAAC,EAAE,MAAM,CAAC;QAAC,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;CAC7C;AAED,qBAAa,qBAAqB;gBAEpB,OAAO,GAAE,mBAAwB;WAIhC,WAAW,IAAI,OAAO,CAAC,OAAO,CAAC;IAI5C,aAAa,IAAI,OAAO;IAIlB,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;IAM3B,MAAM,CAAC,YAAY,EAAE,MAAM,GAAG,OAAO,CAAC,sBAAsB,EAAE,CAAC;IAM/D,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;CAG/B"}

package/dist/face-recognition-ggml.d.ts

@@ -0,0 +1,94 @@
+import { type MediaPipeFaceDetection } from "./face-detector-ggml";
+import type { BoundingBox, FaceProfile } from "./types";
+/**
+ * Configuration for the ggml-backed face embedder.
+ */
+export interface FaceEmbedGgmlConfig {
+    modelPath?: string;
+    modelDir?: string;
+}
+/**
+ * Cosine distance between two 128-d unit-norm embeddings. Matches
+ * `face_embed_distance` in the C library: 0 for identical, 1 for
+ * orthogonal, 2 for antipodal.
+ */
+export declare function cosineDistance(a: Float32Array, b: Float32Array): number;
+/**
+ * L2 distance between two 128-d embeddings. For unit-norm inputs this
+ * is sqrt(2 - 2*dot(a, b)), in [0, 2].
+ */
+export declare function l2Distance(a: Float32Array, b: Float32Array): number;
+/**
+ * ggml-backed 128-d face embedder: a 128-d L2-normalized descriptor per
+ * detected face, consumed by the `FaceRecognition` class below.
+ */
+export declare class FaceEmbedGgmlRecognizer {
+    private readonly cfg;
+    private bindings;
+    private handle;
+    private initialized;
+    private initPromise;
+    constructor(config?: FaceEmbedGgmlConfig);
+    /**
+     * `true` only when both the native library AND the GGUF weights are
+     * on disk.
+     */
+    static isAvailable(): Promise<boolean>;
+    isInitialized(): boolean;
+    initialize(): Promise<void>;
+    private _initialize;
+    /**
+     * Compute a 128-d L2-normalized face embedding from an RGB(A) image
+     * buffer plus a detection record (bbox + BlazeFace landmarks).
+     *
+     * The image is decoded via sharp; pass any sharp-supported format
+     * (PNG, JPEG, raw). `detection` should come from
+     * `BlazeFaceGgmlDetector` so the keypoints already match the
+     * BlazeFace order.
+     */
+    embed(imageBuffer: Buffer, detection: MediaPipeFaceDetection): Promise<Float32Array>;
+    dispose(): Promise<void>;
+}
+/**
+ * A detected face: a native BlazeFace detection plus its 128-d ggml
+ * embedding. Mirrors the fields `VisionService` reads off each result.
+ * The native backend produces no expression / age-gender estimates, so
+ * those attributes are left to higher layers.
+ */
+export interface DetectedFace {
+    detection: {
+        box: BoundingBox;
+    };
+    descriptor: Float32Array;
+}
+/**
+ * Native ggml face recognition: BlazeFace detection + 128-d embedding +
+ * in-memory matching and persistence. When the native `libface` library
+ * or its GGUF weights are not on disk, detection returns an empty list
+ * (recognition is disabled, never faked). Matching and storage are pure
+ * JS and always available.
+ */
+export declare class FaceRecognition {
+    private readonly detector;
+    private readonly embedder;
+    private detectorAvailable;
+    private readonly faceLibrary;
+    private readonly FACE_MATCH_THRESHOLD;
+    private readonly MIN_FACE_SIZE;
+    /**
+     * Detect faces in a raw RGBA frame and compute an embedding for each.
+     * Returns an empty list when the native face backend is unavailable.
+     */
+    detectFaces(imageData: Buffer, width: number, height: number): Promise<DetectedFace[]>;
+    recognizeFace(descriptor: Float32Array): Promise<{
+        profileId: string;
+        distance: number;
+    } | null>;
+    addOrUpdateFace(descriptor: Float32Array, attributes?: Partial<FaceProfile>): Promise<string>;
+    getFaceProfile(profileId: string): FaceProfile | undefined;
+    getAllProfiles(): FaceProfile[];
+    saveFaceLibrary(filePath: string): Promise<void>;
+    loadFaceLibrary(filePath: string): Promise<void>;
+    dispose(): Promise<void>;
+}
+//# sourceMappingURL=face-recognition-ggml.d.ts.map

package/dist/face-recognition-ggml.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"face-recognition-ggml.d.ts","sourceRoot":"","sources":["../src/face-recognition-ggml.ts"],"names":[],"mappings":"AAyBA,OAAO,EAEL,KAAK,sBAAsB,EAC5B,MAAM,sBAAsB,CAAC;AAE9B,OAAO,KAAK,EAAE,WAAW,EAAe,WAAW,EAAE,MAAM,SAAS,CAAC;AAkMrE;;GAEG;AACH,MAAM,WAAW,mBAAmB;IAClC,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED;;;;GAIG;AACH,wBAAgB,cAAc,CAAC,CAAC,EAAE,YAAY,EAAE,CAAC,EAAE,YAAY,GAAG,MAAM,CASvE;AAED;;;GAGG;AACH,wBAAgB,UAAU,CAAC,CAAC,EAAE,YAAY,EAAE,CAAC,EAAE,YAAY,GAAG,MAAM,CAUnE;AAED;;;GAGG;AACH,qBAAa,uBAAuB;IAClC,OAAO,CAAC,QAAQ,CAAC,GAAG,CAA6C;IACjE,OAAO,CAAC,QAAQ,CAAkC;IAClD,OAAO,CAAC,MAAM,CAAiB;IAC/B,OAAO,CAAC,WAAW,CAAS;IAC5B,OAAO,CAAC,WAAW,CAA8B;gBAErC,MAAM,GAAE,mBAAwB;IAO5C;;;OAGG;WACU,WAAW,IAAI,OAAO,CAAC,OAAO,CAAC;IAgB5C,aAAa,IAAI,OAAO;IAIlB,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;YAOnB,WAAW;IAoBzB;;;;;;;;OAQG;IACG,KAAK,CACT,WAAW,EAAE,MAAM,EACnB,SAAS,EAAE,sBAAsB,GAChC,OAAO,CAAC,YAAY,CAAC;IAsBlB,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;CAS/B;AAID;;;;;GAKG;AACH,MAAM,WAAW,YAAY;IAC3B,SAAS,EAAE;QAAE,GAAG,EAAE,WAAW,CAAA;KAAE,CAAC;IAChC,UAAU,EAAE,YAAY,CAAC;CAC1B;AAED;;;;;;GAMG;AACH,qBAAa,eAAe;IAC1B,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAA+B;IACxD,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAiC;IAC1D,OAAO,CAAC,iBAAiB,CAAwB;IACjD,OAAO,CAAC,QAAQ,CAAC,WAAW,CAG1B;IAGF,OAAO,CAAC,QAAQ,CAAC,oBAAoB,CAAO;IAE5C,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAM;IAEpC;;;OAGG;IACG,WAAW,CACf,SAAS,EAAE,MAAM,EACjB,KAAK,EAAE,MAAM,EACb,MAAM,EAAE,MAAM,GACb,OAAO,CAAC,YAAY,EAAE,CAAC;IA+CpB,aAAa,CACjB,UAAU,EAAE,YAAY,GACvB,OAAO,CAAC;QAAE,SAAS,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI,CAAC;IAiBpD,eAAe,CACnB,UAAU,EAAE,YAAY,EACxB,UAAU,CAAC,EAAE,OAAO,CAAC,WAAW,CAAC,GAChC,OAAO,CAAC,MAAM,CAAC;IAgDlB,cAAc,CAAC,SAAS,EAAE,MAAM,GAAG,WAAW,GAAG,SAAS;IAI1D,cAAc,IAAI,WAAW,EAAE;IAIzB,eAAe,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAShD,eAAe,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAahD,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;CAI/B"}

package/dist/get-screen-elements.d.ts

@@ -0,0 +1,90 @@
+/**
+ * Pure element-merge core for the GET_SCREEN action (#9105 Slice 2 / M2).
+ *
+ * GET_SCREEN returns a cheap, token-frugal list of grounded, clickable screen
+ * elements unified from three sources: OCR text boxes, accessibility (AX)
+ * clickables, and (optionally) VLM-detected elements. This module is the
+ * deterministic heart of that envelope — it collapses the three sources into a
+ * single deduplicated, stably-ordered element list, recording each element's
+ * `groundingSources` provenance.
+ *
+ * Like the M1 OCR bridge (`computeruse-ocr-bridge.ts`), this is intentionally
+ * dependency-free and pure: the source types live in `@elizaos/plugin-computeruse`
+ * (`SceneOcrBox` / `SceneAxNode` / `SceneVlmElement`), but we describe their
+ * shapes STRUCTURALLY here rather than importing them, to keep the no-hard-dep
+ * rule. That also makes the merge engine fully unit-testable with zero
+ * environment, decoupled from the runtime/native/model wiring (Slice 3).
+ */
+/** Display-local bounding box `[x, y, w, h]`. */
+export type Bbox = readonly [number, number, number, number];
+/** Structural shape of computeruse's `SceneOcrBox`. */
+export interface OcrBoxLike {
+    readonly id: string;
+    readonly text: string;
+    readonly bbox: Bbox;
+    readonly conf?: number;
+    readonly displayId: number;
+}
+/** Structural shape of computeruse's `SceneAxNode`. */
+export interface AxNodeLike {
+    readonly id: string;
+    readonly role: string;
+    readonly label?: string;
+    readonly bbox: Bbox;
+    readonly actions?: readonly string[];
+    readonly displayId: number;
+}
+/** Structural shape of computeruse's `SceneVlmElement`. */
+export interface VlmElementLike {
+    readonly id: string;
+    readonly kind: string;
+    readonly desc: string;
+    readonly bbox: Bbox;
+    readonly displayId: number;
+}
+export type GroundingSource = "ocr" | "ax" | "vlm";
+/** A single unified, grounded screen element in the GET_SCREEN envelope. */
+export interface GetScreenElement {
+    /** Stable id, preferring the AX id, then OCR, then VLM. */
+    id: string;
+    /** Display-local bbox `[x, y, w, h]` of the representative (highest-priority) source. */
+    bbox: [number, number, number, number];
+    /** User-facing text/label: AX label, else OCR text, else VLM description. */
+    text: string;
+    /** Element kind/role when known: AX role, else VLM kind. */
+    kind?: string;
+    displayId: number;
+    /** AX actions when the element is accessibility-grounded. */
+    actions?: string[];
+    /** Provenance — every source that contributed to this element, in fixed
+     * `ocr < ax < vlm` order for stability. Always non-empty. */
+    groundingSources: GroundingSource[];
+}
+export interface MergeScreenInput {
+    readonly ocr?: readonly OcrBoxLike[];
+    readonly ax?: readonly AxNodeLike[];
+    readonly vlm?: readonly VlmElementLike[];
+}
+export interface MergeScreenOptions {
+    /** Boxes whose IoU exceeds this collapse into one element (default 0.6). */
+    readonly iouThreshold?: number;
+}
+/** Intersection-over-union of two `[x, y, w, h]` boxes. 0 when either is empty
+ * or they don't overlap. */
+export declare function bboxIou(a: Bbox, b: Bbox): number;
+/**
+ * Merge OCR boxes + AX clickables + VLM elements into one deduplicated,
+ * deterministically-ordered element list.
+ *
+ * - Elements from different sources whose bboxes overlap above `iouThreshold`
+ *   (and share a `displayId`) collapse into one element that records all
+ *   contributing sources in `groundingSources`.
+ * - Field precedence is AX > OCR > VLM (AX wins id/label/role; OCR text fills
+ *   in when AX has no label; VLM desc is the last resort).
+ * - Output order is top-to-bottom, then left-to-right, so the envelope is
+ *   stable across turns regardless of input ordering.
+ * - Degrades gracefully: any source may be absent/empty (e.g. accessibility off)
+ *   and the function never throws.
+ */
+export declare function mergeScreenElements(input: MergeScreenInput, options?: MergeScreenOptions): GetScreenElement[];
+//# sourceMappingURL=get-screen-elements.d.ts.map

package/dist/get-screen-elements.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"get-screen-elements.d.ts","sourceRoot":"","sources":["../src/get-screen-elements.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAEH,iDAAiD;AACjD,MAAM,MAAM,IAAI,GAAG,SAAS,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;AAE7D,uDAAuD;AACvD,MAAM,WAAW,UAAU;IACzB,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,IAAI,EAAE,IAAI,CAAC;IACpB,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;CAC5B;AAED,uDAAuD;AACvD,MAAM,WAAW,UAAU;IACzB,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,IAAI,EAAE,IAAI,CAAC;IACpB,QAAQ,CAAC,OAAO,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IACrC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;CAC5B;AAED,2DAA2D;AAC3D,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,IAAI,EAAE,IAAI,CAAC;IACpB,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;CAC5B;AAED,MAAM,MAAM,eAAe,GAAG,KAAK,GAAG,IAAI,GAAG,KAAK,CAAC;AAEnD,4EAA4E;AAC5E,MAAM,WAAW,gBAAgB;IAC/B,2DAA2D;IAC3D,EAAE,EAAE,MAAM,CAAC;IACX,yFAAyF;IACzF,IAAI,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;IACvC,6EAA6E;IAC7E,IAAI,EAAE,MAAM,CAAC;IACb,4DAA4D;IAC5D,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,MAAM,CAAC;IAClB,6DAA6D;IAC7D,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IACnB;iEAC6D;IAC7D,gBAAgB,EAAE,eAAe,EAAE,CAAC;CACrC;AAED,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,CAAC,GAAG,CAAC,EAAE,SAAS,UAAU,EAAE,CAAC;IACrC,QAAQ,CAAC,EAAE,CAAC,EAAE,SAAS,UAAU,EAAE,CAAC;IACpC,QAAQ,CAAC,GAAG,CAAC,EAAE,SAAS,cAAc,EAAE,CAAC;CAC1C;AAED,MAAM,WAAW,kBAAkB;IACjC,4EAA4E;IAC5E,QAAQ,CAAC,YAAY,CAAC,EAAE,MAAM,CAAC;CAChC;AAID;4BAC4B;AAC5B,wBAAgB,OAAO,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,IAAI,GAAG,MAAM,CAchD;AAoBD;;;;;;;;;;;;;GAaG;AACH,wBAAgB,mBAAmB,CACjC,KAAK,EAAE,gBAAgB,EACvB,OAAO,GAAE,kBAAuB,GAC/B,gBAAgB,EAAE,CAkEpB"}

package/dist/get-screen.d.ts

@@ -0,0 +1,60 @@
+/**
+ * GET_SCREEN core (issue #9105 / M2) — token-frugal structured screen readout.
+ *
+ * Returns OCR text + grounded elements (id/text/bbox/semantic position) from a
+ * captured frame using the registered coord-OCR service (native Windows OCR /
+ * docTR / Apple Vision — zero LLM tokens). The raw image is OMITTED by default
+ * (`includeImage:false`) so a CUA loop can read the screen each tick without
+ * spending image tokens; it is only base64-attached when explicitly requested.
+ *
+ * Pure + injectable (the OCR service can be passed in) so it is unit-testable
+ * without a real capture or a registered provider.
+ */
+import { type OcrWithCoordsService } from "./ocr-with-coords.js";
+export interface GetScreenElement {
+    /**
+     * Monotonic 1-based Set-of-Marks number (reading order). A model can pick
+     * `[3]` instead of regressing raw coordinates.
+     */
+    index: number;
+    /** Stable per-result id. */
+    id: string;
+    text: string;
+    /** Display-absolute [x, y, width, height]. */
+    bbox: [number, number, number, number];
+    /** Click target — the integer center of `bbox`, the point `index` resolves to. */
+    center: {
+        x: number;
+        y: number;
+    };
+    semantic_position: string;
+    displayId: number;
+}
+export interface GetScreenResult {
+    op: "get_screen";
+    displayId: number;
+    width: number;
+    height: number;
+    /** When the source frame was captured (ms epoch). */
+    lastChangeTime: number;
+    /** True when a coord-OCR provider was available and ran. */
+    ocrAvailable: boolean;
+    ocrText: string;
+    elements: GetScreenElement[];
+    elementCount: number;
+    /** Base64 PNG — only present when `includeImage` was requested. */
+    image?: string;
+}
+export interface BuildGetScreenOptions {
+    pngBytes: Uint8Array;
+    displayId?: number;
+    includeImage?: boolean;
+    includeOcr?: boolean;
+    capturedAt?: number;
+    /** Override for tests; defaults to the registered coord-OCR service. */
+    ocrService?: OcrWithCoordsService | null;
+}
+export declare function buildGetScreen(opts: BuildGetScreenOptions): Promise<GetScreenResult>;
+/** Human-readable one-line summary for the agent reply. */
+export declare function summarizeGetScreen(r: GetScreenResult): string;
+//# sourceMappingURL=get-screen.d.ts.map

package/dist/get-screen.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"get-screen.d.ts","sourceRoot":"","sources":["../src/get-screen.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAEH,OAAO,EAEL,KAAK,oBAAoB,EAE1B,MAAM,sBAAsB,CAAC;AAE9B,MAAM,WAAW,gBAAgB;IAC/B;;;OAGG;IACH,KAAK,EAAE,MAAM,CAAC;IACd,4BAA4B;IAC5B,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,8CAA8C;IAC9C,IAAI,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;IACvC,kFAAkF;IAClF,MAAM,EAAE;QAAE,CAAC,EAAE,MAAM,CAAC;QAAC,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;IACjC,iBAAiB,EAAE,MAAM,CAAC;IAC1B,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,eAAe;IAC9B,EAAE,EAAE,YAAY,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,qDAAqD;IACrD,cAAc,EAAE,MAAM,CAAC;IACvB,4DAA4D;IAC5D,YAAY,EAAE,OAAO,CAAC;IACtB,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,gBAAgB,EAAE,CAAC;IAC7B,YAAY,EAAE,MAAM,CAAC;IACrB,mEAAmE;IACnE,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,qBAAqB;IACpC,QAAQ,EAAE,UAAU,CAAC;IACrB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,wEAAwE;IACxE,UAAU,CAAC,EAAE,oBAAoB,GAAG,IAAI,CAAC;CAC1C;AAED,wBAAsB,cAAc,CAClC,IAAI,EAAE,qBAAqB,GAC1B,OAAO,CAAC,eAAe,CAAC,CAwD1B;AAED,2DAA2D;AAC3D,wBAAgB,kBAAkB,CAAC,CAAC,EAAE,eAAe,GAAG,MAAM,CAa7D"}

package/dist/image/sharp-compat.d.ts

@@ -0,0 +1,89 @@
+/** Raw-pixel input descriptor (mirrors `sharp.SharpOptions["raw"]`). */
+export interface SharpRawInput {
+    width: number;
+    height: number;
+    channels: 1 | 2 | 3 | 4;
+}
+/** Constructor options subset used by the codebase. */
+export interface SharpFactoryOptions {
+    raw?: SharpRawInput;
+    limitInputPixels?: number | boolean;
+    failOnError?: boolean;
+}
+/** `resize` options subset used by the codebase. */
+export interface SharpResizeOptions {
+    fit?: "fill" | "contain" | "cover" | "inside" | "outside";
+}
+/** RGBA background for `extend` (alpha is accepted but encoded outputs only). */
+export interface SharpColor {
+    r: number;
+    g: number;
+    b: number;
+    alpha?: number;
+}
+export interface SharpExtendOptions {
+    top?: number;
+    bottom?: number;
+    left?: number;
+    right?: number;
+    background?: SharpColor;
+}
+export interface SharpExtractRegion {
+    left: number;
+    top: number;
+    width: number;
+    height: number;
+}
+/** Subset of `sharp.Metadata` the codebase reads. */
+export interface SharpMetadata {
+    width?: number;
+    height?: number;
+    channels?: number;
+    format?: string;
+}
+export interface SharpRawInfo {
+    width: number;
+    height: number;
+    channels: number;
+}
+export interface SharpResolveWithObject {
+    data: Buffer;
+    info: SharpRawInfo;
+}
+/**
+ * The chainable instance surface. This is structurally the subset of
+ * `sharp.Sharp` the codebase touches, so a real `sharp` instance satisfies it
+ * and call sites need no per-backend typing.
+ */
+export interface SharpInstance {
+    metadata(): Promise<SharpMetadata>;
+    resize(width: number, height: number, options?: SharpResizeOptions): SharpInstance;
+    removeAlpha(): SharpInstance;
+    ensureAlpha(): SharpInstance;
+    extract(region: SharpExtractRegion): SharpInstance;
+    extend(options: SharpExtendOptions): SharpInstance;
+    trim(): SharpInstance;
+    clone(): SharpInstance;
+    png(): SharpInstance;
+    jpeg(): SharpInstance;
+    raw(): SharpInstance;
+    toBuffer(): Promise<Buffer>;
+    toBuffer(options: {
+        resolveWithObject: true;
+    }): Promise<SharpResolveWithObject>;
+}
+/** The callable factory surface (`sharp(input, options?)`). */
+export type SharpFactory = (input?: Buffer | Uint8Array, options?: SharpFactoryOptions) => SharpInstance;
+/**
+ * Resolve the image backend. Tries native `sharp` first (dynamic import so the
+ * native addon is never touched at module-eval); on any failure falls back to
+ * the pure-JS jimp shim. The result is cached for the process lifetime.
+ */
+export declare function getSharp(): Promise<SharpFactory>;
+/**
+ * Construct the pure-JS shim factory directly. Exposed so the compat test can
+ * diff the shim against native `sharp` without depending on which backend
+ * `getSharp()` happens to resolve on the host.
+ */
+export declare function createJimpShim(): SharpFactory;
+//# sourceMappingURL=sharp-compat.d.ts.map

package/dist/image/sharp-compat.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"sharp-compat.d.ts","sourceRoot":"","sources":["../../src/image/sharp-compat.ts"],"names":[],"mappings":"AAmBA,wEAAwE;AACxE,MAAM,WAAW,aAAa;IAC5B,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;CACzB;AAED,uDAAuD;AACvD,MAAM,WAAW,mBAAmB;IAClC,GAAG,CAAC,EAAE,aAAa,CAAC;IACpB,gBAAgB,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC;IACpC,WAAW,CAAC,EAAE,OAAO,CAAC;CACvB;AAED,oDAAoD;AACpD,MAAM,WAAW,kBAAkB;IACjC,GAAG,CAAC,EAAE,MAAM,GAAG,SAAS,GAAG,OAAO,GAAG,QAAQ,GAAG,SAAS,CAAC;CAC3D;AAED,iFAAiF;AACjF,MAAM,WAAW,UAAU;IACzB,CAAC,EAAE,MAAM,CAAC;IACV,CAAC,EAAE,MAAM,CAAC;IACV,CAAC,EAAE,MAAM,CAAC;IACV,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,kBAAkB;IACjC,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,UAAU,CAAC,EAAE,UAAU,CAAC;CACzB;AAED,MAAM,WAAW,kBAAkB;IACjC,IAAI,EAAE,MAAM,CAAC;IACb,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,qDAAqD;AACrD,MAAM,WAAW,aAAa;IAC5B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,YAAY;IAC3B,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,WAAW,sBAAsB;IACrC,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,YAAY,CAAC;CACpB;AAED;;;;GAIG;AACH,MAAM,WAAW,aAAa;IAC5B,QAAQ,IAAI,OAAO,CAAC,aAAa,CAAC,CAAC;IACnC,MAAM,CACJ,KAAK,EAAE,MAAM,EACb,MAAM,EAAE,MAAM,EACd,OAAO,CAAC,EAAE,kBAAkB,GAC3B,aAAa,CAAC;IACjB,WAAW,IAAI,aAAa,CAAC;IAC7B,WAAW,IAAI,aAAa,CAAC;IAC7B,OAAO,CAAC,MAAM,EAAE,kBAAkB,GAAG,aAAa,CAAC;IACnD,MAAM,CAAC,OAAO,EAAE,kBAAkB,GAAG,aAAa,CAAC;IACnD,IAAI,IAAI,aAAa,CAAC;IACtB,KAAK,IAAI,aAAa,CAAC;IACvB,GAAG,IAAI,aAAa,CAAC;IACrB,IAAI,IAAI,aAAa,CAAC;IACtB,GAAG,IAAI,aAAa,CAAC;IACrB,QAAQ,IAAI,OAAO,CAAC,MAAM,CAAC,CAAC;IAC5B,QAAQ,CAAC,OAAO,EAAE;QAChB,iBAAiB,EAAE,IAAI,CAAC;KACzB,GAAG,OAAO,CAAC,sBAAsB,CAAC,CAAC;CACrC;AAED,+DAA+D;AAC/D,MAAM,MAAM,YAAY,GAAG,CACzB,KAAK,CAAC,EAAE,MAAM,GAAG,UAAU,EAC3B,OAAO,CAAC,EAAE,mBAAmB,KAC1B,aAAa,CAAC;AAInB;;;;GAIG;AACH,wBAAsB,QAAQ,IAAI,OAAO,CAAC,YAAY,CAAC,CAStD;AAyKD;;;;GAIG;AACH,wBAAgB,cAAc,IAAI,YAAY,CAuB7C"}

package/dist/image-input.d.ts

@@ -0,0 +1,15 @@
+export declare const MAX_VISION_IMAGE_BYTES: number;
+export interface ValidatedVisionImage {
+    readonly width: number;
+    readonly height: number;
+    readonly format: "jpeg" | "png" | "webp";
+    readonly contentType: "image/jpeg" | "image/png" | "image/webp";
+}
+export declare function estimateBase64DecodedBytes(base64: string): number;
+export declare function parseVisionDataImageUrl(value: string): {
+    mimeType: string;
+    base64: string;
+};
+export declare function assertValidVisionImageBuffer(data: Buffer): Promise<ValidatedVisionImage>;
+export declare function assertSafeVisionDataImageUrl(imageUrl: string): Promise<ValidatedVisionImage>;
+//# sourceMappingURL=image-input.d.ts.map

package/dist/image-input.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"image-input.d.ts","sourceRoot":"","sources":["../src/image-input.ts"],"names":[],"mappings":"AAEA,eAAO,MAAM,sBAAsB,QAAmB,CAAC;AASvD,MAAM,WAAW,oBAAoB;IACnC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,MAAM,EAAE,MAAM,GAAG,KAAK,GAAG,MAAM,CAAC;IACzC,QAAQ,CAAC,WAAW,EAAE,YAAY,GAAG,WAAW,GAAG,YAAY,CAAC;CACjE;AAED,wBAAgB,0BAA0B,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAKjE;AAED,wBAAgB,uBAAuB,CAAC,KAAK,EAAE,MAAM,GAAG;IACtD,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,CAAC;CAChB,CA8BA;AAED,wBAAsB,4BAA4B,CAChD,IAAI,EAAE,MAAM,GACX,OAAO,CAAC,oBAAoB,CAAC,CAkC/B;AAED,wBAAsB,4BAA4B,CAChD,QAAQ,EAAE,MAAM,GACf,OAAO,CAAC,oBAAoB,CAAC,CAG/B"}

package/dist/index.d.ts

@@ -0,0 +1,4 @@
+import type { Plugin } from "@elizaos/core";
+export declare const visionPlugin: Plugin;
+export default visionPlugin;
+//# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,eAAe,CAAC;AAwB5C,eAAO,MAAM,YAAY,EAAE,MAuH1B,CAAC;AAEF,eAAe,YAAY,CAAC"}