@elizaos/plugin-vision 2.0.0-alpha.9 → 2.0.3-beta.5

package/native/yolo.cpp/verify/run_ts.mjs

@@ -0,0 +1,26 @@
+// End-to-end integration test of the REAL TypeScript detector path:
+//   yolo-detector.ts -> native/yolo-ffi.ts -> libyolo.dll -> parseYoloV8 -> NMS
+// Run from the plugin root with bun so workspace deps (@elizaos/core, sharp)
+// resolve:
+//   ELIZA_YOLO_GGUF=... ELIZA_YOLO_LIB=... bun native/yolo.cpp/verify/run_ts.mjs
+import { readFileSync } from "node:fs";
+import { dirname, join } from "node:path";
+import { fileURLToPath } from "node:url";
+import { YOLODetector } from "../../../src/yolo-detector.ts";
+
+const HERE = dirname(fileURLToPath(import.meta.url));
+const img = readFileSync(join(HERE, "bus.jpg"));
+
+console.error("isAvailable:", await YOLODetector.isAvailable());
+const det = new YOLODetector({ scoreThreshold: 0.25, nmsIouThreshold: 0.5 });
+await det.initialize();
+const objs = await det.detect(img);
+console.log(`detections: ${objs.length}`);
+for (const o of objs) {
+  const b = o.boundingBox;
+  console.log(
+    `  ${o.type.padEnd(12)} ${o.confidence.toFixed(3)} ` +
+      `bbox=(${b.x.toFixed(0)},${b.y.toFixed(0)},${b.width.toFixed(0)},${b.height.toFixed(0)})`,
+  );
+}
+await det.dispose();

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@elizaos/plugin-vision",
   "description": "Vision plugin for ElizaOS - provides camera integration and visual awareness",
-  "version": "2.0.0-alpha.9",
+  "version": "2.0.3-beta.5",
   "type": "module",
   "main": "dist/index.js",
   "module": "dist/index.js",
@@ -29,53 +29,79 @@
   "exports": {
     "./package.json": "./package.json",
     ".": {
+      "types": "./dist/index.d.ts",
+      "eliza-source": {
+        "types": "./src/index.ts",
+        "import": "./src/index.ts",
+        "default": "./src/index.ts"
+      },
       "import": {
         "types": "./dist/index.d.ts",
         "default": "./dist/index.js"
       }
+    },
+    "./*.css": "./dist/*.css",
+    "./*": {
+      "types": "./dist/*.d.ts",
+      "eliza-source": {
+        "types": "./src/*.ts",
+        "import": "./src/*.ts",
+        "default": "./src/*.ts"
+      },
+      "import": "./dist/*.js",
+      "default": "./dist/*.js"
     }
   },
   "files": [
+    "registry-entry.json",
     "dist",
+    "native",
+    "scripts",
     "README.md",
     ".npmignore",
     "package.json",
-    "build.config.ts"
+    "auto-enable.ts"
   ],
-  "dependencies": {
-    "@elizaos/core": "2.0.0-alpha.3",
-    "axios": "^1.7.2",
-    "canvas": "^3.1.2",
-    "face-api.js": "^0.22.2",
-    "sharp": "^0.34.3",
-    "tesseract.js": "^6.0.1",
-    "zod": "^4.3.6"
+  "elizaos": {
+    "plugin": {
+      "autoEnableModule": "./auto-enable.ts",
+      "capabilities": [
+        "vision"
+      ]
+    }
   },
-  "optionalDependencies": {
-    "@tensorflow-models/coco-ssd": "^2.2.3",
-    "@tensorflow-models/mobilenet": "^2.1.0",
-    "@tensorflow-models/pose-detection": "^2.1.0",
-    "@tensorflow-models/posenet": "^2.2.2",
-    "@tensorflow/tfjs-node": "^4.22.0"
+  "dependencies": {
+    "@elizaos/core": "2.0.3-beta.5",
+    "jimp": "^1.6.0",
+    "sharp": "^0.34.3"
   },
   "devDependencies": {
-    "@biomejs/biome": "^2.3.11",
+    "@biomejs/biome": "^2.4.14",
+    "@elizaos/plugin-computeruse": "2.0.3-beta.5",
     "@types/bun": "^1.3.5",
     "@types/node": "^25.0.3",
-    "@types/tesseract.js": "^2.0.0",
-    "dotenv": "^17.2.3",
-    "tsx": "^4.7.0",
-    "typescript": "^5.9.3"
+    "typescript": "^6.0.3",
+    "vitest": "^4.1.4"
+  },
+  "peerDependencies": {
+    "@elizaos/plugin-computeruse": "2.0.3-beta.5"
+  },
+  "peerDependenciesMeta": {
+    "@elizaos/plugin-computeruse": {
+      "optional": true
+    }
   },
   "scripts": {
     "dev": "bun run build.ts --watch",
-    "clean": "rm -rf dist && rm -rf .turbo",
+    "clean": "node ../../packages/scripts/rm-path-recursive.mjs dist .turbo",
     "lint": "echo \"Lint skipped for release\"",
     "typecheck": "echo \"Typecheck skipped for release\"",
     "test": "vitest run",
     "lint:check": "bun run lint",
     "build": "bun run build.ts",
     "build:ts": "bun run build.ts",
+    "build:native": "bun run native/yolo.cpp/build.mjs",
+    "build:weights": "python native/yolo.cpp/scripts/convert.py --variant yolov8n",
     "format": "bunx @biomejs/biome format --write .",
     "format:check": "bunx @biomejs/biome format ."
   },
@@ -95,8 +121,8 @@
       }
     }
   },
-  "gitHead": "d5bd5c43bfebeb7ac02f9e029f924cb6cd5c2ec7",
-  "milady": {
+  "gitHead": "ff6157011c9459670021cc28a6797592a78b8817",
+  "eliza": {
     "platforms": [
       "node"
     ],

package/registry-entry.json

@@ -0,0 +1,43 @@
+{
+  "id": "vision",
+  "name": "Vision",
+  "description": "Vision plugin for ElizaOS - provides camera integration and visual awareness",
+  "npmName": "@elizaos/plugin-vision",
+  "version": "2.0.0-beta.0",
+  "source": "bundled",
+  "tags": ["vision", "camera", "ai-vision"],
+  "config": {
+    "CAMERA_NAME": {
+      "type": "string",
+      "required": false,
+      "sensitive": false,
+      "label": "Name",
+      "help": "Camera name to search for (lowercase partial match)",
+      "advanced": false
+    },
+    "PIXEL_CHANGE_THRESHOLD": {
+      "type": "number",
+      "required": false,
+      "sensitive": false,
+      "label": "Change Threshold",
+      "help": "Percentage of pixels that must change to trigger VLM update (default: 50)",
+      "advanced": false
+    }
+  },
+  "render": {
+    "visible": true,
+    "pinTo": [],
+    "style": "card",
+    "icon": "Eye",
+    "group": "media",
+    "groupOrder": 8,
+    "actions": ["enable", "configure"]
+  },
+  "resources": {
+    "homepage": "https://elizaos.ai",
+    "repository": "https://github.com/elizaos/elizaos"
+  },
+  "dependsOn": [],
+  "kind": "plugin",
+  "subtype": "media"
+}

package/scripts/vendor-tesseract-linux.mjs

@@ -0,0 +1,177 @@
+#!/usr/bin/env node
+/**
+ * Stage a PORTABLE tesseract into the app bundle so on-device/desktop OCR
+ * "just ships and works" with no host `apt install tesseract-ocr` (#9105).
+ *
+ * Output layout (consumed by `ocr-service-linux-tesseract.ts` `resolveTesseract`
+ * via `ELIZA_VISION_VENDOR_DIR`):
+ *
+ *   <out>/tesseract/
+ *     bin/tesseract                 the CLI
+ *     lib/libtesseract.so* liblept.so*   the shared libs (rest come from the host)
+ *     tessdata/<lang>.traineddata   the language model(s)
+ *     tessdata/configs/tsv ...      REQUIRED — tesseract reads the `tsv` output
+ *                                   config from here; without configs/ the
+ *                                   `tsv` arg silently fails ("Can't open tsv")
+ *                                   and the OCR returns zero rows.
+ *
+ * Linux x64 only (the desktop build host). macOS uses Apple Vision, Windows uses
+ * Windows.Media.Ocr, Android uses the native Tesseract4Android bridge — each
+ * provider is platform-resolved in plugin-vision/src/index.ts.
+ *
+ * Usage: node scripts/vendor-tesseract-linux.mjs [--out <dir>] [--langs eng,...]
+ * Build-time prereq: `apt-get download` works (no sudo needed — it only fetches
+ * .debs); `dpkg-deb` extracts them. Run on the Linux build host.
+ */
+import { execFileSync } from "node:child_process";
+import {
+  copyFileSync,
+  cpSync,
+  existsSync,
+  mkdirSync,
+  mkdtempSync,
+  readdirSync,
+  realpathSync,
+  statSync,
+} from "node:fs";
+import { tmpdir } from "node:os";
+import { dirname, join } from "node:path";
+import { fileURLToPath } from "node:url";
+
+const here = dirname(fileURLToPath(import.meta.url));
+function arg(name, def) {
+  const i = process.argv.indexOf(name);
+  return i >= 0 && process.argv[i + 1] ? process.argv[i + 1] : def;
+}
+const outDir = arg("--out", join(here, "..", "vendor"));
+const langs = arg("--langs", "eng")
+  .split(",")
+  .map((s) => s.trim())
+  .filter(Boolean);
+const dest = join(outDir, "tesseract");
+
+if (process.platform !== "linux") {
+  console.log(
+    `[vendor-tesseract] non-linux host (${process.platform}); skipping (platform provider handles OCR).`,
+  );
+  process.exit(0);
+}
+
+const work = mkdtempSync(join(tmpdir(), "vendor-tess-"));
+const pkgs = [
+  "tesseract-ocr",
+  "libtesseract5",
+  "liblept5",
+  ...langs.map((l) => `tesseract-ocr-${l}`),
+];
+console.log(`[vendor-tesseract] downloading: ${pkgs.join(", ")}`);
+execFileSync("apt-get", ["download", ...pkgs], { cwd: work, stdio: "inherit" });
+
+const root = join(work, "root");
+for (const deb of readdirSync(work).filter((f) => f.endsWith(".deb"))) {
+  execFileSync("dpkg-deb", ["-x", join(work, deb), root], { stdio: "inherit" });
+}
+
+mkdirSync(join(dest, "bin"), { recursive: true });
+mkdirSync(join(dest, "lib"), { recursive: true });
+mkdirSync(join(dest, "tessdata"), { recursive: true });
+
+cpSync(join(root, "usr/bin/tesseract"), join(dest, "bin/tesseract"));
+const libDir = join(root, "usr/lib/x86_64-linux-gnu");
+for (const so of readdirSync(libDir).filter((f) =>
+  /^(libtesseract|liblept)\.so/.test(f),
+)) {
+  // Dereference to the real .so bytes and write them under the SONAME name.
+  // Debian ships libtesseract.so.5 etc. as symlinks into the extraction dir;
+  // copying them as symlinks (dereference:false) leaves the bundle pointing at
+  // a temp dir that is deleted on exit — the "portable" bundle then dlopen-fails
+  // with "libtesseract.so.5: cannot open shared object file". Writing real bytes
+  // under each name makes the bundle self-contained.
+  const realSo = realpathSync(join(libDir, so));
+  copyFileSync(realSo, join(dest, "lib", so));
+}
+
+// Copy the FULL transitive shared-lib closure. libtesseract pulls in leptonica,
+// which pulls in libjpeg/png/tiff/webp/openjp2/gif/zlib/... — none of which the
+// two .debs above provide. Without them the bundled binary dlopen-fails on a
+// clean host, defeating the whole "no host install" goal. We let `ldd` resolve
+// the binary's deps with the bundle's own lib/ on the search path (so it can
+// find the libtesseract/leptonica we just staged and recurse into their deps),
+// then copy every resolved .so EXCEPT the glibc/loader set that every Linux
+// target is guaranteed to provide.
+const GLIBC_PROVIDED =
+  /^(ld-linux.*|linux-vdso|libc|libm|libdl|libpthread|librt|libresolv|libgcc_s|libstdc\+\+)\.so/;
+function lddClosure(binPath) {
+  let out = "";
+  try {
+    out = execFileSync("ldd", [binPath], {
+      encoding: "utf8",
+      env: { ...process.env, LD_LIBRARY_PATH: join(dest, "lib") },
+    });
+  } catch (err) {
+    // `ldd` is glibc-only; on a musl/non-glibc build host we can't compute the
+    // closure. Fail loud rather than silently shipping an incomplete bundle.
+    throw new Error(
+      `[vendor-tesseract] ldd failed (${err.message}); cannot compute the shared-lib closure on this host`,
+    );
+  }
+  const resolved = [];
+  for (const line of out.split("\n")) {
+    const m = line.match(/=>\s*(\/\S+\.so\S*)/);
+    if (m) resolved.push(m[1]);
+  }
+  return resolved;
+}
+let bundledExtra = 0;
+for (const soPath of lddClosure(join(dest, "bin", "tesseract"))) {
+  // ldd can resolve a path with a trailing slash (e.g. a symlink dir entry);
+  // strip it so the SONAME basename is never empty (an empty basename made the
+  // target the lib/ dir itself → cpSync EISDIR).
+  const cleanSrc = soPath.replace(/\/+$/, "");
+  const base = cleanSrc.split("/").pop();
+  if (!base || GLIBC_PROVIDED.test(base)) continue;
+  const target = join(dest, "lib", base);
+  if (existsSync(target)) continue; // already staged (libtesseract/liblept)
+  // dereference the symlink to the real .so file and copy its bytes under the
+  // SONAME the binary asks for, so the dynamic loader finds it by that name.
+  const realSrc = realpathSync(cleanSrc);
+  if (!statSync(realSrc).isFile()) continue;
+  copyFileSync(realSrc, target);
+  bundledExtra++;
+}
+console.log(
+  `[vendor-tesseract] bundled ${bundledExtra} transitive shared lib(s) via ldd closure.`,
+);
+
+// tessdata lives under usr/share/tesseract-ocr/<ver>/tessdata — copy the WHOLE
+// dir (traineddata + the REQUIRED configs/ + tessconfigs/). Pick the HIGHEST
+// version dir (numeric-aware) so a host with multiple tesseract data versions
+// installed stages the newest, not whichever readdir happens to list first.
+const shareTess = join(root, "usr/share/tesseract-ocr");
+const ver = readdirSync(shareTess)
+  .filter((d) => existsSync(join(shareTess, d, "tessdata")))
+  .sort((a, b) => b.localeCompare(a, undefined, { numeric: true }))[0];
+if (!ver)
+  throw new Error(
+    "[vendor-tesseract] tessdata not found in downloaded package",
+  );
+cpSync(join(shareTess, ver, "tessdata"), join(dest, "tessdata"), {
+  recursive: true,
+});
+
+if (!existsSync(join(dest, "tessdata", "configs", "tsv"))) {
+  throw new Error(
+    "[vendor-tesseract] tessdata/configs/tsv missing — TSV OCR output would fail",
+  );
+}
+for (const l of langs) {
+  if (!existsSync(join(dest, "tessdata", `${l}.traineddata`))) {
+    throw new Error(`[vendor-tesseract] ${l}.traineddata missing`);
+  }
+}
+console.log(
+  `[vendor-tesseract] staged portable tesseract → ${dest} (langs: ${langs.join(", ")})`,
+);
+console.log(
+  `[vendor-tesseract] set ELIZA_VISION_VENDOR_DIR=${outDir} so the runtime resolves it.`,
+);

package/build.config.ts

@@ -1,70 +0,0 @@
-import type { BuildConfig } from "bun";
-
-// Main build configuration
-export const buildConfig: BuildConfig = {
-  entrypoints: ["./src/index.ts"],
-  outdir: "./dist",
-  target: "node",
-  format: "esm",
-  splitting: false,
-  sourcemap: "external",
-  external: [
-    "fs",
-    "path",
-    "http",
-    "https",
-    "crypto",
-    "node:fs",
-    "node:path",
-    "node:http",
-    "node:https",
-    "node:crypto",
-    "node:stream",
-    "node:buffer",
-    "node:util",
-    "node:events",
-    "node:url",
-    "vitest",
-    "dotenv",
-    "zod",
-    "@elizaos/core",
-    "@elizaos/plugin-message-handling",
-    "@tensorflow-models/coco-ssd",
-    "@tensorflow-models/pose-detection",
-    "@tensorflow-models/posenet",
-    "@tensorflow/tfjs-node",
-    "@tensorflow-models/mobilenet",
-    "axios",
-    "canvas",
-    "face-api.js",
-    "sharp",
-    "tesseract.js",
-  ],
-  naming: "[dir]/[name].[ext]",
-};
-
-// Workers build configuration
-export const workersConfig: BuildConfig = {
-  entrypoints: [
-    "./src/workers/screen-capture-worker.ts",
-    "./src/workers/florence2-worker.ts",
-    "./src/workers/ocr-worker.ts",
-  ],
-  outdir: "./dist/workers",
-  target: "node",
-  format: "cjs", // Workers need CommonJS format
-  splitting: false,
-  sourcemap: true,
-  external: [
-    "sharp",
-    "@tensorflow/tfjs-node",
-    "@tensorflow-models/mobilenet",
-    "@mapbox/node-pre-gyp",
-    "mock-aws-s3",
-    "aws-sdk",
-    "nock",
-    "canvas",
-    "face-api.js",
-  ],
-  naming: "[name].[ext]",
-};