@elizaos/plugin-vision 2.0.0-beta.1 → 2.0.3-beta.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (128) hide show
  1. package/LICENSE +21 -0
  2. package/README.md +73 -301
  3. package/dist/action.d.ts +3 -0
  4. package/dist/action.d.ts.map +1 -0
  5. package/dist/audio-capture-stream.d.ts +42 -0
  6. package/dist/audio-capture-stream.d.ts.map +1 -0
  7. package/dist/audio-capture.d.ts +25 -0
  8. package/dist/audio-capture.d.ts.map +1 -0
  9. package/dist/computeruse-ocr-bridge.d.ts +50 -0
  10. package/dist/computeruse-ocr-bridge.d.ts.map +1 -0
  11. package/dist/config.d.ts +68 -0
  12. package/dist/config.d.ts.map +1 -0
  13. package/dist/describe-backpressure.d.ts +90 -0
  14. package/dist/describe-backpressure.d.ts.map +1 -0
  15. package/dist/dirty-tile-describer.d.ts +102 -0
  16. package/dist/dirty-tile-describer.d.ts.map +1 -0
  17. package/dist/dirty-tile-scene.d.ts +56 -0
  18. package/dist/dirty-tile-scene.d.ts.map +1 -0
  19. package/dist/entity-tracker.d.ts +33 -0
  20. package/dist/entity-tracker.d.ts.map +1 -0
  21. package/dist/face-detector-ggml.d.ts +60 -0
  22. package/dist/face-detector-ggml.d.ts.map +1 -0
  23. package/dist/face-detector-mediapipe.d.ts +25 -0
  24. package/dist/face-detector-mediapipe.d.ts.map +1 -0
  25. package/dist/face-recognition-ggml.d.ts +94 -0
  26. package/dist/face-recognition-ggml.d.ts.map +1 -0
  27. package/dist/get-screen-elements.d.ts +90 -0
  28. package/dist/get-screen-elements.d.ts.map +1 -0
  29. package/dist/get-screen.d.ts +60 -0
  30. package/dist/get-screen.d.ts.map +1 -0
  31. package/dist/image/sharp-compat.d.ts +89 -0
  32. package/dist/image/sharp-compat.d.ts.map +1 -0
  33. package/dist/image-input.d.ts +15 -0
  34. package/dist/image-input.d.ts.map +1 -0
  35. package/dist/index.d.ts +4 -0
  36. package/dist/index.d.ts.map +1 -0
  37. package/dist/index.js +7957 -6238
  38. package/dist/index.js.map +41 -26
  39. package/dist/lifecycle.d.ts +94 -0
  40. package/dist/lifecycle.d.ts.map +1 -0
  41. package/dist/mobile/capacitor-camera.d.ts +85 -0
  42. package/dist/mobile/capacitor-camera.d.ts.map +1 -0
  43. package/dist/native/doctr-ffi.d.ts +40 -0
  44. package/dist/native/doctr-ffi.d.ts.map +1 -0
  45. package/dist/native/yolo-ffi.d.ts +21 -0
  46. package/dist/native/yolo-ffi.d.ts.map +1 -0
  47. package/dist/ocr-host-windows.d.ts +34 -0
  48. package/dist/ocr-host-windows.d.ts.map +1 -0
  49. package/dist/ocr-service-apple-vision-macos.d.ts +51 -0
  50. package/dist/ocr-service-apple-vision-macos.d.ts.map +1 -0
  51. package/dist/ocr-service-doctr.d.ts +61 -0
  52. package/dist/ocr-service-doctr.d.ts.map +1 -0
  53. package/dist/ocr-service-linux-tesseract.d.ts +85 -0
  54. package/dist/ocr-service-linux-tesseract.d.ts.map +1 -0
  55. package/dist/ocr-service-paddleocr.d.ts +59 -0
  56. package/dist/ocr-service-paddleocr.d.ts.map +1 -0
  57. package/dist/ocr-service-windows.d.ts +41 -0
  58. package/dist/ocr-service-windows.d.ts.map +1 -0
  59. package/dist/ocr-service.d.ts +91 -0
  60. package/dist/ocr-service.d.ts.map +1 -0
  61. package/dist/ocr-with-coords.d.ts +103 -0
  62. package/dist/ocr-with-coords.d.ts.map +1 -0
  63. package/dist/person-detector.d.ts +17 -0
  64. package/dist/person-detector.d.ts.map +1 -0
  65. package/dist/provider.d.ts +3 -0
  66. package/dist/provider.d.ts.map +1 -0
  67. package/dist/routes.d.ts +7 -0
  68. package/dist/routes.d.ts.map +1 -0
  69. package/dist/screen-capture-bridge.d.ts +51 -0
  70. package/dist/screen-capture-bridge.d.ts.map +1 -0
  71. package/dist/screen-capture.d.ts +17 -0
  72. package/dist/screen-capture.d.ts.map +1 -0
  73. package/dist/screen-tiler.d.ts +75 -0
  74. package/dist/screen-tiler.d.ts.map +1 -0
  75. package/dist/service.d.ts +176 -0
  76. package/dist/service.d.ts.map +1 -0
  77. package/dist/set-of-marks-provider.d.ts +64 -0
  78. package/dist/set-of-marks-provider.d.ts.map +1 -0
  79. package/dist/som.d.ts +135 -0
  80. package/dist/som.d.ts.map +1 -0
  81. package/dist/som.js +184 -0
  82. package/dist/som.js.map +11 -0
  83. package/dist/test-input.d.ts +25 -0
  84. package/dist/test-input.d.ts.map +1 -0
  85. package/dist/types.d.ts +241 -0
  86. package/dist/types.d.ts.map +1 -0
  87. package/dist/vision-context-augmenter.d.ts +93 -0
  88. package/dist/vision-context-augmenter.d.ts.map +1 -0
  89. package/dist/vision-worker-manager.d.ts +51 -0
  90. package/dist/vision-worker-manager.d.ts.map +1 -0
  91. package/dist/workers/ocr-worker.d.ts +2 -0
  92. package/dist/workers/ocr-worker.d.ts.map +1 -0
  93. package/dist/workers/ocr-worker.js +1075 -7821
  94. package/dist/workers/ocr-worker.js.map +10 -51
  95. package/dist/workers/screen-capture-worker.d.ts +2 -0
  96. package/dist/workers/screen-capture-worker.d.ts.map +1 -0
  97. package/dist/workers/screen-capture-worker.js +364 -6
  98. package/dist/workers/screen-capture-worker.js.map +5 -4
  99. package/dist/workers/worker-logger.d.ts +10 -0
  100. package/dist/workers/worker-logger.d.ts.map +1 -0
  101. package/dist/yolo-detector.d.ts +37 -0
  102. package/dist/yolo-detector.d.ts.map +1 -0
  103. package/native/doctr.cpp/CMakeLists.txt +58 -0
  104. package/native/doctr.cpp/README.md +62 -0
  105. package/native/doctr.cpp/include/doctr.h +91 -0
  106. package/native/doctr.cpp/scripts/convert.py +98 -0
  107. package/native/doctr.cpp/src/doctr_det.cpp +112 -0
  108. package/native/doctr.cpp/src/doctr_rec.cpp +103 -0
  109. package/native/macos-vision-ocr.swift +113 -0
  110. package/native/mobilefacenet.cpp/README.md +13 -0
  111. package/native/movenet.cpp/README.md +10 -0
  112. package/native/retinaface.cpp/README.md +12 -0
  113. package/native/yolo.cpp/CMakeLists.txt +57 -0
  114. package/native/yolo.cpp/README.md +64 -0
  115. package/native/yolo.cpp/build.mjs +76 -0
  116. package/native/yolo.cpp/include/yolo.h +62 -0
  117. package/native/yolo.cpp/scripts/convert.py +248 -0
  118. package/native/yolo.cpp/src/yolo.cpp +425 -0
  119. package/native/yolo.cpp/verify/compare.py +99 -0
  120. package/native/yolo.cpp/verify/make_ref.py +75 -0
  121. package/native/yolo.cpp/verify/run_ggml.mjs +78 -0
  122. package/native/yolo.cpp/verify/run_ts.mjs +26 -0
  123. package/package.json +39 -21
  124. package/registry-entry.json +43 -0
  125. package/scripts/vendor-tesseract-linux.mjs +177 -0
  126. package/build.config.ts +0 -89
  127. package/dist/workers/florence2-worker.js +0 -779
  128. package/dist/workers/florence2-worker.js.map +0 -13
@@ -0,0 +1 @@
1
+ {"version":3,"file":"get-screen.d.ts","sourceRoot":"","sources":["../src/get-screen.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAEH,OAAO,EAEL,KAAK,oBAAoB,EAE1B,MAAM,sBAAsB,CAAC;AAE9B,MAAM,WAAW,gBAAgB;IAC/B;;;OAGG;IACH,KAAK,EAAE,MAAM,CAAC;IACd,4BAA4B;IAC5B,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,8CAA8C;IAC9C,IAAI,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;IACvC,kFAAkF;IAClF,MAAM,EAAE;QAAE,CAAC,EAAE,MAAM,CAAC;QAAC,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;IACjC,iBAAiB,EAAE,MAAM,CAAC;IAC1B,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,eAAe;IAC9B,EAAE,EAAE,YAAY,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,qDAAqD;IACrD,cAAc,EAAE,MAAM,CAAC;IACvB,4DAA4D;IAC5D,YAAY,EAAE,OAAO,CAAC;IACtB,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,gBAAgB,EAAE,CAAC;IAC7B,YAAY,EAAE,MAAM,CAAC;IACrB,mEAAmE;IACnE,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,qBAAqB;IACpC,QAAQ,EAAE,UAAU,CAAC;IACrB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,wEAAwE;IACxE,UAAU,CAAC,EAAE,oBAAoB,GAAG,IAAI,CAAC;CAC1C;AAED,wBAAsB,cAAc,CAClC,IAAI,EAAE,qBAAqB,GAC1B,OAAO,CAAC,eAAe,CAAC,CAwD1B;AAED,2DAA2D;AAC3D,wBAAgB,kBAAkB,CAAC,CAAC,EAAE,eAAe,GAAG,MAAM,CAa7D"}
@@ -0,0 +1,89 @@
1
+ /** Raw-pixel input descriptor (mirrors `sharp.SharpOptions["raw"]`). */
2
+ export interface SharpRawInput {
3
+ width: number;
4
+ height: number;
5
+ channels: 1 | 2 | 3 | 4;
6
+ }
7
+ /** Constructor options subset used by the codebase. */
8
+ export interface SharpFactoryOptions {
9
+ raw?: SharpRawInput;
10
+ limitInputPixels?: number | boolean;
11
+ failOnError?: boolean;
12
+ }
13
+ /** `resize` options subset used by the codebase. */
14
+ export interface SharpResizeOptions {
15
+ fit?: "fill" | "contain" | "cover" | "inside" | "outside";
16
+ }
17
+ /** RGBA background for `extend` (alpha is accepted but encoded outputs only). */
18
+ export interface SharpColor {
19
+ r: number;
20
+ g: number;
21
+ b: number;
22
+ alpha?: number;
23
+ }
24
+ export interface SharpExtendOptions {
25
+ top?: number;
26
+ bottom?: number;
27
+ left?: number;
28
+ right?: number;
29
+ background?: SharpColor;
30
+ }
31
+ export interface SharpExtractRegion {
32
+ left: number;
33
+ top: number;
34
+ width: number;
35
+ height: number;
36
+ }
37
+ /** Subset of `sharp.Metadata` the codebase reads. */
38
+ export interface SharpMetadata {
39
+ width?: number;
40
+ height?: number;
41
+ channels?: number;
42
+ format?: string;
43
+ }
44
+ export interface SharpRawInfo {
45
+ width: number;
46
+ height: number;
47
+ channels: number;
48
+ }
49
+ export interface SharpResolveWithObject {
50
+ data: Buffer;
51
+ info: SharpRawInfo;
52
+ }
53
+ /**
54
+ * The chainable instance surface. This is structurally the subset of
55
+ * `sharp.Sharp` the codebase touches, so a real `sharp` instance satisfies it
56
+ * and call sites need no per-backend typing.
57
+ */
58
+ export interface SharpInstance {
59
+ metadata(): Promise<SharpMetadata>;
60
+ resize(width: number, height: number, options?: SharpResizeOptions): SharpInstance;
61
+ removeAlpha(): SharpInstance;
62
+ ensureAlpha(): SharpInstance;
63
+ extract(region: SharpExtractRegion): SharpInstance;
64
+ extend(options: SharpExtendOptions): SharpInstance;
65
+ trim(): SharpInstance;
66
+ clone(): SharpInstance;
67
+ png(): SharpInstance;
68
+ jpeg(): SharpInstance;
69
+ raw(): SharpInstance;
70
+ toBuffer(): Promise<Buffer>;
71
+ toBuffer(options: {
72
+ resolveWithObject: true;
73
+ }): Promise<SharpResolveWithObject>;
74
+ }
75
+ /** The callable factory surface (`sharp(input, options?)`). */
76
+ export type SharpFactory = (input?: Buffer | Uint8Array, options?: SharpFactoryOptions) => SharpInstance;
77
+ /**
78
+ * Resolve the image backend. Tries native `sharp` first (dynamic import so the
79
+ * native addon is never touched at module-eval); on any failure falls back to
80
+ * the pure-JS jimp shim. The result is cached for the process lifetime.
81
+ */
82
+ export declare function getSharp(): Promise<SharpFactory>;
83
+ /**
84
+ * Construct the pure-JS shim factory directly. Exposed so the compat test can
85
+ * diff the shim against native `sharp` without depending on which backend
86
+ * `getSharp()` happens to resolve on the host.
87
+ */
88
+ export declare function createJimpShim(): SharpFactory;
89
+ //# sourceMappingURL=sharp-compat.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"sharp-compat.d.ts","sourceRoot":"","sources":["../../src/image/sharp-compat.ts"],"names":[],"mappings":"AAmBA,wEAAwE;AACxE,MAAM,WAAW,aAAa;IAC5B,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;CACzB;AAED,uDAAuD;AACvD,MAAM,WAAW,mBAAmB;IAClC,GAAG,CAAC,EAAE,aAAa,CAAC;IACpB,gBAAgB,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC;IACpC,WAAW,CAAC,EAAE,OAAO,CAAC;CACvB;AAED,oDAAoD;AACpD,MAAM,WAAW,kBAAkB;IACjC,GAAG,CAAC,EAAE,MAAM,GAAG,SAAS,GAAG,OAAO,GAAG,QAAQ,GAAG,SAAS,CAAC;CAC3D;AAED,iFAAiF;AACjF,MAAM,WAAW,UAAU;IACzB,CAAC,EAAE,MAAM,CAAC;IACV,CAAC,EAAE,MAAM,CAAC;IACV,CAAC,EAAE,MAAM,CAAC;IACV,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,kBAAkB;IACjC,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,UAAU,CAAC,EAAE,UAAU,CAAC;CACzB;AAED,MAAM,WAAW,kBAAkB;IACjC,IAAI,EAAE,MAAM,CAAC;IACb,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,qDAAqD;AACrD,MAAM,WAAW,aAAa;IAC5B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,YAAY;IAC3B,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,WAAW,sBAAsB;IACrC,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,YAAY,CAAC;CACpB;AAED;;;;GAIG;AACH,MAAM,WAAW,aAAa;IAC5B,QAAQ,IAAI,OAAO,CAAC,aAAa,CAAC,CAAC;IACnC,MAAM,CACJ,KAAK,EAAE,MAAM,EACb,MAAM,EAAE,MAAM,EACd,OAAO,CAAC,EAAE,kBAAkB,GAC3B,aAAa,CAAC;IACjB,WAAW,IAAI,aAAa,CAAC;IAC7B,WAAW,IAAI,aAAa,CAAC;IAC7B,OAAO,CAAC,MAAM,EAAE,kBAAkB,GAAG,aAAa,CAAC;IACnD,MAAM,CAAC,OAAO,EAAE,kBAAkB,GAAG,aAAa,CAAC;IACnD,IAAI,IAAI,aAAa,CAAC;IACtB,KAAK,IAAI,aAAa,CAAC;IACvB,GAAG,IAAI,aAAa,CAAC;IACrB,IAAI,IAAI,aAAa,CAAC;IACtB,GAAG,IAAI,aAAa,CAAC;IACrB,QAAQ,IAAI,OAAO,CAAC,MAAM,CAAC,CAAC;IAC5B,QAAQ,CAAC,OAAO,EAAE;QAChB,iBAAiB,EAAE,IAAI,CAAC;KACzB,GAAG,OAAO,CAAC,sBAAsB,CAAC,CAAC;CACrC;AAED,+DAA+D;AAC/D,MAAM,MAAM,YAAY,GAAG,CACzB,KAAK,CAAC,EAAE,MAAM,GAAG,UAAU,EAC3B,OAAO,CAAC,EAAE,mBAAmB,KAC1B,aAAa,CAAC;AAInB;;;;GAIG;AACH,wBAAsB,QAAQ,IAAI,OAAO,CAAC,YAAY,CAAC,CAStD;AAyKD;;;;GAIG;AACH,wBAAgB,cAAc,IAAI,YAAY,CAuB7C"}
@@ -0,0 +1,15 @@
1
+ export declare const MAX_VISION_IMAGE_BYTES: number;
2
+ export interface ValidatedVisionImage {
3
+ readonly width: number;
4
+ readonly height: number;
5
+ readonly format: "jpeg" | "png" | "webp";
6
+ readonly contentType: "image/jpeg" | "image/png" | "image/webp";
7
+ }
8
+ export declare function estimateBase64DecodedBytes(base64: string): number;
9
+ export declare function parseVisionDataImageUrl(value: string): {
10
+ mimeType: string;
11
+ base64: string;
12
+ };
13
+ export declare function assertValidVisionImageBuffer(data: Buffer): Promise<ValidatedVisionImage>;
14
+ export declare function assertSafeVisionDataImageUrl(imageUrl: string): Promise<ValidatedVisionImage>;
15
+ //# sourceMappingURL=image-input.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"image-input.d.ts","sourceRoot":"","sources":["../src/image-input.ts"],"names":[],"mappings":"AAEA,eAAO,MAAM,sBAAsB,QAAmB,CAAC;AASvD,MAAM,WAAW,oBAAoB;IACnC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,MAAM,EAAE,MAAM,GAAG,KAAK,GAAG,MAAM,CAAC;IACzC,QAAQ,CAAC,WAAW,EAAE,YAAY,GAAG,WAAW,GAAG,YAAY,CAAC;CACjE;AAED,wBAAgB,0BAA0B,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAKjE;AAED,wBAAgB,uBAAuB,CAAC,KAAK,EAAE,MAAM,GAAG;IACtD,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,CAAC;CAChB,CA8BA;AAED,wBAAsB,4BAA4B,CAChD,IAAI,EAAE,MAAM,GACX,OAAO,CAAC,oBAAoB,CAAC,CAkC/B;AAED,wBAAsB,4BAA4B,CAChD,QAAQ,EAAE,MAAM,GACf,OAAO,CAAC,oBAAoB,CAAC,CAG/B"}
@@ -0,0 +1,4 @@
1
+ import type { Plugin } from "@elizaos/core";
2
+ export declare const visionPlugin: Plugin;
3
+ export default visionPlugin;
4
+ //# sourceMappingURL=index.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,eAAe,CAAC;AAwB5C,eAAO,MAAM,YAAY,EAAE,MAuH1B,CAAC;AAEF,eAAe,YAAY,CAAC"}