@elizaos/plugin-vision 1.2.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (106) hide show
  1. package/.npmignore +5 -0
  2. package/README.md +270 -0
  3. package/build.config.ts +70 -0
  4. package/dist/action.d.ts +8 -0
  5. package/dist/action.js +1212 -0
  6. package/dist/action.js.map +1 -0
  7. package/dist/audio-capture-stream.d.ts +42 -0
  8. package/dist/audio-capture-stream.js +516 -0
  9. package/dist/audio-capture-stream.js.map +1 -0
  10. package/dist/audio-capture.d.ts +25 -0
  11. package/dist/audio-capture.js +412 -0
  12. package/dist/audio-capture.js.map +1 -0
  13. package/dist/basic.test.d.ts +1 -0
  14. package/dist/basic.test.js +97 -0
  15. package/dist/basic.test.js.map +1 -0
  16. package/dist/config.d.ts +73 -0
  17. package/dist/config.js +254 -0
  18. package/dist/config.js.map +1 -0
  19. package/dist/entity-tracker.d.ts +32 -0
  20. package/dist/entity-tracker.js +361 -0
  21. package/dist/entity-tracker.js.map +1 -0
  22. package/dist/errors.d.ts +67 -0
  23. package/dist/errors.js +395 -0
  24. package/dist/errors.js.map +1 -0
  25. package/dist/face-recognition.d.ts +31 -0
  26. package/dist/face-recognition.js +332 -0
  27. package/dist/face-recognition.js.map +1 -0
  28. package/dist/florence2-local.d.ts +25 -0
  29. package/dist/florence2-local.js +280 -0
  30. package/dist/florence2-local.js.map +1 -0
  31. package/dist/florence2-model.d.ts +36 -0
  32. package/dist/florence2-model.js +503 -0
  33. package/dist/florence2-model.js.map +1 -0
  34. package/dist/index.d.ts +3 -0
  35. package/dist/index.js +73 -0
  36. package/dist/index.js.map +1 -0
  37. package/dist/ocr-service-real.d.ts +32 -0
  38. package/dist/ocr-service-real.js +396 -0
  39. package/dist/ocr-service-real.js.map +1 -0
  40. package/dist/ocr-service.d.ts +28 -0
  41. package/dist/ocr-service.js +216 -0
  42. package/dist/ocr-service.js.map +1 -0
  43. package/dist/provider.d.ts +2 -0
  44. package/dist/provider.js +285 -0
  45. package/dist/provider.js.map +1 -0
  46. package/dist/screen-capture.d.ts +16 -0
  47. package/dist/screen-capture.js +302 -0
  48. package/dist/screen-capture.js.map +1 -0
  49. package/dist/service.d.ts +73 -0
  50. package/dist/service.js +1662 -0
  51. package/dist/service.js.map +1 -0
  52. package/dist/tests/e2e/index.d.ts +8 -0
  53. package/dist/tests/e2e/index.js +33 -0
  54. package/dist/tests/e2e/index.js.map +1 -0
  55. package/dist/tests/e2e/run-local.d.ts +2 -0
  56. package/dist/tests/e2e/run-local.js +166 -0
  57. package/dist/tests/e2e/run-local.js.map +1 -0
  58. package/dist/tests/e2e/screen-vision.d.ts +11 -0
  59. package/dist/tests/e2e/screen-vision.js +384 -0
  60. package/dist/tests/e2e/screen-vision.js.map +1 -0
  61. package/dist/tests/e2e/vision-autonomy.d.ts +11 -0
  62. package/dist/tests/e2e/vision-autonomy.js +375 -0
  63. package/dist/tests/e2e/vision-autonomy.js.map +1 -0
  64. package/dist/tests/e2e/vision-basic.d.ts +11 -0
  65. package/dist/tests/e2e/vision-basic.js +434 -0
  66. package/dist/tests/e2e/vision-basic.js.map +1 -0
  67. package/dist/tests/e2e/vision-capture-log.d.ts +11 -0
  68. package/dist/tests/e2e/vision-capture-log.js +302 -0
  69. package/dist/tests/e2e/vision-capture-log.js.map +1 -0
  70. package/dist/tests/e2e/vision-runtime.d.ts +11 -0
  71. package/dist/tests/e2e/vision-runtime.js +357 -0
  72. package/dist/tests/e2e/vision-runtime.js.map +1 -0
  73. package/dist/tests/e2e/vision-worker-tests.d.ts +11 -0
  74. package/dist/tests/e2e/vision-worker-tests.js +466 -0
  75. package/dist/tests/e2e/vision-worker-tests.js.map +1 -0
  76. package/dist/tests/test-pattern-generator.d.ts +40 -0
  77. package/dist/tests/test-pattern-generator.js +191 -0
  78. package/dist/tests/test-pattern-generator.js.map +1 -0
  79. package/dist/tests.d.ts +3 -0
  80. package/dist/tests.js +11 -0
  81. package/dist/tests.js.map +1 -0
  82. package/dist/types.d.ts +222 -0
  83. package/dist/types.js +16 -0
  84. package/dist/types.js.map +1 -0
  85. package/dist/vision-models.d.ts +47 -0
  86. package/dist/vision-models.js +501 -0
  87. package/dist/vision-models.js.map +1 -0
  88. package/dist/vision-worker-manager.d.ts +61 -0
  89. package/dist/vision-worker-manager.js +668 -0
  90. package/dist/vision-worker-manager.js.map +1 -0
  91. package/dist/workers/florence2-worker-simple.d.ts +13 -0
  92. package/dist/workers/florence2-worker-simple.js +121 -0
  93. package/dist/workers/florence2-worker-simple.js.map +1 -0
  94. package/dist/workers/florence2-worker.d.ts +1 -0
  95. package/dist/workers/florence2-worker.js +328 -0
  96. package/dist/workers/florence2-worker.js.map +1 -0
  97. package/dist/workers/ocr-worker.d.ts +1 -0
  98. package/dist/workers/ocr-worker.js +354 -0
  99. package/dist/workers/ocr-worker.js.map +1 -0
  100. package/dist/workers/screen-capture-worker.d.ts +1 -0
  101. package/dist/workers/screen-capture-worker.js +427 -0
  102. package/dist/workers/screen-capture-worker.js.map +1 -0
  103. package/dist/workers/worker-logger.d.ts +9 -0
  104. package/dist/workers/worker-logger.js +95 -0
  105. package/dist/workers/worker-logger.js.map +1 -0
  106. package/package.json +100 -0
package/dist/vision-worker-manager.js.map ADDED
@@ -0,0 +1 @@
1
+ {"version":3,"file":"vision-worker-manager.js","sourceRoot":"","sources":["../src/vision-worker-manager.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,iDAAwC;AACxC,2BAA6B;AAC7B,6BAAmC;AACnC,sCAAuC;AAevC;IA6CE,6BAAY,MAAoB;QA1ChC,UAAU;QACF,wBAAmB,GAAkB,IAAI,CAAC;QAC1C,oBAAe,GAAkB,IAAI,CAAC;QACtC,cAAS,GAAkB,IAAI,CAAC;QAaxC,eAAe;QACE,uBAAkB,GAAG,EAAE,GAAG,IAAI,GAAG,IAAI,CAAC,CAAC,qBAAqB;QAC5D,2BAAsB,GAAG,EAAE,GAAG,IAAI,GAAG,IAAI,CAAC,CAAC,mBAAmB;QAC9D,qBAAgB,GAAG,CAAC,GAAG,IAAI,GAAG,IAAI,CAAC,CAAC,mBAAmB;QAExE,iBAAiB;QACA,mBAAc,GAAG,CAAC,CAAC;QACnB,gBAAW,GAAG,CAAC,CAAC;QAChB,iBAAY,GAAG,CAAC,CAAC;QACjB,kBAAa,GAAG,CAAC,CAAC;QAClB,oBAAe,GAAG,CAAC,CAAC;QACpB,gBAAW,GAAG,EAAE,CAAC;QAElC,eAAe;QACP,gBAAW,GAAG,IAAI,GAAG,EAAuB,CAAC;QAErD,8BAA8B;QACtB,wBAAmB,GAAyB,IAAI,CAAC;QACjD,2BAAsB,GAAG,IAAI,GAAG,EAA2B,CAAC;QAC5D,oBAAe,GAAqB,IAAI,CAAC;QACzC,yBAAoB,GAAG,CAAC,CAAC,CAAC;QAElC,0BAA0B;QAClB,oBAAe,GAAG,IAAI,GAAG,EAAkB,CAAC;QACnC,yBAAoB,GAAG,CAAC,CAAC;QAGxC,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QAErB,4BAA4B;QAC5B,IAAI,CAAC,YAAY,GAAG,IAAI,iBAAiB,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC;QACnE,IAAI,CAAC,sBAAsB,GAAG,IAAI,iBAAiB,CAAC,IAAI,CAAC,sBAAsB,CAAC,CAAC;QACjF,IAAI,CAAC,gBAAgB,GAAG,IAAI,iBAAiB,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;QAErE,eAAe;QACf,IAAI,CAAC,iBAAiB,GAAG,IAAI,UAAU,CAAC,IAAI,CAAC,YAAY,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;QACjE,IAAI,CAAC,cAAc,GAAG,IAAI,QAAQ,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;QACtD,IAAI,CAAC,oBAAoB,GAAG,IAAI,QAAQ,CAAC,IAAI,CAAC,sBAAsB,CAAC,CAAC;QACtE,IAAI,CAAC,cAAc,GAAG,IAAI,QAAQ,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;IAC5D,CAAC;IAEK,wCAAU,GAAhB;;;;;;wBACE,aAAM,CAAC,IAAI,CAAC,sDAAsD,CAAC,CAAC;;;;wBAGlE,8BAA8B;wBAC9B,qBAAM,IAAI,CAAC,wBAAwB,EAAE,EAAA;;wBADrC,8BAA8B;wBAC9B,SAAqC,CAAC;6BAGlC,IAAI,CAAC,MAAM,CAAC,gBAAgB,EAA5B,wBAA4B;wBAC9B,qBAAM,IAAI,CAAC,oBAAoB,EAAE,EAAA;;wBAAjC,SAAiC,CAAC;;;6BAIhC,IAAI,CAAC,MAAM,CAAC,UAAU,EAAtB,wBAAsB;wBACxB,qBAAM,IAAI,CAAC,cAAc,EAAE,EAAA;;wBAA3B,SAA2B,CAAC;;;wBAG9B,aAAM,CAAC,IAAI,CAAC,+CAA+C,CAAC,CAAC;;;;wBAE7D,aAAM,CAAC,KAAK,CAAC,qDAAqD,EAAE,OAAK,CAAC,CAAC;wBAC3E,MAAM,OAAK,CAAC;;;;;KAEf;IAEa,sDAAwB,GAAtC;;;;;gBACQ,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,SAAS,EAAE,0BAA0B,CAAC,CAAC;gBAE/E,IAAI,CAAC,mBAAmB,GAAG,IAAI,uBAAM,CAAC,UAAU,EAAE;oBAChD,UAAU,EAAE;wBACV,MAAM,EAAE;4BACN,YAAY,EAAE,IAAI,CAAC,MAAM,CAAC,YAAY;4BACtC,kBAAkB,EAAE,IAAI,CAAC,MAAM,CAAC,kBAAkB;4BAClD,SAAS,EAAE,IAAI,CAAC,MAAM,CAAC,eAAe;4BACtC,gBAAgB,EAAE,IAAI,CAAC,kBAAkB;yBAC1C;wBACD,YAAY,EAAE,IAAI,CAAC,YAAY;qBAChC;iBACF,CAAC,CAAC;gBAEH,IAAI,CAAC,mBAAmB,CAAC,EAAE,CAAC,SAAS,EAAE,UAAC,GAAG;oBACzC,IAAI,GAAG,CAAC,IAAI,KAAK,KAAK,EAAE,CAAC;wBACvB,KAAI,CAAC,WAAW,CAAC,GAAG,CAAC,eAAe,EAAE;4BACpC,GAAG,EAAE,GAAG,CAAC,GAAG;4BACZ,UAAU,EAAE,GAAG,CAAC,UAAU;4BAC1B,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE;yBACvB,CAAC,CAAC;oBACL,CAAC;yBAAM,IAAI,GAAG,CAAC,IAAI,KAAK,OAAO,EAAE,CAAC;wBAChC,aAAM,CAAC,KAAK,CAAC,8BAA8B,EAAE,GAAG,CAAC,KAAK,CAAC,CAAC;oBAC1D,CAAC;yBAAM,IAAI,GAAG,CAAC,IAAI,KAAK,KAAK,EAAE,CAAC;wBAC9B,6BAA6B;wBAC7B,KAAI,CAAC,eAAe,CAAC,qBAAqB,EAAE,GAAG,CAAC,CAAC;oBACnD,CAAC;gBACH,CAAC,CAAC,CAAC;gBAEH,IAAI,CAAC,mBAAmB,CAAC,EAAE,CAAC,OAAO,EAAE,UAAC,KAAK;oBACzC,aAAM,CAAC,KAAK,CAAC,qCAAqC,EAAE,KAAK,CAAC,CAAC;oBAC3D,wCAAwC;oBACxC,UAAU,CAAC,cAAM,OAAA,KAAI,CAAC,0BAA0B,EAAE,EAAjC,CAAiC,EAAE,IAAI,CAAC,CAAC;gBAC5D,CAAC,CAAC,CAAC;gBAEH,IAAI,CAAC,mBAAmB,CAAC,EAAE,CAAC,MAAM,EAAE,UAAC,IAAI;oBACvC,IAAI,IAAI,KAAK,CAAC,EAAE,CAAC;wBACf,aAAM,CAAC,KAAK,CAAC,8DAAuD,IAAI,CAAE,CAAC,CAAC;wBAC5E,wCAAwC;wBACxC,UAAU,CAAC,cAAM,OAAA,KAAI,CAAC,0BAA0B,EAAE,EAAjC,CAAiC,EAAE,IAAI,CAAC,CAAC;oBAC5D,CAAC;gBACH,CAAC,CAAC,CAAC;;;;KACJ;IAEa,kDAAoB,GAAlC;;;;;gBACQ,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,SAAS,EAAE,qBAAqB,CAAC,CAAC;gBAGpE,aAAa,GAAa,EAAE,CAAC;gBAC7B,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC,QAAQ,IAAI,GAAG,CAAC;gBACvC,aAAa,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,GAAG,QAAQ,CAAC,CAAC;gBAC3C,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,aAAa,GAAG,CAAC,CAAC,CAAC;gBAC1C,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,aAAa,GAAG,CAAC,CAAC,CAAC;gBAEhD,4CAA4C;gBAC5C,KAAS,CAAC,GAAG,SAAS,GAAG,CAAC,EAAE,CAAC,IAAI,SAAS,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;oBACpD,KAAS,CAAC,GAAG,SAAS,GAAG,CAAC,EAAE,CAAC,IAAI,SAAS,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;wBACpD,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;4BACrB,aAAa,CAAC,IAAI,CAAC,CAAC,GAAG,aAAa,GAAG,CAAC,CAAC,CAAC;wBAC5C,CAAC;oBACH,CAAC;gBACH,CAAC;gBAED,IAAI,CAAC,eAAe,GAAG,IAAI,uBAAM,CAAC,UAAU,EAAE;oBAC5C,UAAU,EAAE;wBACV,MAAM,EAAE;4BACN,QAAQ,EAAE,IAAI,CAAC,MAAM,CAAC,QAAQ,IAAI,GAAG;4BACrC,aAAa,eAAA;yBACd;wBACD,YAAY,EAAE,IAAI,CAAC,YAAY;wBAC/B,aAAa,EAAE,IAAI,CAAC,sBAAsB;qBAC3C;iBACF,CAAC,CAAC;gBAEH,IAAI,CAAC,eAAe,CAAC,EAAE,CAAC,SAAS,EAAE,UAAC,GAAG;oBACrC,IAAI,GAAG,CAAC,IAAI,KAAK,KAAK,EAAE,CAAC;wBACvB,KAAI,CAAC,WAAW,CAAC,GAAG,CAAC,WAAW,EAAE;4BAChC,GAAG,EAAE,GAAG,CAAC,GAAG;4BACZ,UAAU,EAAE,GAAG,CAAC,UAAU;4BAC1B,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE;yBACvB,CAAC,CAAC;oBACL,CAAC;yBAAM,IAAI,GAAG,CAAC,IAAI,KAAK,eAAe,EAAE,CAAC;wBACxC,8BAA8B;wBAC9B,KAAI,CAAC,oBAAoB,CAAC,GAAG,CAAC,CAAC;oBACjC,CAAC;yBAAM,IAAI,GAAG,CAAC,IAAI,KAAK,OAAO,EAAE,CAAC;wBAChC,aAAM,CAAC,KAAK,CAAC,0BAA0B,EAAE,GAAG,CAAC,KAAK,CAAC,CAAC;oBACtD,CAAC;yBAAM,IAAI,GAAG,CAAC,IAAI,KAAK,KAAK,EAAE,CAAC;wBAC9B,6BAA6B;wBAC7B,KAAI,CAAC,eAAe,CAAC,iBAAiB,EAAE,GAAG,CAAC,CAAC;oBAC/C,CAAC;gBACH,CAAC,CAAC,CAAC;gBAEH,IAAI,CAAC,eAAe,CAAC,EAAE,CAAC,OAAO,EAAE,UAAC,KAAK;oBACrC,aAAM,CAAC,KAAK,CAAC,iCAAiC,EAAE,KAAK,CAAC,CAAC;oBACvD,wCAAwC;oBACxC,UAAU,CAAC,cAAM,OAAA,KAAI,CAAC,sBAAsB,EAAE,EAA7B,CAA6B,EAAE,IAAI,CAAC,CAAC;gBACxD,CAAC,CAAC,CAAC;gBAEH,IAAI,CAAC,eAAe,CAAC,EAAE,CAAC,MAAM,EAAE,UAAC,IAAI;oBACnC,IAAI,IAAI,KAAK,CAAC,EAAE,CAAC;wBACf,aAAM,CAAC,KAAK,CAAC,0DAAmD,IAAI,CAAE,CAAC,CAAC;wBACxE,wCAAwC;wBACxC,UAAU,CAAC,cAAM,OAAA,KAAI,CAAC,sBAAsB,EAAE,EAA7B,CAA6B,EAAE,IAAI,CAAC,CAAC;oBACxD,CAAC;gBACH,CAAC,CAAC,CAAC;;;;KACJ;IAEa,4CAAc,GAA5B;;;;;gBACQ,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,SAAS,EAAE,eAAe,CAAC,CAAC;gBAEpE,IAAI,CAAC,SAAS,GAAG,IAAI,uBAAM,CAAC,UAAU,EAAE;oBACtC,UAAU,EAAE;wBACV,MAAM,EAAE;4BACN,iBAAiB,EAAE,IAAI;4BACvB,QAAQ,EAAE,IAAI,CAAC,MAAM,CAAC,QAAQ,IAAI,GAAG;4BACrC,WAAW,EAAE,IAAI,CAAC,MAAM,CAAC,WAAW;yBACrC;wBACD,YAAY,EAAE,IAAI,CAAC,YAAY;wBAC/B,aAAa,EAAE,IAAI,CAAC,gBAAgB;qBACrC;iBACF,CAAC,CAAC;gBAEH,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC,SAAS,EAAE,UAAC,GAAG;oBAC/B,IAAI,GAAG,CAAC,IAAI,KAAK,KAAK,EAAE,CAAC;wBACvB,KAAI,CAAC,WAAW,CAAC,GAAG,CAAC,KAAK,EAAE;4BAC1B,GAAG,EAAE,GAAG,CAAC,GAAG;4BACZ,UAAU,EAAE,GAAG,CAAC,UAAU;4BAC1B,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE;yBACvB,CAAC,CAAC;oBACL,CAAC;yBAAM,IAAI,GAAG,CAAC,IAAI,KAAK,cAAc,EAAE,CAAC;wBACvC,0BAA0B;wBAC1B,KAAI,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC;oBAC3B,CAAC;yBAAM,IAAI,GAAG,CAAC,IAAI,KAAK,OAAO,EAAE,CAAC;wBAChC,aAAM,CAAC,KAAK,CAAC,oBAAoB,EAAE,GAAG,CAAC,KAAK,CAAC,CAAC;oBAChD,CAAC;yBAAM,IAAI,GAAG,CAAC,IAAI,KAAK,KAAK,EAAE,CAAC;wBAC9B,6BAA6B;wBAC7B,KAAI,CAAC,eAAe,CAAC,WAAW,EAAE,GAAG,CAAC,CAAC;oBACzC,CAAC;gBACH,CAAC,CAAC,CAAC;gBAEH,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC,OAAO,EAAE,UAAC,KAAK;oBAC/B,aAAM,CAAC,KAAK,CAAC,2BAA2B,EAAE,KAAK,CAAC,CAAC;oBACjD,wCAAwC;oBACxC,UAAU,CAAC,cAAM,OAAA,KAAI,CAAC,gBAAgB,EAAE,EAAvB,CAAuB,EAAE,IAAI,CAAC,CAAC;gBAClD,CAAC,CAAC,CAAC;gBAEH,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC,MAAM,EAAE,UAAC,IAAI;oBAC7B,IAAI,IAAI,KAAK,CAAC,EAAE,CAAC;wBACf,aAAM,CAAC,KAAK,CAAC,oDAA6C,IAAI,CAAE,CAAC,CAAC;wBAClE,wCAAwC;wBACxC,UAAU,CAAC,cAAM,OAAA,KAAI,CAAC,gBAAgB,EAAE,EAAvB,CAAuB,EAAE,IAAI,CAAC,CAAC;oBAClD,CAAC;gBACH,CAAC,CAAC,CAAC;;;;KACJ;IAEO,kDAAoB,GAA5B,UAA6B,GAAQ;QACnC,iCAAiC;QACjC,IAAI,CAAC;YACH,IAAM,MAAM,GAAG,GAAG,CAAC,MAAM,CAAC;YAC1B,IAAM,MAAM,GAAG,IAAI,CAAC,mBAAmB,CAAC,MAAM,CAAC,CAAC;YAChD,IAAI,MAAM,EAAE,CAAC;gBACX,IAAI,CAAC,sBAAsB,CAAC,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;YAClD,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,aAAM,CAAC,KAAK,CAAC,yDAAyD,EAAE,KAAK,CAAC,CAAC;QACjF,CAAC;IACH,CAAC;IAEO,4CAAc,GAAtB,UAAuB,IAAS;QAC9B,qCAAqC;QACrC,IAAI,CAAC;YACH,IAAM,MAAM,GAAG,IAAI,CAAC,aAAa,EAAE,CAAC;YACpC,IAAI,MAAM,EAAE,CAAC;gBACX,IAAI,CAAC,eAAe,GAAG,MAAM,CAAC;YAChC,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,aAAM,CAAC,KAAK,CAAC,mDAAmD,EAAE,KAAK,CAAC,CAAC;QAC3E,CAAC;IACH,CAAC;IAEO,iDAAmB,GAA3B,UAA4B,MAAc;QACxC,IAAI,CAAC;YACH,+BAA+B;YAC/B,IAAM,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,kBAAkB,CAAC,CAAC;YAC/C,IAAI,CAAC,KAAK,EAAE,CAAC;gBACX,OAAO,IAAI,CAAC;YACd,CAAC;YAED,IAAM,GAAG,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YACnC,IAAM,GAAG,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YACnC,IAAM,SAAS,GAAG,GAAG,GAAG,EAAE,GAAG,GAAG,CAAC,CAAC,0BAA0B;YAE5D,IAAM,mBAAmB,GAAG,EAAE,CAAC;YAC/B,IAAM,eAAe,GAAG,IAAI,CAAC;YAC7B,IAAM,MAAM,GAAG,mBAAmB,GAAG,SAAS,GAAG,eAAe,CAAC;YAEjE,cAAc;YACd,IAAM,QAAM,GAAG,IAAI,CAAC,oBAAoB,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;YACjE,IAAI,QAAM,KAAK,CAAC,IAAI,QAAM,GAAG,eAAe,GAAG,CAAC,EAAE,CAAC;gBACjD,OAAO,IAAI,CAAC;YACd,CAAC;YAED,iBAAiB;YACjB,IAAM,KAAK,GAAG,IAAI,UAAU,CAAC,QAAM,CAAC,CAAC;YACrC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,QAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBAChC,KAAK,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,oBAAoB,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;YAChE,CAAC;YAED,IAAM,IAAI,GAAG,IAAI,kBAAW,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YAC7C,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAC1B,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,aAAM,CAAC,KAAK,CAAC,wDAAwD,EAAE,KAAK,CAAC,CAAC;YAC9E,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;IAEO,2CAAa,GAArB;QACE,IAAI,CAAC;YACH,IAAM,mBAAmB,GAAG,EAAE,CAAC;YAC/B,IAAM,MAAM,GAAG,mBAAmB,CAAC;YAEnC,cAAc;YACd,IAAM,QAAM,GAAG,IAAI,CAAC,cAAc,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;YAC3D,IAAI,QAAM,KAAK,CAAC,EAAE,CAAC;gBACjB,OAAO,IAAI,CAAC;YACd,CAAC;YAED,8BAA8B;YAC9B,IAAM,QAAQ,GAAG,IAAI,CAAC,cAAc,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,IAAI,CAAC,CAAC;YACjE,IAAM,UAAU,GAAG,IAAI,CAAC,cAAc,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,IAAI,CAAC,CAAC;YAEpE,iBAAiB;YACjB,IAAM,UAAU,GAAG,MAAM,GAAG,EAAE,CAAC;YAC/B,IAAM,KAAK,GAAG,IAAI,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,QAAM,EAAE,KAAK,CAAC,CAAC,CAAC;YACtD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBACtC,KAAK,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,cAAc,CAAC,QAAQ,CAAC,UAAU,GAAG,CAAC,CAAC,CAAC;YAC1D,CAAC;YAED,IAAM,IAAI,GAAG,IAAI,kBAAW,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YAC7C,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAC1B,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,aAAM,CAAC,KAAK,CAAC,kDAAkD,EAAE,KAAK,CAAC,CAAC;YACxE,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;IAED,kDAAkD;IAElD,oDAAsB,GAAtB;QACE,8BAA8B;QAC9B,IAAM,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,iBAAiB,EAAE,IAAI,CAAC,cAAc,CAAC,CAAC;QAE1E,IAAI,OAAO,IAAI,IAAI,CAAC,oBAAoB,EAAE,CAAC;YACzC,OAAO,IAAI,CAAC,mBAAmB,CAAC;QAClC,CAAC;QAED,IAAI,CAAC;YACH,IAAM,KAAK,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,iBAAiB,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;YACrE,IAAM,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,iBAAiB,EAAE,IAAI,CAAC,YAAY,CAAC,CAAC;YACvE,IAAM,aAAa,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,iBAAiB,EAAE,IAAI,CAAC,aAAa,CAAC,CAAC;YAC/E,IAAM,SAAS,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,iBAAiB,EAAE,IAAI,CAAC,eAAe,CAAC,CAAC;YAE7E,+BAA+B;YAC/B,IAAI,CAAC,mBAAmB,GAAG;gBACzB,SAAS,WAAA;gBACT,KAAK,OAAA;gBACL,MAAM,QAAA;gBACN,IAAI,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,8CAA8C;gBACrE,KAAK,EAAE,IAAI,CAAC,aAAa,CAAC,KAAK,EAAE,MAAM,CAAC;aACzC,CAAC;YAEF,IAAI,CAAC,oBAAoB,GAAG,OAAO,CAAC;QACtC,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,aAAM,CAAC,KAAK,CAAC,sDAAsD,EAAE,KAAK,CAAC,CAAC;QAC9E,CAAC;QAED,OAAO,IAAI,CAAC,mBAAmB,CAAC;IAClC,CAAC;IAED,oDAAsB,GAAtB;;QACE,IAAM,aAAa,GAAG,IAAI,CAAC,sBAAsB,EAAE,CAAC;QAEpD,iCAAiC;QACjC,IAAM,iBAAiB,GAAa,EAAE,CAAC;QACvC,IAAM,UAAU,GAAU,EAAE,CAAC;QAC7B,IAAM,OAAO,GAAG,IAAI,GAAG,EAAU,CAAC;QAElC,IAAI,CAAC,sBAAsB,CAAC,OAAO,CAAC,UAAC,MAAM;YACzC,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;gBACnB,iBAAiB,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;YACzC,CAAC;YACD,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;gBACnB,UAAU,CAAC,IAAI,OAAf,UAAU,EAAS,MAAM,CAAC,OAAO,EAAE;YACrC,CAAC;YACD,IAAI,MAAM,CAAC,IAAI,EAAE,CAAC;gBAChB,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,UAAC,GAAG,IAAK,OAAA,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,EAAhB,CAAgB,CAAC,CAAC;YACjD,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,IAAM,MAAM,GAAG,IAAI,CAAC,aAAa,CAAC,CAAA,aAAa,aAAb,aAAa,uBAAb,aAAa,CAAE,KAAK,KAAI,CAAC,EAAE,CAAA,aAAa,aAAb,aAAa,uBAAb,aAAa,CAAE,MAAM,KAAI,CAAC,CAAC,CAAC;QAEzF,OAAO;YACL,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;YACrB,WAAW,EAAE,iBAAiB,CAAC,IAAI,CAAC,IAAI,CAAC;YACzC,OAAO,EAAE,UAAU;YACnB,MAAM,EAAE,EAAE,EAAE,0CAA0C;YACtD,YAAY,EAAE,IAAI;YAClB,gBAAgB,EAAE,GAAG;YACrB,aAAa,EAAE,IAAI,CAAC,mBAAmB,IAAI,SAAS;YACpD,cAAc,EAAE;gBACd,aAAa,EAAE,MAAA,IAAI,CAAC,eAAe,0CAAE,QAAQ;gBAC7C,UAAU,EAAE;oBACV,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;oBACrB,SAAS,EAAE,IAAI,CAAC,sBAAsB,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,CAAC,KAAK;oBAC5D,GAAG,EAAE,IAAI,CAAC,eAAe,IAAI,SAAS;iBACvC;gBACD,WAAW,EAAE,UAAG,CAAA,aAAa,aAAb,aAAa,uBAAb,aAAa,CAAE,KAAK,CAAC,MAAM,KAAI,CAAC,oBAAiB;gBACjE,UAAU,EAAE,UAAU,CAAC,GAAG,CAAC,UAAC,GAAG,IAAK,OAAA,CAAC;oBACnC,IAAI,EAAE,GAAG,CAAC,KAAK;oBACf,IAAI,EAAE,EAAE;oBACR,QAAQ,EAAE,GAAG,CAAC,IAAI;iBACnB,CAAC,EAJkC,CAIlC,CAAC;aACJ;SACF,CAAC;IACJ,CAAC;IAEO,2CAAa,GAArB,UACE,KAAa,EACb,MAAc;QAUd,IAAM,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC,QAAQ,IAAI,GAAG,CAAC;QAC7C,IAAM,KAAK,GAQN,EAAE,CAAC;QAER,KAAK,IAAI,GAAG,GAAG,CAAC,EAAE,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,QAAQ,CAAC,EAAE,GAAG,EAAE,EAAE,CAAC;YAC5D,KAAK,IAAI,GAAG,GAAG,CAAC,EAAE,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,GAAG,QAAQ,CAAC,EAAE,GAAG,EAAE,EAAE,CAAC;gBAC3D,IAAM,CAAC,GAAG,GAAG,GAAG,QAAQ,CAAC;gBACzB,IAAM,CAAC,GAAG,GAAG,GAAG,QAAQ,CAAC;gBACzB,KAAK,CAAC,IAAI,CAAC;oBACT,EAAE,EAAE,eAAQ,GAAG,cAAI,GAAG,CAAE;oBACxB,GAAG,KAAA;oBACH,GAAG,KAAA;oBACH,CAAC,GAAA;oBACD,CAAC,GAAA;oBACD,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,QAAQ,EAAE,KAAK,GAAG,CAAC,CAAC;oBACpC,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,QAAQ,EAAE,MAAM,GAAG,CAAC,CAAC;iBACvC,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QAED,OAAO,KAAK,CAAC;IACf,CAAC;IAED,4CAAc,GAAd;QACE,OAAO,IAAI,GAAG,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;IACnC,CAAC;IAEK,6CAAe,GAArB,UAAsB,KAAa;;;gBACjC,IAAI,IAAI,CAAC,mBAAmB,EAAE,CAAC;oBAC7B,IAAI,CAAC,mBAAmB,CAAC,WAAW,CAAC;wBACnC,IAAI,EAAE,aAAa;wBACnB,YAAY,EAAE,KAAK;qBACpB,CAAC,CAAC;gBACL,CAAC;;;;KACF;IAEK,4CAAc,GAApB,UACE,OAAuE;;;gBAEvE,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;oBACnB,IAAI,CAAC,SAAS,CAAC,WAAW,CAAC;wBACzB,IAAI,EAAE,gBAAgB;wBACtB,OAAO,SAAA;qBACR,CAAC,CAAC;gBACL,CAAC;;;;KACF;IAEK,kCAAI,GAAV;;;;;;;wBACE,aAAM,CAAC,IAAI,CAAC,+CAA+C,CAAC,CAAC;wBAEvD,YAAY,GAAoB,EAAE,CAAC;wBAEzC,IAAI,IAAI,CAAC,mBAAmB,EAAE,CAAC;4BAC7B,YAAY,CAAC,IAAI,CACf,IAAI,OAAO,CAAC,UAAC,OAAO;gCAClB,KAAI,CAAC,mBAAoB,CAAC,IAAI,CAAC,MAAM,EAAE,cAAM,OAAA,OAAO,EAAE,EAAT,CAAS,CAAC,CAAC;gCACxD,KAAI,CAAC,mBAAoB,CAAC,WAAW,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC,CAAC;4BAC1D,CAAC,CAAC,CACH,CAAC;wBACJ,CAAC;wBAED,IAAI,IAAI,CAAC,eAAe,EAAE,CAAC;4BACzB,YAAY,CAAC,IAAI,CACf,IAAI,OAAO,CAAC,UAAC,OAAO;gCAClB,KAAI,CAAC,eAAgB,CAAC,IAAI,CAAC,MAAM,EAAE,cAAM,OAAA,OAAO,EAAE,EAAT,CAAS,CAAC,CAAC;gCACpD,KAAI,CAAC,eAAgB,CAAC,WAAW,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC,CAAC;4BACtD,CAAC,CAAC,CACH,CAAC;wBACJ,CAAC;wBAED,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;4BACnB,YAAY,CAAC,IAAI,CACf,IAAI,OAAO,CAAC,UAAC,OAAO;gCAClB,KAAI,CAAC,SAAU,CAAC,IAAI,CAAC,MAAM,EAAE,cAAM,OAAA,OAAO,EAAE,EAAT,CAAS,CAAC,CAAC;gCAC9C,KAAI,CAAC,SAAU,CAAC,WAAW,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC,CAAC;4BAChD,CAAC,CAAC,CACH,CAAC;wBACJ,CAAC;wBAED,qBAAM,OAAO,CAAC,GAAG,CAAC,YAAY,CAAC,EAAA;;wBAA/B,SAA+B,CAAC;wBAChC,aAAM,CAAC,IAAI,CAAC,2CAA2C,CAAC,CAAC;;;;;KAC1D;IAEO,6CAAe,GAAvB,UAAwB,UAAkB,EAAE,GAAQ;QAC1C,IAAA,KAAK,GAAoB,GAAG,MAAvB,EAAE,OAAO,GAAW,GAAG,QAAd,EAAE,IAAI,GAAK,GAAG,KAAR,CAAS;QACrC,IAAM,gBAAgB,GAAG,WAAI,UAAU,eAAK,OAAO,CAAE,CAAC;QAEtD,QAAQ,KAAK,EAAE,CAAC;YACd,KAAK,MAAM;gBACT,aAAM,CAAC,IAAI,OAAX,aAAM,iBAAM,gBAAgB,GAAK,IAAI,UAAE;gBACvC,MAAM;YACR,KAAK,MAAM;gBACT,aAAM,CAAC,IAAI,OAAX,aAAM,iBAAM,gBAAgB,GAAK,IAAI,UAAE;gBACvC,MAAM;YACR,KAAK,OAAO;gBACV,aAAM,CAAC,KAAK,OAAZ,aAAM,iBAAO,gBAAgB,GAAK,IAAI,UAAE;gBACxC,MAAM;YACR,KAAK,OAAO;gBACV,aAAM,CAAC,KAAK,OAAZ,aAAM,iBAAO,gBAAgB,GAAK,IAAI,UAAE;gBACxC,MAAM;QACV,CAAC;IACH,CAAC;IAEa,wDAA0B,GAAxC;;;;;;wBACQ,QAAQ,GAAG,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC;wBAEhE,IAAI,QAAQ,IAAI,IAAI,CAAC,oBAAoB,EAAE,CAAC;4BAC1C,aAAM,CAAC,KAAK,CAAC,8EAA8E,CAAC,CAAC;4BAC7F,sBAAO;wBACT,CAAC;wBAED,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,eAAe,EAAE,QAAQ,GAAG,CAAC,CAAC,CAAC;wBACxD,aAAM,CAAC,IAAI,CAAC,0EAAmE,QAAQ,GAAG,CAAC,MAAG,CAAC,CAAC;;;;wBAG9F,2BAA2B;wBAC3B,IAAI,IAAI,CAAC,mBAAmB,EAAE,CAAC;4BAC7B,IAAI,CAAC,mBAAmB,CAAC,kBAAkB,EAAE,CAAC;4BAC9C,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC;wBAClC,CAAC;wBAED,mBAAmB;wBACnB,qBAAM,IAAI,CAAC,wBAAwB,EAAE,EAAA;;wBADrC,mBAAmB;wBACnB,SAAqC,CAAC;wBAEtC,4CAA4C;wBAC5C,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,eAAe,EAAE,CAAC,CAAC,CAAC;;;;wBAE7C,aAAM,CAAC,KAAK,CAAC,gEAAgE,EAAE,OAAK,CAAC,CAAC;;;;;;KAEzF;IAEa,oDAAsB,GAApC;;;;;;wBACQ,QAAQ,GAAG,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;wBAE5D,IAAI,QAAQ,IAAI,IAAI,CAAC,oBAAoB,EAAE,CAAC;4BAC1C,aAAM,CAAC,KAAK,CAAC,yEAAyE,CAAC,CAAC;4BACxF,sBAAO;wBACT,CAAC;wBAED,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,WAAW,EAAE,QAAQ,GAAG,CAAC,CAAC,CAAC;wBACpD,aAAM,CAAC,IAAI,CAAC,qEAA8D,QAAQ,GAAG,CAAC,MAAG,CAAC,CAAC;;;;wBAGzF,2BAA2B;wBAC3B,IAAI,IAAI,CAAC,eAAe,EAAE,CAAC;4BACzB,IAAI,CAAC,eAAe,CAAC,kBAAkB,EAAE,CAAC;4BAC1C,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC;wBAC9B,CAAC;wBAED,mBAAmB;wBACnB,qBAAM,IAAI,CAAC,oBAAoB,EAAE,EAAA;;wBADjC,mBAAmB;wBACnB,SAAiC,CAAC;wBAElC,4CAA4C;wBAC5C,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC;;;;wBAEzC,aAAM,CAAC,KAAK,CAAC,2DAA2D,EAAE,OAAK,CAAC,CAAC;;;;;;KAEpF;IAEa,8CAAgB,GAA9B;;;;;;wBACQ,QAAQ,GAAG,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;wBAEtD,IAAI,QAAQ,IAAI,IAAI,CAAC,oBAAoB,EAAE,CAAC;4BAC1C,aAAM,CAAC,KAAK,CAAC,mEAAmE,CAAC,CAAC;4BAClF,sBAAO;wBACT,CAAC;wBAED,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,KAAK,EAAE,QAAQ,GAAG,CAAC,CAAC,CAAC;wBAC9C,aAAM,CAAC,IAAI,CAAC,+DAAwD,QAAQ,GAAG,CAAC,MAAG,CAAC,CAAC;;;;wBAGnF,2BAA2B;wBAC3B,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;4BACnB,IAAI,CAAC,SAAS,CAAC,kBAAkB,EAAE,CAAC;4BACpC,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;wBACxB,CAAC;wBAED,mBAAmB;wBACnB,qBAAM,IAAI,CAAC,cAAc,EAAE,EAAA;;wBAD3B,mBAAmB;wBACnB,SAA2B,CAAC;wBAE5B,4CAA4C;wBAC5C,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;;;;wBAEnC,aAAM,CAAC,KAAK,CAAC,qDAAqD,EAAE,OAAK,CAAC,CAAC;;;;;;KAE9E;IACH,0BAAC;AAAD,CAAC,AAxmBD,IAwmBC;AAxmBY,kDAAmB","sourcesContent":["import { Worker } from 'worker_threads';\nimport * as path from 'path';\nimport { TextDecoder } from 'util';\nimport { logger } from '@elizaos/core';\nimport type {\n VisionConfig,\n ScreenCapture,\n Florence2Result,\n OCRResult,\n EnhancedSceneDescription,\n} from './types';\n\ninterface WorkerStats {\n fps: number;\n frameCount: number;\n lastUpdate: number;\n}\n\nexport class VisionWorkerManager {\n private config: VisionConfig;\n\n // Workers\n private screenCaptureWorker: Worker | null = null;\n private florence2Worker: Worker | null = null;\n private ocrWorker: Worker | null = null;\n\n // Shared buffers\n private screenBuffer: SharedArrayBuffer;\n private florence2ResultsBuffer: SharedArrayBuffer;\n private ocrResultsBuffer: SharedArrayBuffer;\n\n // Buffer views\n private screenAtomicState: Int32Array;\n private screenDataView: DataView;\n private florence2ResultsView: DataView;\n private ocrResultsView: DataView;\n\n // Buffer sizes\n private readonly SCREEN_BUFFER_SIZE = 50 * 1024 * 1024; // 50MB for 4K screen\n private readonly FLORENCE2_RESULTS_SIZE = 10 * 1024 * 1024; // 10MB for results\n private readonly OCR_RESULTS_SIZE = 5 * 1024 * 1024; // 5MB for OCR text\n\n // Atomic indices\n private readonly FRAME_ID_INDEX = 0;\n private readonly WIDTH_INDEX = 2;\n private readonly HEIGHT_INDEX = 3;\n private readonly DISPLAY_INDEX = 4;\n private readonly TIMESTAMP_INDEX = 5;\n private readonly DATA_OFFSET = 24;\n\n // Worker stats\n private workerStats = new Map<string, WorkerStats>();\n\n // Latest processed data cache\n private latestScreenCapture: ScreenCapture | null = null;\n private latestFlorence2Results = new Map<string, Florence2Result>();\n private latestOCRResult: OCRResult | null = null;\n private lastProcessedFrameId = -1;\n\n // Worker restart tracking\n private restartAttempts = new Map<string, number>();\n private readonly MAX_RESTART_ATTEMPTS = 3;\n\n constructor(config: VisionConfig) {\n this.config = config;\n\n // Initialize shared buffers\n this.screenBuffer = new SharedArrayBuffer(this.SCREEN_BUFFER_SIZE);\n this.florence2ResultsBuffer = new SharedArrayBuffer(this.FLORENCE2_RESULTS_SIZE);\n this.ocrResultsBuffer = new SharedArrayBuffer(this.OCR_RESULTS_SIZE);\n\n // Create views\n this.screenAtomicState = new Int32Array(this.screenBuffer, 0, 6);\n this.screenDataView = new DataView(this.screenBuffer);\n this.florence2ResultsView = new DataView(this.florence2ResultsBuffer);\n this.ocrResultsView = new DataView(this.ocrResultsBuffer);\n }\n\n async initialize(): Promise<void> {\n logger.info('[VisionWorkerManager] Initializing worker threads...');\n\n try {\n // Start screen capture worker\n await this.startScreenCaptureWorker();\n\n // Start Florence-2 worker if enabled\n if (this.config.florence2Enabled) {\n await this.startFlorence2Worker();\n }\n\n // Start OCR worker if enabled\n if (this.config.ocrEnabled) {\n await this.startOCRWorker();\n }\n\n logger.info('[VisionWorkerManager] All workers initialized');\n } catch (error) {\n logger.error('[VisionWorkerManager] Failed to initialize workers:', error);\n throw error;\n }\n }\n\n private async startScreenCaptureWorker(): Promise<void> {\n const workerPath = path.join(__dirname, 'workers', 'screen-capture-worker.js');\n\n this.screenCaptureWorker = new Worker(workerPath, {\n workerData: {\n config: {\n displayIndex: this.config.displayIndex,\n captureAllDisplays: this.config.captureAllDisplays,\n targetFPS: this.config.targetScreenFPS,\n sharedBufferSize: this.SCREEN_BUFFER_SIZE,\n },\n sharedBuffer: this.screenBuffer,\n },\n });\n\n this.screenCaptureWorker.on('message', (msg) => {\n if (msg.type === 'fps') {\n this.workerStats.set('screenCapture', {\n fps: msg.fps,\n frameCount: msg.frameCount,\n lastUpdate: Date.now(),\n });\n } else if (msg.type === 'error') {\n logger.error('[ScreenCaptureWorker] Error:', msg.error);\n } else if (msg.type === 'log') {\n // Handle worker log messages\n this.handleWorkerLog('ScreenCaptureWorker', msg);\n }\n });\n\n this.screenCaptureWorker.on('error', (error) => {\n logger.error('[ScreenCaptureWorker] Worker error:', error);\n // Attempt to restart worker after error\n setTimeout(() => this.restartScreenCaptureWorker(), 1000);\n });\n\n this.screenCaptureWorker.on('exit', (code) => {\n if (code !== 0) {\n logger.error(`[ScreenCaptureWorker] Worker stopped with exit code ${code}`);\n // Attempt to restart worker after crash\n setTimeout(() => this.restartScreenCaptureWorker(), 1000);\n }\n });\n }\n\n private async startFlorence2Worker(): Promise<void> {\n const workerPath = path.join(__dirname, 'workers', 'florence2-worker.js');\n\n // Calculate priority tiles (center tiles)\n const priorityTiles: number[] = [];\n const tileSize = this.config.tileSize || 256;\n const estimatedCols = Math.ceil(1920 / tileSize); // Estimate based on common resolution\n const centerRow = Math.floor(estimatedCols / 2);\n const centerCol = Math.floor(estimatedCols / 2);\n\n // Add center and adjacent tiles as priority\n for (let r = centerRow - 1; r <= centerRow + 1; r++) {\n for (let c = centerCol - 1; c <= centerCol + 1; c++) {\n if (r >= 0 && c >= 0) {\n priorityTiles.push(r * estimatedCols + c);\n }\n }\n }\n\n this.florence2Worker = new Worker(workerPath, {\n workerData: {\n config: {\n tileSize: this.config.tileSize || 256,\n priorityTiles,\n },\n sharedBuffer: this.screenBuffer,\n resultsBuffer: this.florence2ResultsBuffer,\n },\n });\n\n this.florence2Worker.on('message', (msg) => {\n if (msg.type === 'fps') {\n this.workerStats.set('florence2', {\n fps: msg.fps,\n frameCount: msg.frameCount,\n lastUpdate: Date.now(),\n });\n } else if (msg.type === 'tile_analyzed') {\n // Update latest results cache\n this.updateFlorence2Cache(msg);\n } else if (msg.type === 'error') {\n logger.error('[Florence2Worker] Error:', msg.error);\n } else if (msg.type === 'log') {\n // Handle worker log messages\n this.handleWorkerLog('Florence2Worker', msg);\n }\n });\n\n this.florence2Worker.on('error', (error) => {\n logger.error('[Florence2Worker] Worker error:', error);\n // Attempt to restart worker after error\n setTimeout(() => this.restartFlorence2Worker(), 1000);\n });\n\n this.florence2Worker.on('exit', (code) => {\n if (code !== 0) {\n logger.error(`[Florence2Worker] Worker stopped with exit code ${code}`);\n // Attempt to restart worker after crash\n setTimeout(() => this.restartFlorence2Worker(), 1000);\n }\n });\n }\n\n private async startOCRWorker(): Promise<void> {\n const workerPath = path.join(__dirname, 'workers', 'ocr-worker.js');\n\n this.ocrWorker = new Worker(workerPath, {\n workerData: {\n config: {\n processFullScreen: true,\n tileSize: this.config.tileSize || 256,\n textRegions: this.config.textRegions,\n },\n sharedBuffer: this.screenBuffer,\n resultsBuffer: this.ocrResultsBuffer,\n },\n });\n\n this.ocrWorker.on('message', (msg) => {\n if (msg.type === 'fps') {\n this.workerStats.set('ocr', {\n fps: msg.fps,\n frameCount: msg.frameCount,\n lastUpdate: Date.now(),\n });\n } else if (msg.type === 'ocr_complete') {\n // Update latest OCR cache\n this.updateOCRCache(msg);\n } else if (msg.type === 'error') {\n logger.error('[OCRWorker] Error:', msg.error);\n } else if (msg.type === 'log') {\n // Handle worker log messages\n this.handleWorkerLog('OCRWorker', msg);\n }\n });\n\n this.ocrWorker.on('error', (error) => {\n logger.error('[OCRWorker] Worker error:', error);\n // Attempt to restart worker after error\n setTimeout(() => this.restartOCRWorker(), 1000);\n });\n\n this.ocrWorker.on('exit', (code) => {\n if (code !== 0) {\n logger.error(`[OCRWorker] Worker stopped with exit code ${code}`);\n // Attempt to restart worker after crash\n setTimeout(() => this.restartOCRWorker(), 1000);\n }\n });\n }\n\n private updateFlorence2Cache(msg: any): void {\n // Read result from shared buffer\n try {\n const tileId = msg.tileId;\n const result = this.readFlorence2Result(tileId);\n if (result) {\n this.latestFlorence2Results.set(tileId, result);\n }\n } catch (error) {\n logger.error('[VisionWorkerManager] Failed to update Florence2 cache:', error);\n }\n }\n\n private updateOCRCache(_msg: any): void {\n // Read OCR result from shared buffer\n try {\n const result = this.readOCRResult();\n if (result) {\n this.latestOCRResult = result;\n }\n } catch (error) {\n logger.error('[VisionWorkerManager] Failed to update OCR cache:', error);\n }\n }\n\n private readFlorence2Result(tileId: string): Florence2Result | null {\n try {\n // Calculate tile index from ID\n const match = tileId.match(/tile-(\\d+)-(\\d+)/);\n if (!match) {\n return null;\n }\n\n const row = parseInt(match[1], 10);\n const col = parseInt(match[2], 10);\n const tileIndex = row * 10 + col; // Assuming max 10 columns\n\n const RESULTS_HEADER_SIZE = 16;\n const MAX_RESULT_SIZE = 4096;\n const offset = RESULTS_HEADER_SIZE + tileIndex * MAX_RESULT_SIZE;\n\n // Read length\n const length = this.florence2ResultsView.getUint32(offset, true);\n if (length === 0 || length > MAX_RESULT_SIZE - 4) {\n return null;\n }\n\n // Read JSON data\n const bytes = new Uint8Array(length);\n for (let i = 0; i < length; i++) {\n bytes[i] = this.florence2ResultsView.getUint8(offset + 4 + i);\n }\n\n const json = new TextDecoder().decode(bytes);\n return JSON.parse(json);\n } catch (error) {\n logger.error('[VisionWorkerManager] Failed to read Florence2 result:', error);\n return null;\n }\n }\n\n private readOCRResult(): OCRResult | null {\n try {\n const RESULTS_HEADER_SIZE = 16;\n const offset = RESULTS_HEADER_SIZE;\n\n // Read length\n const length = this.ocrResultsView.getUint32(offset, true);\n if (length === 0) {\n return null;\n }\n\n // Read frame ID and timestamp\n const _frameId = this.ocrResultsView.getUint32(offset + 4, true);\n const _timestamp = this.ocrResultsView.getFloat64(offset + 8, true);\n\n // Read JSON data\n const dataOffset = offset + 16;\n const bytes = new Uint8Array(Math.min(length, 65536));\n for (let i = 0; i < bytes.length; i++) {\n bytes[i] = this.ocrResultsView.getUint8(dataOffset + i);\n }\n\n const json = new TextDecoder().decode(bytes);\n return JSON.parse(json);\n } catch (error) {\n logger.error('[VisionWorkerManager] Failed to read OCR result:', error);\n return null;\n }\n }\n\n // Public API - Non-blocking access to latest data\n\n getLatestScreenCapture(): ScreenCapture | null {\n // Read current frame metadata\n const frameId = Atomics.load(this.screenAtomicState, this.FRAME_ID_INDEX);\n\n if (frameId <= this.lastProcessedFrameId) {\n return this.latestScreenCapture;\n }\n\n try {\n const width = Atomics.load(this.screenAtomicState, this.WIDTH_INDEX);\n const height = Atomics.load(this.screenAtomicState, this.HEIGHT_INDEX);\n const _displayIndex = Atomics.load(this.screenAtomicState, this.DISPLAY_INDEX);\n const timestamp = Atomics.load(this.screenAtomicState, this.TIMESTAMP_INDEX);\n\n // Create screen capture object\n this.latestScreenCapture = {\n timestamp,\n width,\n height,\n data: Buffer.alloc(0), // We don't copy the full data for performance\n tiles: this.generateTiles(width, height),\n };\n\n this.lastProcessedFrameId = frameId;\n } catch (error) {\n logger.error('[VisionWorkerManager] Failed to read screen capture:', error);\n }\n\n return this.latestScreenCapture;\n }\n\n getLatestEnhancedScene(): EnhancedSceneDescription {\n const screenCapture = this.getLatestScreenCapture();\n\n // Combine all Florence-2 results\n const florence2Captions: string[] = [];\n const allObjects: any[] = [];\n const allTags = new Set<string>();\n\n this.latestFlorence2Results.forEach((result) => {\n if (result.caption) {\n florence2Captions.push(result.caption);\n }\n if (result.objects) {\n allObjects.push(...result.objects);\n }\n if (result.tags) {\n result.tags.forEach((tag) => allTags.add(tag));\n }\n });\n\n const _tiles = this.generateTiles(screenCapture?.width || 0, screenCapture?.height || 0);\n\n return {\n timestamp: Date.now(),\n description: florence2Captions.join('. '),\n objects: allObjects,\n people: [], // Could be populated by TensorFlow worker\n sceneChanged: true,\n changePercentage: 100,\n screenCapture: this.latestScreenCapture || undefined,\n screenAnalysis: {\n fullScreenOCR: this.latestOCRResult?.fullText,\n activeTile: {\n timestamp: Date.now(),\n florence2: this.latestFlorence2Results.values().next().value,\n ocr: this.latestOCRResult || undefined,\n },\n gridSummary: `${screenCapture?.tiles.length || 0} tiles analyzed`,\n uiElements: allObjects.map((obj) => ({\n type: obj.label,\n text: '',\n position: obj.bbox,\n })),\n },\n };\n }\n\n private generateTiles(\n width: number,\n height: number\n ): Array<{\n id: string;\n row: number;\n col: number;\n x: number;\n y: number;\n width: number;\n height: number;\n }> {\n const tileSize = this.config.tileSize || 256;\n const tiles: Array<{\n id: string;\n row: number;\n col: number;\n x: number;\n y: number;\n width: number;\n height: number;\n }> = [];\n\n for (let row = 0; row < Math.ceil(height / tileSize); row++) {\n for (let col = 0; col < Math.ceil(width / tileSize); col++) {\n const x = col * tileSize;\n const y = row * tileSize;\n tiles.push({\n id: `tile-${row}-${col}`,\n row,\n col,\n x,\n y,\n width: Math.min(tileSize, width - x),\n height: Math.min(tileSize, height - y),\n });\n }\n }\n\n return tiles;\n }\n\n getWorkerStats(): Map<string, WorkerStats> {\n return new Map(this.workerStats);\n }\n\n async setDisplayIndex(index: number): Promise<void> {\n if (this.screenCaptureWorker) {\n this.screenCaptureWorker.postMessage({\n type: 'set_display',\n displayIndex: index,\n });\n }\n }\n\n async setTextRegions(\n regions: Array<{ x: number; y: number; width: number; height: number }>\n ): Promise<void> {\n if (this.ocrWorker) {\n this.ocrWorker.postMessage({\n type: 'update_regions',\n regions,\n });\n }\n }\n\n async stop(): Promise<void> {\n logger.info('[VisionWorkerManager] Stopping all workers...');\n\n const stopPromises: Promise<void>[] = [];\n\n if (this.screenCaptureWorker) {\n stopPromises.push(\n new Promise((resolve) => {\n this.screenCaptureWorker!.once('exit', () => resolve());\n this.screenCaptureWorker!.postMessage({ type: 'stop' });\n })\n );\n }\n\n if (this.florence2Worker) {\n stopPromises.push(\n new Promise((resolve) => {\n this.florence2Worker!.once('exit', () => resolve());\n this.florence2Worker!.postMessage({ type: 'stop' });\n })\n );\n }\n\n if (this.ocrWorker) {\n stopPromises.push(\n new Promise((resolve) => {\n this.ocrWorker!.once('exit', () => resolve());\n this.ocrWorker!.postMessage({ type: 'stop' });\n })\n );\n }\n\n await Promise.all(stopPromises);\n logger.info('[VisionWorkerManager] All workers stopped');\n }\n\n private handleWorkerLog(workerName: string, msg: any): void {\n const { level, message, args } = msg;\n const formattedMessage = `[${workerName}] ${message}`;\n\n switch (level) {\n case 'info':\n logger.info(formattedMessage, ...args);\n break;\n case 'warn':\n logger.warn(formattedMessage, ...args);\n break;\n case 'error':\n logger.error(formattedMessage, ...args);\n break;\n case 'debug':\n logger.debug(formattedMessage, ...args);\n break;\n }\n }\n\n private async restartScreenCaptureWorker(): Promise<void> {\n const attempts = this.restartAttempts.get('screenCapture') || 0;\n\n if (attempts >= this.MAX_RESTART_ATTEMPTS) {\n logger.error('[VisionWorkerManager] Max restart attempts reached for screen capture worker');\n return;\n }\n\n this.restartAttempts.set('screenCapture', attempts + 1);\n logger.info(`[VisionWorkerManager] Restarting screen capture worker (attempt ${attempts + 1})`);\n\n try {\n // Clean up existing worker\n if (this.screenCaptureWorker) {\n this.screenCaptureWorker.removeAllListeners();\n this.screenCaptureWorker = null;\n }\n\n // Start new worker\n await this.startScreenCaptureWorker();\n\n // Reset restart counter on successful start\n this.restartAttempts.set('screenCapture', 0);\n } catch (error) {\n logger.error('[VisionWorkerManager] Failed to restart screen capture worker:', error);\n }\n }\n\n private async restartFlorence2Worker(): Promise<void> {\n const attempts = this.restartAttempts.get('florence2') || 0;\n\n if (attempts >= this.MAX_RESTART_ATTEMPTS) {\n logger.error('[VisionWorkerManager] Max restart attempts reached for Florence2 worker');\n return;\n }\n\n this.restartAttempts.set('florence2', attempts + 1);\n logger.info(`[VisionWorkerManager] Restarting Florence2 worker (attempt ${attempts + 1})`);\n\n try {\n // Clean up existing worker\n if (this.florence2Worker) {\n this.florence2Worker.removeAllListeners();\n this.florence2Worker = null;\n }\n\n // Start new worker\n await this.startFlorence2Worker();\n\n // Reset restart counter on successful start\n this.restartAttempts.set('florence2', 0);\n } catch (error) {\n logger.error('[VisionWorkerManager] Failed to restart Florence2 worker:', error);\n }\n }\n\n private async restartOCRWorker(): Promise<void> {\n const attempts = this.restartAttempts.get('ocr') || 0;\n\n if (attempts >= this.MAX_RESTART_ATTEMPTS) {\n logger.error('[VisionWorkerManager] Max restart attempts reached for OCR worker');\n return;\n }\n\n this.restartAttempts.set('ocr', attempts + 1);\n logger.info(`[VisionWorkerManager] Restarting OCR worker (attempt ${attempts + 1})`);\n\n try {\n // Clean up existing worker\n if (this.ocrWorker) {\n this.ocrWorker.removeAllListeners();\n this.ocrWorker = null;\n }\n\n // Start new worker\n await this.startOCRWorker();\n\n // Reset restart counter on successful start\n this.restartAttempts.set('ocr', 0);\n } catch (error) {\n logger.error('[VisionWorkerManager] Failed to restart OCR worker:', error);\n }\n }\n}\n"]}
package/dist/workers/florence2-worker-simple.d.ts ADDED
@@ -0,0 +1,13 @@
1
+ import type { Florence2Result } from '../types';
2
+ export declare class Florence2WorkerModel {
3
+ private initialized;
4
+ initialize(): Promise<void>;
5
+ analyzeTile(tile: {
6
+ data: Buffer;
7
+ width: number;
8
+ height: number;
9
+ }): Promise<Florence2Result>;
10
+ analyzeImage(imageBuffer: Buffer): Promise<Florence2Result>;
11
+ isInitialized(): boolean;
12
+ dispose(): Promise<void>;
13
+ }
package/dist/workers/florence2-worker-simple.js ADDED
@@ -0,0 +1,121 @@
1
+ "use strict";
2
+ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
3
+ function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
4
+ return new (P || (P = Promise))(function (resolve, reject) {
5
+ function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
6
+ function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
7
+ function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
8
+ step((generator = generator.apply(thisArg, _arguments || [])).next());
9
+ });
10
+ };
11
+ var __generator = (this && this.__generator) || function (thisArg, body) {
12
+ var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g = Object.create((typeof Iterator === "function" ? Iterator : Object).prototype);
13
+ return g.next = verb(0), g["throw"] = verb(1), g["return"] = verb(2), typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g;
14
+ function verb(n) { return function (v) { return step([n, v]); }; }
15
+ function step(op) {
16
+ if (f) throw new TypeError("Generator is already executing.");
17
+ while (g && (g = 0, op[0] && (_ = 0)), _) try {
18
+ if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t;
19
+ if (y = 0, t) op = [op[0] & 2, t.value];
20
+ switch (op[0]) {
21
+ case 0: case 1: t = op; break;
22
+ case 4: _.label++; return { value: op[1], done: false };
23
+ case 5: _.label++; y = op[1]; op = [0]; continue;
24
+ case 7: op = _.ops.pop(); _.trys.pop(); continue;
25
+ default:
26
+ if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; }
27
+ if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; }
28
+ if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; }
29
+ if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; }
30
+ if (t[2]) _.ops.pop();
31
+ _.trys.pop(); continue;
32
+ }
33
+ op = body.call(thisArg, _);
34
+ } catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; }
35
+ if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true };
36
+ }
37
+ };
38
+ Object.defineProperty(exports, "__esModule", { value: true });
39
+ exports.Florence2WorkerModel = void 0;
40
+ var sharp_1 = require("sharp");
41
+ var Florence2WorkerModel = /** @class */ (function () {
42
+ function Florence2WorkerModel() {
43
+ this.initialized = false;
44
+ }
45
+ Florence2WorkerModel.prototype.initialize = function () {
46
+ return __awaiter(this, void 0, void 0, function () {
47
+ return __generator(this, function (_a) {
48
+ this.initialized = true;
49
+ return [2 /*return*/];
50
+ });
51
+ });
52
+ };
53
+ Florence2WorkerModel.prototype.analyzeTile = function (tile) {
54
+ return __awaiter(this, void 0, void 0, function () {
55
+ var metadata, stats, brightness, isLight, caption, objects, colorVariance;
56
+ return __generator(this, function (_a) {
57
+ switch (_a.label) {
58
+ case 0: return [4 /*yield*/, (0, sharp_1.default)(tile.data).metadata()];
59
+ case 1:
60
+ metadata = _a.sent();
61
+ return [4 /*yield*/, (0, sharp_1.default)(tile.data).stats()];
62
+ case 2:
63
+ stats = _a.sent();
64
+ brightness = (stats.channels[0].mean + stats.channels[1].mean + stats.channels[2].mean) / 3;
65
+ isLight = brightness > 180;
66
+ caption = 'Screen region';
67
+ if (metadata.width && metadata.height) {
68
+ if (metadata.width > metadata.height * 1.5) {
69
+ caption = 'Wide screen area';
70
+ }
71
+ else if (metadata.height > metadata.width * 1.5) {
72
+ caption = 'Tall screen area';
73
+ }
74
+ }
75
+ if (isLight) {
76
+ caption += ' with bright content';
77
+ }
78
+ else {
79
+ caption += ' with dark content';
80
+ }
81
+ objects = [];
82
+ colorVariance = stats.channels.reduce(function (sum, ch) { return sum + ch.stdev; }, 0) / 3;
83
+ if (colorVariance < 50) {
84
+ objects.push({
85
+ label: 'ui_element',
86
+ bbox: { x: 0, y: 0, width: metadata.width || 100, height: metadata.height || 100 },
87
+ confidence: 0.7,
88
+ });
89
+ }
90
+ return [2 /*return*/, {
91
+ caption: caption,
92
+ objects: objects,
93
+ regions: [],
94
+ tags: ['screen', 'ui'],
95
+ }];
96
+ }
97
+ });
98
+ });
99
+ };
100
+ Florence2WorkerModel.prototype.analyzeImage = function (imageBuffer) {
101
+ return __awaiter(this, void 0, void 0, function () {
102
+ return __generator(this, function (_a) {
103
+ return [2 /*return*/, this.analyzeTile({ data: imageBuffer, width: 0, height: 0 })];
104
+ });
105
+ });
106
+ };
107
+ Florence2WorkerModel.prototype.isInitialized = function () {
108
+ return this.initialized;
109
+ };
110
+ Florence2WorkerModel.prototype.dispose = function () {
111
+ return __awaiter(this, void 0, void 0, function () {
112
+ return __generator(this, function (_a) {
113
+ this.initialized = false;
114
+ return [2 /*return*/];
115
+ });
116
+ });
117
+ };
118
+ return Florence2WorkerModel;
119
+ }());
120
+ exports.Florence2WorkerModel = Florence2WorkerModel;
121
+ //# sourceMappingURL=florence2-worker-simple.js.map
package/dist/workers/florence2-worker-simple.js.map ADDED
@@ -0,0 +1 @@
1
+ {"version":3,"file":"florence2-worker-simple.js","sourceRoot":"","sources":["../../src/workers/florence2-worker-simple.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AACA,+BAA0B;AAE1B;IAAA;QACU,gBAAW,GAAG,KAAK,CAAC;IAoE9B,CAAC;IAlEO,yCAAU,GAAhB;;;gBACE,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;;;;KACzB;IAEK,0CAAW,GAAjB,UAAkB,IAIjB;;;;;4BAEkB,qBAAM,IAAA,eAAK,EAAC,IAAI,CAAC,IAAI,CAAC,CAAC,QAAQ,EAAE,EAAA;;wBAA5C,QAAQ,GAAG,SAAiC;wBACpC,qBAAM,IAAA,eAAK,EAAC,IAAI,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAA;;wBAAtC,KAAK,GAAG,SAA8B;wBAGtC,UAAU,GACd,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI,GAAG,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI,GAAG,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;wBAC3E,OAAO,GAAG,UAAU,GAAG,GAAG,CAAC;wBAG7B,OAAO,GAAG,eAAe,CAAC;wBAC9B,IAAI,QAAQ,CAAC,KAAK,IAAI,QAAQ,CAAC,MAAM,EAAE,CAAC;4BACtC,IAAI,QAAQ,CAAC,KAAK,GAAG,QAAQ,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC;gCAC3C,OAAO,GAAG,kBAAkB,CAAC;4BAC/B,CAAC;iCAAM,IAAI,QAAQ,CAAC,MAAM,GAAG,QAAQ,CAAC,KAAK,GAAG,GAAG,EAAE,CAAC;gCAClD,OAAO,GAAG,kBAAkB,CAAC;4BAC/B,CAAC;wBACH,CAAC;wBAED,IAAI,OAAO,EAAE,CAAC;4BACZ,OAAO,IAAI,sBAAsB,CAAC;wBACpC,CAAC;6BAAM,CAAC;4BACN,OAAO,IAAI,oBAAoB,CAAC;wBAClC,CAAC;wBAGK,OAAO,GAAoE,EAAE,CAAC;wBAG9E,aAAa,GAAG,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,UAAC,GAAG,EAAE,EAAE,IAAK,OAAA,GAAG,GAAG,EAAE,CAAC,KAAK,EAAd,CAAc,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC;wBAChF,IAAI,aAAa,GAAG,EAAE,EAAE,CAAC;4BACvB,OAAO,CAAC,IAAI,CAAC;gCACX,KAAK,EAAE,YAAY;gCACnB,IAAI,EAAE,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,KAAK,EAAE,QAAQ,CAAC,KAAK,IAAI,GAAG,EAAE,MAAM,EAAE,QAAQ,CAAC,MAAM,IAAI,GAAG,EAAE;gCAClF,UAAU,EAAE,GAAG;6BAChB,CAAC,CAAC;wBACL,CAAC;wBAED,sBAAO;gCACL,OAAO,SAAA;gCACP,OAAO,SAAA;gCACP,OAAO,EAAE,EAAE;gCACX,IAAI,EAAE,CAAC,QAAQ,EAAE,IAAI,CAAC;6BACvB,EAAC;;;;KACH;IAEK,2CAAY,GAAlB,UAAmB,WAAmB;;;gBACpC,sBAAO,IAAI,CAAC,WAAW,CAAC,EAAE,IAAI,EAAE,WAAW,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,CAAC,EAAC;;;KACrE;IAED,4CAAa,GAAb;QACE,OAAO,IAAI,CAAC,WAAW,CAAC;IAC1B,CAAC;IAEK,sCAAO,GAAb;;;gBACE,IAAI,CAAC,WAAW,GAAG,KAAK,CAAC;;;;KAC1B;IACH,2BAAC;AAAD,CAAC,AArED,IAqEC;AArEY,oDAAoB","sourcesContent":["import type { Florence2Result, BoundingBox } from '../types';\nimport sharp from 'sharp';\n\nexport class Florence2WorkerModel {\n private initialized = false;\n\n async initialize(): Promise<void> {\n this.initialized = true;\n }\n\n async analyzeTile(tile: {\n data: Buffer;\n width: number;\n height: number;\n }): Promise<Florence2Result> {\n // Simple analysis without TensorFlow\n const metadata = await sharp(tile.data).metadata();\n const stats = await sharp(tile.data).stats();\n\n // Determine scene characteristics\n const brightness =\n (stats.channels[0].mean + stats.channels[1].mean + stats.channels[2].mean) / 3;\n const isLight = brightness > 180;\n\n // Generate caption based on tile characteristics\n let caption = 'Screen region';\n if (metadata.width && metadata.height) {\n if (metadata.width > metadata.height * 1.5) {\n caption = 'Wide screen area';\n } else if (metadata.height > metadata.width * 1.5) {\n caption = 'Tall screen area';\n }\n }\n\n if (isLight) {\n caption += ' with bright content';\n } else {\n caption += ' with dark content';\n }\n\n // Detect potential UI elements based on color patterns\n const objects: Array<{ label: string; bbox: BoundingBox; confidence: number }> = [];\n\n // Simple heuristic: if mostly uniform color, might be a UI element\n const colorVariance = stats.channels.reduce((sum, ch) => sum + ch.stdev, 0) / 3;\n if (colorVariance < 50) {\n objects.push({\n label: 'ui_element',\n bbox: { x: 0, y: 0, width: metadata.width || 100, height: metadata.height || 100 },\n confidence: 0.7,\n });\n }\n\n return {\n caption,\n objects,\n regions: [],\n tags: ['screen', 'ui'],\n };\n }\n\n async analyzeImage(imageBuffer: Buffer): Promise<Florence2Result> {\n return this.analyzeTile({ data: imageBuffer, width: 0, height: 0 });\n }\n\n isInitialized(): boolean {\n return this.initialized;\n }\n\n async dispose(): Promise<void> {\n this.initialized = false;\n }\n}\n"]}
package/dist/workers/florence2-worker.d.ts ADDED
@@ -0,0 +1 @@
1
+ export {};
package/dist/workers/florence2-worker.js ADDED
@@ -0,0 +1,328 @@
1
+ "use strict";
2
+ var __assign = (this && this.__assign) || function () {
3
+ __assign = Object.assign || function(t) {
4
+ for (var s, i = 1, n = arguments.length; i < n; i++) {
5
+ s = arguments[i];
6
+ for (var p in s) if (Object.prototype.hasOwnProperty.call(s, p))
7
+ t[p] = s[p];
8
+ }
9
+ return t;
10
+ };
11
+ return __assign.apply(this, arguments);
12
+ };
13
+ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
14
+ function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
15
+ return new (P || (P = Promise))(function (resolve, reject) {
16
+ function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
17
+ function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
18
+ function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
19
+ step((generator = generator.apply(thisArg, _arguments || [])).next());
20
+ });
21
+ };
22
+ var __generator = (this && this.__generator) || function (thisArg, body) {
23
+ var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g = Object.create((typeof Iterator === "function" ? Iterator : Object).prototype);
24
+ return g.next = verb(0), g["throw"] = verb(1), g["return"] = verb(2), typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g;
25
+ function verb(n) { return function (v) { return step([n, v]); }; }
26
+ function step(op) {
27
+ if (f) throw new TypeError("Generator is already executing.");
28
+ while (g && (g = 0, op[0] && (_ = 0)), _) try {
29
+ if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t;
30
+ if (y = 0, t) op = [op[0] & 2, t.value];
31
+ switch (op[0]) {
32
+ case 0: case 1: t = op; break;
33
+ case 4: _.label++; return { value: op[1], done: false };
34
+ case 5: _.label++; y = op[1]; op = [0]; continue;
35
+ case 7: op = _.ops.pop(); _.trys.pop(); continue;
36
+ default:
37
+ if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; }
38
+ if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; }
39
+ if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; }
40
+ if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; }
41
+ if (t[2]) _.ops.pop();
42
+ _.trys.pop(); continue;
43
+ }
44
+ op = body.call(thisArg, _);
45
+ } catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; }
46
+ if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true };
47
+ }
48
+ };
49
+ Object.defineProperty(exports, "__esModule", { value: true });
50
+ var worker_threads_1 = require("worker_threads");
51
+ var worker_logger_1 = require("./worker-logger");
52
+ var florence2_model_1 = require("../florence2-model");
53
+ var sharp_1 = require("sharp");
54
+ var Florence2Worker = /** @class */ (function () {
55
+ function Florence2Worker(config, sharedBuffer, resultsBuffer) {
56
+ this.isRunning = true;
57
+ this.frameCount = 0;
58
+ this.lastFPSReport = Date.now();
59
+ this.lastFrameId = -1;
60
+ // Atomic indices for input buffer
61
+ this.FRAME_ID_INDEX = 0;
62
+ this.WRITE_LOCK_INDEX = 1;
63
+ this.WIDTH_INDEX = 2;
64
+ this.HEIGHT_INDEX = 3;
65
+ this.DISPLAY_INDEX = 4;
66
+ this.TIMESTAMP_INDEX = 5;
67
+ this.DATA_OFFSET = 24;
68
+ // Results buffer structure
69
+ this.RESULTS_HEADER_SIZE = 16;
70
+ this.MAX_RESULT_SIZE = 4096; // Per tile
71
+ this.config = config;
72
+ this.sharedBuffer = sharedBuffer;
73
+ this.dataView = new DataView(sharedBuffer);
74
+ this.atomicState = new Int32Array(sharedBuffer, 0, 6);
75
+ this.resultsBuffer = resultsBuffer;
76
+ this.resultsView = new DataView(resultsBuffer);
77
+ this.florence2 = new florence2_model_1.Florence2Model();
78
+ }
79
+ Florence2Worker.prototype.initialize = function () {
80
+ return __awaiter(this, void 0, void 0, function () {
81
+ return __generator(this, function (_a) {
82
+ switch (_a.label) {
83
+ case 0: return [4 /*yield*/, this.florence2.initialize()];
84
+ case 1:
85
+ _a.sent();
86
+ worker_logger_1.logger.info('[Florence2Worker] Initialized and ready');
87
+ return [2 /*return*/];
88
+ }
89
+ });
90
+ });
91
+ };
92
+ Florence2Worker.prototype.run = function () {
93
+ return __awaiter(this, void 0, void 0, function () {
94
+ var currentFrameId, now, fps, error_1;
95
+ return __generator(this, function (_a) {
96
+ switch (_a.label) {
97
+ case 0: return [4 /*yield*/, this.initialize()];
98
+ case 1:
99
+ _a.sent();
100
+ worker_logger_1.logger.info('[Florence2Worker] Starting analysis loop...');
101
+ _a.label = 2;
102
+ case 2:
103
+ if (!this.isRunning) return [3 /*break*/, 11];
104
+ _a.label = 3;
105
+ case 3:
106
+ _a.trys.push([3, 8, , 10]);
107
+ currentFrameId = Atomics.load(this.atomicState, this.FRAME_ID_INDEX);
108
+ if (!(currentFrameId > this.lastFrameId)) return [3 /*break*/, 5];
109
+ return [4 /*yield*/, this.processFrame()];
110
+ case 4:
111
+ _a.sent();
112
+ this.lastFrameId = currentFrameId;
113
+ this.frameCount++;
114
+ now = Date.now();
115
+ if (now - this.lastFPSReport >= 1000) {
116
+ fps = this.frameCount / ((now - this.lastFPSReport) / 1000);
117
+ worker_logger_1.logger.info("[Florence2Worker] Analysis FPS: ".concat(fps.toFixed(2)));
118
+ worker_threads_1.parentPort === null || worker_threads_1.parentPort === void 0 ? void 0 : worker_threads_1.parentPort.postMessage({
119
+ type: 'fps',
120
+ fps: fps,
121
+ frameCount: this.frameCount,
122
+ });
123
+ this.frameCount = 0;
124
+ this.lastFPSReport = now;
125
+ }
126
+ return [3 /*break*/, 7];
127
+ case 5:
128
+ // No new frame, brief yield
129
+ return [4 /*yield*/, new Promise(function (resolve) { return setImmediate(resolve); })];
130
+ case 6:
131
+ // No new frame, brief yield
132
+ _a.sent();
133
+ _a.label = 7;
134
+ case 7: return [3 /*break*/, 10];
135
+ case 8:
136
+ error_1 = _a.sent();
137
+ worker_logger_1.logger.error('[Florence2Worker] Processing error:', error_1);
138
+ return [4 /*yield*/, new Promise(function (resolve) { return setTimeout(resolve, 100); })];
139
+ case 9:
140
+ _a.sent();
141
+ return [3 /*break*/, 10];
142
+ case 10: return [3 /*break*/, 2];
143
+ case 11: return [2 /*return*/];
144
+ }
145
+ });
146
+ });
147
+ };
148
+ Florence2Worker.prototype.processFrame = function () {
149
+ return __awaiter(this, void 0, void 0, function () {
150
+ var metadata, tiles, tilesToProcess, i, tile, tileBuffer, result, error_2;
151
+ var _a;
152
+ return __generator(this, function (_b) {
153
+ switch (_b.label) {
154
+ case 0:
155
+ metadata = {
156
+ frameId: Atomics.load(this.atomicState, this.FRAME_ID_INDEX),
157
+ width: Atomics.load(this.atomicState, this.WIDTH_INDEX),
158
+ height: Atomics.load(this.atomicState, this.HEIGHT_INDEX),
159
+ displayIndex: Atomics.load(this.atomicState, this.DISPLAY_INDEX),
160
+ timestamp: Atomics.load(this.atomicState, this.TIMESTAMP_INDEX),
161
+ };
162
+ tiles = this.calculateTiles(metadata.width, metadata.height);
163
+ tilesToProcess = this.config.priorityTiles
164
+ ? this.config.priorityTiles.map(function (i) { return tiles[i]; }).filter(Boolean)
165
+ : tiles;
166
+ i = 0;
167
+ _b.label = 1;
168
+ case 1:
169
+ if (!(i < tilesToProcess.length)) return [3 /*break*/, 8];
170
+ tile = tilesToProcess[i];
171
+ if (!tile) {
172
+ return [3 /*break*/, 7];
173
+ }
174
+ _b.label = 2;
175
+ case 2:
176
+ _b.trys.push([2, 6, , 7]);
177
+ return [4 /*yield*/, this.extractTileFromSharedBuffer(tile, metadata)];
178
+ case 3:
179
+ tileBuffer = _b.sent();
180
+ return [4 /*yield*/, this.florence2.analyzeTile(__assign(__assign({}, tile), { data: tileBuffer }))];
181
+ case 4:
182
+ result = _b.sent();
183
+ // Write result to results buffer
184
+ return [4 /*yield*/, this.writeResultToBuffer(tile.id, result, metadata.frameId)];
185
+ case 5:
186
+ // Write result to results buffer
187
+ _b.sent();
188
+ // Notify main thread
189
+ worker_threads_1.parentPort === null || worker_threads_1.parentPort === void 0 ? void 0 : worker_threads_1.parentPort.postMessage({
190
+ type: 'tile_analyzed',
191
+ tileId: tile.id,
192
+ frameId: metadata.frameId,
193
+ displayIndex: metadata.displayIndex,
194
+ hasObjects: (((_a = result.objects) === null || _a === void 0 ? void 0 : _a.length) || 0) > 0,
195
+ caption: result.caption,
196
+ });
197
+ return [3 /*break*/, 7];
198
+ case 6:
199
+ error_2 = _b.sent();
200
+ worker_logger_1.logger.error("[Florence2Worker] Failed to analyze tile ".concat(tile.id, ":"), error_2);
201
+ return [3 /*break*/, 7];
202
+ case 7:
203
+ i++;
204
+ return [3 /*break*/, 1];
205
+ case 8: return [2 /*return*/];
206
+ }
207
+ });
208
+ });
209
+ };
210
+ Florence2Worker.prototype.calculateTiles = function (width, height) {
211
+ var tileSize = this.config.tileSize;
212
+ var tiles = [];
213
+ for (var row = 0; row < Math.ceil(height / tileSize); row++) {
214
+ for (var col = 0; col < Math.ceil(width / tileSize); col++) {
215
+ var x = col * tileSize;
216
+ var y = row * tileSize;
217
+ var tileWidth = Math.min(tileSize, width - x);
218
+ var tileHeight = Math.min(tileSize, height - y);
219
+ tiles.push({
220
+ id: "tile-".concat(row, "-").concat(col),
221
+ row: row,
222
+ col: col,
223
+ x: x,
224
+ y: y,
225
+ width: tileWidth,
226
+ height: tileHeight,
227
+ });
228
+ }
229
+ }
230
+ return tiles;
231
+ };
232
+ Florence2Worker.prototype.extractTileFromSharedBuffer = function (tile, metadata) {
233
+ return __awaiter(this, void 0, void 0, function () {
234
+ var bytesPerPixel, rowStride, tileData, row, sourceY, sourceOffset, destOffset, i, pngBuffer;
235
+ return __generator(this, function (_a) {
236
+ switch (_a.label) {
237
+ case 0:
238
+ bytesPerPixel = 4;
239
+ rowStride = metadata.width * bytesPerPixel;
240
+ tileData = Buffer.allocUnsafe(tile.width * tile.height * bytesPerPixel);
241
+ // Copy tile data row by row
242
+ for (row = 0; row < tile.height; row++) {
243
+ sourceY = tile.y + row;
244
+ sourceOffset = this.DATA_OFFSET + sourceY * rowStride + tile.x * bytesPerPixel;
245
+ destOffset = row * tile.width * bytesPerPixel;
246
+ // Copy one row of tile data
247
+ for (i = 0; i < tile.width * bytesPerPixel; i++) {
248
+ tileData[destOffset + i] = this.dataView.getUint8(sourceOffset + i);
249
+ }
250
+ }
251
+ return [4 /*yield*/, (0, sharp_1.default)(tileData, {
252
+ raw: {
253
+ width: tile.width,
254
+ height: tile.height,
255
+ channels: 4,
256
+ },
257
+ })
258
+ .png()
259
+ .toBuffer()];
260
+ case 1:
261
+ pngBuffer = _a.sent();
262
+ return [2 /*return*/, pngBuffer];
263
+ }
264
+ });
265
+ });
266
+ };
267
+ Florence2Worker.prototype.writeResultToBuffer = function (tileId, result, frameId) {
268
+ return __awaiter(this, void 0, void 0, function () {
269
+ var resultJson, resultBytes, match, row, col, tileIndex, offset, i;
270
+ return __generator(this, function (_a) {
271
+ resultJson = JSON.stringify(__assign({ tileId: tileId, frameId: frameId, timestamp: Date.now() }, result));
272
+ resultBytes = Buffer.from(resultJson, 'utf-8');
273
+ match = tileId.match(/tile-(\d+)-(\d+)/);
274
+ if (!match) {
275
+ return [2 /*return*/];
276
+ }
277
+ row = parseInt(match[1], 10);
278
+ col = parseInt(match[2], 10);
279
+ tileIndex = row * 10 + col;
280
+ offset = this.RESULTS_HEADER_SIZE + tileIndex * this.MAX_RESULT_SIZE;
281
+ // Write length
282
+ this.resultsView.setUint32(offset, resultBytes.length, true);
283
+ // Write data
284
+ for (i = 0; i < Math.min(resultBytes.length, this.MAX_RESULT_SIZE - 4); i++) {
285
+ this.resultsView.setUint8(offset + 4 + i, resultBytes[i]);
286
+ }
287
+ return [2 /*return*/];
288
+ });
289
+ });
290
+ };
291
+ Florence2Worker.prototype.stop = function () {
292
+ this.isRunning = false;
293
+ };
294
+ Florence2Worker.prototype.dispose = function () {
295
+ return __awaiter(this, void 0, void 0, function () {
296
+ return __generator(this, function (_a) {
297
+ switch (_a.label) {
298
+ case 0: return [4 /*yield*/, this.florence2.dispose()];
299
+ case 1:
300
+ _a.sent();
301
+ return [2 /*return*/];
302
+ }
303
+ });
304
+ });
305
+ };
306
+ return Florence2Worker;
307
+ }());
308
+ // Worker entry point
309
+ if (worker_threads_1.parentPort) {
310
+ var config = worker_threads_1.workerData.config, sharedBuffer = worker_threads_1.workerData.sharedBuffer, resultsBuffer = worker_threads_1.workerData.resultsBuffer;
311
+ var worker_1 = new Florence2Worker(config, sharedBuffer, resultsBuffer);
312
+ // Handle messages from main thread
313
+ worker_threads_1.parentPort.on('message', function (msg) {
314
+ if (msg.type === 'stop') {
315
+ worker_1.stop();
316
+ worker_1.dispose().then(function () {
317
+ worker_threads_1.parentPort === null || worker_threads_1.parentPort === void 0 ? void 0 : worker_threads_1.parentPort.postMessage({ type: 'stopped' });
318
+ });
319
+ }
320
+ });
321
+ // Run the worker
322
+ worker_1.run().catch(function (error) {
323
+ worker_logger_1.logger.error('[Florence2Worker] Fatal error:', error);
324
+ worker_threads_1.parentPort === null || worker_threads_1.parentPort === void 0 ? void 0 : worker_threads_1.parentPort.postMessage({ type: 'error', error: error.message });
325
+ process.exit(1);
326
+ });
327
+ }
328
+ //# sourceMappingURL=florence2-worker.js.map
package/dist/workers/florence2-worker.js.map ADDED
@@ -0,0 +1 @@
1
+ {"version":3,"file":"florence2-worker.js","sourceRoot":"","sources":["../../src/workers/florence2-worker.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,iDAAwD;AACxD,iDAAyC;AACzC,sDAAoD;AAEpD,+BAA0B;AAe1B;IA0BE,yBACE,MAAoB,EACpB,YAA+B,EAC/B,aAAgC;QArB1B,cAAS,GAAG,IAAI,CAAC;QACjB,eAAU,GAAG,CAAC,CAAC;QACf,kBAAa,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC3B,gBAAW,GAAG,CAAC,CAAC,CAAC;QAEzB,kCAAkC;QACjB,mBAAc,GAAG,CAAC,CAAC;QACnB,qBAAgB,GAAG,CAAC,CAAC;QACrB,gBAAW,GAAG,CAAC,CAAC;QAChB,iBAAY,GAAG,CAAC,CAAC;QACjB,kBAAa,GAAG,CAAC,CAAC;QAClB,oBAAe,GAAG,CAAC,CAAC;QACpB,gBAAW,GAAG,EAAE,CAAC;QAElC,2BAA2B;QACV,wBAAmB,GAAG,EAAE,CAAC;QACzB,oBAAe,GAAG,IAAI,CAAC,CAAC,WAAW;QAOlD,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,YAAY,GAAG,YAAY,CAAC;QACjC,IAAI,CAAC,QAAQ,GAAG,IAAI,QAAQ,CAAC,YAAY,CAAC,CAAC;QAC3C,IAAI,CAAC,WAAW,GAAG,IAAI,UAAU,CAAC,YAAY,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;QACtD,IAAI,CAAC,aAAa,GAAG,aAAa,CAAC;QACnC,IAAI,CAAC,WAAW,GAAG,IAAI,QAAQ,CAAC,aAAa,CAAC,CAAC;QAC/C,IAAI,CAAC,SAAS,GAAG,IAAI,gCAAc,EAAE,CAAC;IACxC,CAAC;IAEK,oCAAU,GAAhB;;;;4BACE,qBAAM,IAAI,CAAC,SAAS,CAAC,UAAU,EAAE,EAAA;;wBAAjC,SAAiC,CAAC;wBAClC,sBAAM,CAAC,IAAI,CAAC,yCAAyC,CAAC,CAAC;;;;;KACxD;IAEK,6BAAG,GAAT;;;;;4BACE,qBAAM,IAAI,CAAC,UAAU,EAAE,EAAA;;wBAAvB,SAAuB,CAAC;wBAExB,sBAAM,CAAC,IAAI,CAAC,6CAA6C,CAAC,CAAC;;;6BAEpD,IAAI,CAAC,SAAS;;;;wBAGX,cAAc,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,IAAI,CAAC,cAAc,CAAC,CAAC;6BAEvE,CAAA,cAAc,GAAG,IAAI,CAAC,WAAW,CAAA,EAAjC,wBAAiC;wBACnC,qBAAM,IAAI,CAAC,YAAY,EAAE,EAAA;;wBAAzB,SAAyB,CAAC;wBAC1B,IAAI,CAAC,WAAW,GAAG,cAAc,CAAC;wBAClC,IAAI,CAAC,UAAU,EAAE,CAAC;wBAGZ,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;wBACvB,IAAI,GAAG,GAAG,IAAI,CAAC,aAAa,IAAI,IAAI,EAAE,CAAC;4BAC/B,GAAG,GAAG,IAAI,CAAC,UAAU,GAAG,CAAC,CAAC,GAAG,GAAG,IAAI,CAAC,aAAa,CAAC,GAAG,IAAI,CAAC,CAAC;4BAClE,sBAAM,CAAC,IAAI,CAAC,0CAAmC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,CAAE,CAAC,CAAC;4BAEjE,2BAAU,aAAV,2BAAU,uBAAV,2BAAU,CAAE,WAAW,CAAC;gCACtB,IAAI,EAAE,KAAK;gCACX,GAAG,KAAA;gCACH,UAAU,EAAE,IAAI,CAAC,UAAU;6BAC5B,CAAC,CAAC;4BAEH,IAAI,CAAC,UAAU,GAAG,CAAC,CAAC;4BACpB,IAAI,CAAC,aAAa,GAAG,GAAG,CAAC;wBAC3B,CAAC;;;oBAED,4BAA4B;oBAC5B,qBAAM,IAAI,OAAO,CAAC,UAAC,OAAO,IAAK,OAAA,YAAY,CAAC,OAAO,CAAC,EAArB,CAAqB,CAAC,EAAA;;wBADrD,4BAA4B;wBAC5B,SAAqD,CAAC;;;;;wBAGxD,sBAAM,CAAC,KAAK,CAAC,qCAAqC,EAAE,OAAK,CAAC,CAAC;wBAC3D,qBAAM,IAAI,OAAO,CAAC,UAAC,OAAO,IAAK,OAAA,UAAU,CAAC,OAAO,EAAE,GAAG,CAAC,EAAxB,CAAwB,CAAC,EAAA;;wBAAxD,SAAwD,CAAC;;;;;;;KAG9D;IAEa,sCAAY,GAA1B;;;;;;;wBAEQ,QAAQ,GAAmB;4BAC/B,OAAO,EAAE,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,IAAI,CAAC,cAAc,CAAC;4BAC5D,KAAK,EAAE,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,IAAI,CAAC,WAAW,CAAC;4BACvD,MAAM,EAAE,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,IAAI,CAAC,YAAY,CAAC;4BACzD,YAAY,EAAE,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,IAAI,CAAC,aAAa,CAAC;4BAChE,SAAS,EAAE,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,IAAI,CAAC,eAAe,CAAC;yBAChE,CAAC;wBAGI,KAAK,GAAG,IAAI,CAAC,cAAc,CAAC,QAAQ,CAAC,KAAK,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC;wBAG7D,cAAc,GAAG,IAAI,CAAC,MAAM,CAAC,aAAa;4BAC9C,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,GAAG,CAAC,UAAC,CAAC,IAAK,OAAA,KAAK,CAAC,CAAC,CAAC,EAAR,CAAQ,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC;4BAChE,CAAC,CAAC,KAAK,CAAC;wBAGD,CAAC,GAAG,CAAC;;;6BAAE,CAAA,CAAC,GAAG,cAAc,CAAC,MAAM,CAAA;wBACjC,IAAI,GAAG,cAAc,CAAC,CAAC,CAAC,CAAC;wBAC/B,IAAI,CAAC,IAAI,EAAE,CAAC;4BACV,wBAAS;wBACX,CAAC;;;;wBAIoB,qBAAM,IAAI,CAAC,2BAA2B,CAAC,IAAI,EAAE,QAAQ,CAAC,EAAA;;wBAAnE,UAAU,GAAG,SAAsD;wBAG1D,qBAAM,IAAI,CAAC,SAAS,CAAC,WAAW,uBAC1C,IAAI,KACP,IAAI,EAAE,UAAU,IAChB,EAAA;;wBAHI,MAAM,GAAG,SAGb;wBAEF,iCAAiC;wBACjC,qBAAM,IAAI,CAAC,mBAAmB,CAAC,IAAI,CAAC,EAAE,EAAE,MAAM,EAAE,QAAQ,CAAC,OAAO,CAAC,EAAA;;wBADjE,iCAAiC;wBACjC,SAAiE,CAAC;wBAElE,qBAAqB;wBACrB,2BAAU,aAAV,2BAAU,uBAAV,2BAAU,CAAE,WAAW,CAAC;4BACtB,IAAI,EAAE,eAAe;4BACrB,MAAM,EAAE,IAAI,CAAC,EAAE;4BACf,OAAO,EAAE,QAAQ,CAAC,OAAO;4BACzB,YAAY,EAAE,QAAQ,CAAC,YAAY;4BACnC,UAAU,EAAE,CAAC,CAAA,MAAA,MAAM,CAAC,OAAO,0CAAE,MAAM,KAAI,CAAC,CAAC,GAAG,CAAC;4BAC7C,OAAO,EAAE,MAAM,CAAC,OAAO;yBACxB,CAAC,CAAC;;;;wBAEH,sBAAM,CAAC,KAAK,CAAC,mDAA4C,IAAI,CAAC,EAAE,MAAG,EAAE,OAAK,CAAC,CAAC;;;wBA7BrC,CAAC,EAAE,CAAA;;;;;;KAgC/C;IAEO,wCAAc,GAAtB,UAAuB,KAAa,EAAE,MAAc;QAClD,IAAM,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC;QACtC,IAAM,KAAK,GAAiB,EAAE,CAAC;QAE/B,KAAK,IAAI,GAAG,GAAG,CAAC,EAAE,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,QAAQ,CAAC,EAAE,GAAG,EAAE,EAAE,CAAC;YAC5D,KAAK,IAAI,GAAG,GAAG,CAAC,EAAE,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,GAAG,QAAQ,CAAC,EAAE,GAAG,EAAE,EAAE,CAAC;gBAC3D,IAAM,CAAC,GAAG,GAAG,GAAG,QAAQ,CAAC;gBACzB,IAAM,CAAC,GAAG,GAAG,GAAG,QAAQ,CAAC;gBACzB,IAAM,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,QAAQ,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC;gBAChD,IAAM,UAAU,GAAG,IAAI,CAAC,GAAG,CAAC,QAAQ,EAAE,MAAM,GAAG,CAAC,CAAC,CAAC;gBAElD,KAAK,CAAC,IAAI,CAAC;oBACT,EAAE,EAAE,eAAQ,GAAG,cAAI,GAAG,CAAE;oBACxB,GAAG,KAAA;oBACH,GAAG,KAAA;oBACH,CAAC,GAAA;oBACD,CAAC,GAAA;oBACD,KAAK,EAAE,SAAS;oBAChB,MAAM,EAAE,UAAU;iBACnB,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QAED,OAAO,KAAK,CAAC;IACf,CAAC;IAEa,qDAA2B,GAAzC,UACE,IAAgB,EAChB,QAAwB;;;;;;wBAGlB,aAAa,GAAG,CAAC,CAAC;wBAClB,SAAS,GAAG,QAAQ,CAAC,KAAK,GAAG,aAAa,CAAC;wBAG3C,QAAQ,GAAG,MAAM,CAAC,WAAW,CAAC,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,MAAM,GAAG,aAAa,CAAC,CAAC;wBAE9E,4BAA4B;wBAC5B,KAAS,GAAG,GAAG,CAAC,EAAE,GAAG,GAAG,IAAI,CAAC,MAAM,EAAE,GAAG,EAAE,EAAE,CAAC;4BACrC,OAAO,GAAG,IAAI,CAAC,CAAC,GAAG,GAAG,CAAC;4BACvB,YAAY,GAAG,IAAI,CAAC,WAAW,GAAG,OAAO,GAAG,SAAS,GAAG,IAAI,CAAC,CAAC,GAAG,aAAa,CAAC;4BAC/E,UAAU,GAAG,GAAG,GAAG,IAAI,CAAC,KAAK,GAAG,aAAa,CAAC;4BAEpD,4BAA4B;4BAC5B,KAAS,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,KAAK,GAAG,aAAa,EAAE,CAAC,EAAE,EAAE,CAAC;gCACpD,QAAQ,CAAC,UAAU,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,YAAY,GAAG,CAAC,CAAC,CAAC;4BACtE,CAAC;wBACH,CAAC;wBAGiB,qBAAM,IAAA,eAAK,EAAC,QAAQ,EAAE;gCACtC,GAAG,EAAE;oCACH,KAAK,EAAE,IAAI,CAAC,KAAK;oCACjB,MAAM,EAAE,IAAI,CAAC,MAAM;oCACnB,QAAQ,EAAE,CAAC;iCACZ;6BACF,CAAC;iCACC,GAAG,EAAE;iCACL,QAAQ,EAAE,EAAA;;wBARP,SAAS,GAAG,SAQL;wBAEb,sBAAO,SAAS,EAAC;;;;KAClB;IAEa,6CAAmB,GAAjC,UACE,MAAc,EACd,MAAuB,EACvB,OAAe;;;;gBAGT,UAAU,GAAG,IAAI,CAAC,SAAS,YAC/B,MAAM,QAAA,EACN,OAAO,SAAA,EACP,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE,IAClB,MAAM,EACT,CAAC;gBAEG,WAAW,GAAG,MAAM,CAAC,IAAI,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;gBAG/C,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,kBAAkB,CAAC,CAAC;gBAC/C,IAAI,CAAC,KAAK,EAAE,CAAC;oBACX,sBAAO;gBACT,CAAC;gBAEK,GAAG,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;gBAC7B,GAAG,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;gBAC7B,SAAS,GAAG,GAAG,GAAG,EAAE,GAAG,GAAG,CAAC;gBAG3B,MAAM,GAAG,IAAI,CAAC,mBAAmB,GAAG,SAAS,GAAG,IAAI,CAAC,eAAe,CAAC;gBAE3E,eAAe;gBACf,IAAI,CAAC,WAAW,CAAC,SAAS,CAAC,MAAM,EAAE,WAAW,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;gBAE7D,aAAa;gBACb,KAAS,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,WAAW,CAAC,MAAM,EAAE,IAAI,CAAC,eAAe,GAAG,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;oBAChF,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,GAAG,CAAC,EAAE,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC;gBAC5D,CAAC;;;;KACF;IAED,8BAAI,GAAJ;QACE,IAAI,CAAC,SAAS,GAAG,KAAK,CAAC;IACzB,CAAC;IAEK,iCAAO,GAAb;;;;4BACE,qBAAM,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,EAAA;;wBAA9B,SAA8B,CAAC;;;;;KAChC;IACH,sBAAC;AAAD,CAAC,AAtPD,IAsPC;AAED,qBAAqB;AACrB,IAAI,2BAAU,EAAE,CAAC;IACP,IAAA,MAAM,GAAkC,2BAAU,OAA5C,EAAE,YAAY,GAAoB,2BAAU,aAA9B,EAAE,aAAa,GAAK,2BAAU,cAAf,CAAgB;IAC3D,IAAM,QAAM,GAAG,IAAI,eAAe,CAAC,MAAM,EAAE,YAAY,EAAE,aAAa,CAAC,CAAC;IAExE,mCAAmC;IACnC,2BAAU,CAAC,EAAE,CAAC,SAAS,EAAE,UAAC,GAAG;QAC3B,IAAI,GAAG,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;YACxB,QAAM,CAAC,IAAI,EAAE,CAAC;YACd,QAAM,CAAC,OAAO,EAAE,CAAC,IAAI,CAAC;gBACpB,2BAAU,aAAV,2BAAU,uBAAV,2BAAU,CAAE,WAAW,CAAC,EAAE,IAAI,EAAE,SAAS,EAAE,CAAC,CAAC;YAC/C,CAAC,CAAC,CAAC;QACL,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,iBAAiB;IACjB,QAAM,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,UAAC,KAAK;QACvB,sBAAM,CAAC,KAAK,CAAC,gCAAgC,EAAE,KAAK,CAAC,CAAC;QACtD,2BAAU,aAAV,2BAAU,uBAAV,2BAAU,CAAE,WAAW,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;QACjE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC,CAAC,CAAC;AACL,CAAC","sourcesContent":["import { parentPort, workerData } from 'worker_threads';\nimport { logger } from './worker-logger';\nimport { Florence2Model } from '../florence2-model';\nimport type { Florence2Result, ScreenTile } from '../types';\nimport sharp from 'sharp';\n\ninterface WorkerConfig {\n tileSize: number;\n priorityTiles?: number[]; // Indices of tiles to prioritize\n}\n\ninterface SharedMetadata {\n frameId: number;\n width: number;\n height: number;\n displayIndex: number;\n timestamp: number;\n}\n\nclass Florence2Worker {\n private config: WorkerConfig;\n private sharedBuffer: SharedArrayBuffer;\n private dataView: DataView;\n private atomicState: Int32Array;\n private resultsBuffer: SharedArrayBuffer;\n private resultsView: DataView;\n private florence2: Florence2Model;\n private isRunning = true;\n private frameCount = 0;\n private lastFPSReport = Date.now();\n private lastFrameId = -1;\n\n // Atomic indices for input buffer\n private readonly FRAME_ID_INDEX = 0;\n private readonly WRITE_LOCK_INDEX = 1;\n private readonly WIDTH_INDEX = 2;\n private readonly HEIGHT_INDEX = 3;\n private readonly DISPLAY_INDEX = 4;\n private readonly TIMESTAMP_INDEX = 5;\n private readonly DATA_OFFSET = 24;\n\n // Results buffer structure\n private readonly RESULTS_HEADER_SIZE = 16;\n private readonly MAX_RESULT_SIZE = 4096; // Per tile\n\n constructor(\n config: WorkerConfig,\n sharedBuffer: SharedArrayBuffer,\n resultsBuffer: SharedArrayBuffer\n ) {\n this.config = config;\n this.sharedBuffer = sharedBuffer;\n this.dataView = new DataView(sharedBuffer);\n this.atomicState = new Int32Array(sharedBuffer, 0, 6);\n this.resultsBuffer = resultsBuffer;\n this.resultsView = new DataView(resultsBuffer);\n this.florence2 = new Florence2Model();\n }\n\n async initialize(): Promise<void> {\n await this.florence2.initialize();\n logger.info('[Florence2Worker] Initialized and ready');\n }\n\n async run(): Promise<void> {\n await this.initialize();\n\n logger.info('[Florence2Worker] Starting analysis loop...');\n\n while (this.isRunning) {\n try {\n // Check for new frame\n const currentFrameId = Atomics.load(this.atomicState, this.FRAME_ID_INDEX);\n\n if (currentFrameId > this.lastFrameId) {\n await this.processFrame();\n this.lastFrameId = currentFrameId;\n this.frameCount++;\n\n // Report FPS\n const now = Date.now();\n if (now - this.lastFPSReport >= 1000) {\n const fps = this.frameCount / ((now - this.lastFPSReport) / 1000);\n logger.info(`[Florence2Worker] Analysis FPS: ${fps.toFixed(2)}`);\n\n parentPort?.postMessage({\n type: 'fps',\n fps,\n frameCount: this.frameCount,\n });\n\n this.frameCount = 0;\n this.lastFPSReport = now;\n }\n } else {\n // No new frame, brief yield\n await new Promise((resolve) => setImmediate(resolve));\n }\n } catch (error) {\n logger.error('[Florence2Worker] Processing error:', error);\n await new Promise((resolve) => setTimeout(resolve, 100));\n }\n }\n }\n\n private async processFrame(): Promise<void> {\n // Read metadata atomically\n const metadata: SharedMetadata = {\n frameId: Atomics.load(this.atomicState, this.FRAME_ID_INDEX),\n width: Atomics.load(this.atomicState, this.WIDTH_INDEX),\n height: Atomics.load(this.atomicState, this.HEIGHT_INDEX),\n displayIndex: Atomics.load(this.atomicState, this.DISPLAY_INDEX),\n timestamp: Atomics.load(this.atomicState, this.TIMESTAMP_INDEX),\n };\n\n // Calculate tiles\n const tiles = this.calculateTiles(metadata.width, metadata.height);\n\n // Process priority tiles first\n const tilesToProcess = this.config.priorityTiles\n ? this.config.priorityTiles.map((i) => tiles[i]).filter(Boolean)\n : tiles;\n\n // Process tiles\n for (let i = 0; i < tilesToProcess.length; i++) {\n const tile = tilesToProcess[i];\n if (!tile) {\n continue;\n }\n\n try {\n // Extract tile data from shared buffer\n const tileBuffer = await this.extractTileFromSharedBuffer(tile, metadata);\n\n // Analyze with Florence-2\n const result = await this.florence2.analyzeTile({\n ...tile,\n data: tileBuffer,\n });\n\n // Write result to results buffer\n await this.writeResultToBuffer(tile.id, result, metadata.frameId);\n\n // Notify main thread\n parentPort?.postMessage({\n type: 'tile_analyzed',\n tileId: tile.id,\n frameId: metadata.frameId,\n displayIndex: metadata.displayIndex,\n hasObjects: (result.objects?.length || 0) > 0,\n caption: result.caption,\n });\n } catch (error) {\n logger.error(`[Florence2Worker] Failed to analyze tile ${tile.id}:`, error);\n }\n }\n }\n\n private calculateTiles(width: number, height: number): ScreenTile[] {\n const tileSize = this.config.tileSize;\n const tiles: ScreenTile[] = [];\n\n for (let row = 0; row < Math.ceil(height / tileSize); row++) {\n for (let col = 0; col < Math.ceil(width / tileSize); col++) {\n const x = col * tileSize;\n const y = row * tileSize;\n const tileWidth = Math.min(tileSize, width - x);\n const tileHeight = Math.min(tileSize, height - y);\n\n tiles.push({\n id: `tile-${row}-${col}`,\n row,\n col,\n x,\n y,\n width: tileWidth,\n height: tileHeight,\n });\n }\n }\n\n return tiles;\n }\n\n private async extractTileFromSharedBuffer(\n tile: ScreenTile,\n metadata: SharedMetadata\n ): Promise<Buffer> {\n // Calculate byte positions for the tile\n const bytesPerPixel = 4; // RGBA\n const rowStride = metadata.width * bytesPerPixel;\n\n // Create buffer for tile data\n const tileData = Buffer.allocUnsafe(tile.width * tile.height * bytesPerPixel);\n\n // Copy tile data row by row\n for (let row = 0; row < tile.height; row++) {\n const sourceY = tile.y + row;\n const sourceOffset = this.DATA_OFFSET + sourceY * rowStride + tile.x * bytesPerPixel;\n const destOffset = row * tile.width * bytesPerPixel;\n\n // Copy one row of tile data\n for (let i = 0; i < tile.width * bytesPerPixel; i++) {\n tileData[destOffset + i] = this.dataView.getUint8(sourceOffset + i);\n }\n }\n\n // Convert raw RGBA to PNG for Florence-2\n const pngBuffer = await sharp(tileData, {\n raw: {\n width: tile.width,\n height: tile.height,\n channels: 4,\n },\n })\n .png()\n .toBuffer();\n\n return pngBuffer;\n }\n\n private async writeResultToBuffer(\n tileId: string,\n result: Florence2Result,\n frameId: number\n ): Promise<void> {\n // Serialize result to JSON\n const resultJson = JSON.stringify({\n tileId,\n frameId,\n timestamp: Date.now(),\n ...result,\n });\n\n const resultBytes = Buffer.from(resultJson, 'utf-8');\n\n // Calculate tile index from ID\n const match = tileId.match(/tile-(\\d+)-(\\d+)/);\n if (!match) {\n return;\n }\n\n const row = parseInt(match[1], 10);\n const col = parseInt(match[2], 10);\n const tileIndex = row * 10 + col; // Assuming max 10 columns\n\n // Write to results buffer\n const offset = this.RESULTS_HEADER_SIZE + tileIndex * this.MAX_RESULT_SIZE;\n\n // Write length\n this.resultsView.setUint32(offset, resultBytes.length, true);\n\n // Write data\n for (let i = 0; i < Math.min(resultBytes.length, this.MAX_RESULT_SIZE - 4); i++) {\n this.resultsView.setUint8(offset + 4 + i, resultBytes[i]);\n }\n }\n\n stop(): void {\n this.isRunning = false;\n }\n\n async dispose(): Promise<void> {\n await this.florence2.dispose();\n }\n}\n\n// Worker entry point\nif (parentPort) {\n const { config, sharedBuffer, resultsBuffer } = workerData;\n const worker = new Florence2Worker(config, sharedBuffer, resultsBuffer);\n\n // Handle messages from main thread\n parentPort.on('message', (msg) => {\n if (msg.type === 'stop') {\n worker.stop();\n worker.dispose().then(() => {\n parentPort?.postMessage({ type: 'stopped' });\n });\n }\n });\n\n // Run the worker\n worker.run().catch((error) => {\n logger.error('[Florence2Worker] Fatal error:', error);\n parentPort?.postMessage({ type: 'error', error: error.message });\n process.exit(1);\n });\n}\n"]}
package/dist/workers/ocr-worker.d.ts ADDED
@@ -0,0 +1 @@
1
+ export {};