@zappar/zappar-cv 0.4.0-beta.2 → 0.4.0-beta.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (55) hide show
  1. package/CHANGELOG.md +79 -34
  2. package/README.md +2 -2
  3. package/lib/camera-source-map.d.ts +10 -0
  4. package/lib/camera-source-map.js +40 -0
  5. package/lib/camera-source.d.ts +2 -5
  6. package/lib/camera-source.js +3 -13
  7. package/lib/compatibility.js +1 -1
  8. package/lib/drawcamera.js +2 -1
  9. package/lib/gen/zappar-client.js +6 -1
  10. package/lib/gen/zappar-cwrap.js +37 -21
  11. package/lib/gen/zappar-native.d.ts +11 -0
  12. package/lib/gen/zappar-native.js +13 -1
  13. package/lib/gen/zappar-server.js +1 -1
  14. package/lib/gen/zappar.d.ts +5 -0
  15. package/lib/gen/zappar.js +5 -3
  16. package/lib/gfx.d.ts +1 -0
  17. package/lib/gfx.js +4 -0
  18. package/lib/html-element-source.d.ts +3 -20
  19. package/lib/html-element-source.js +19 -263
  20. package/lib/image-process-gl.d.ts +37 -0
  21. package/lib/image-process-gl.js +265 -0
  22. package/lib/imagebitmap-camera-source.d.ts +36 -0
  23. package/lib/imagebitmap-camera-source.js +277 -0
  24. package/lib/mstp-camera-source.d.ts +29 -0
  25. package/lib/mstp-camera-source.js +230 -0
  26. package/lib/native.js +45 -8
  27. package/lib/pipeline.d.ts +13 -3
  28. package/lib/pipeline.js +95 -26
  29. package/lib/profile.d.ts +2 -0
  30. package/lib/profile.js +55 -44
  31. package/lib/sequencerecorder.d.ts +1 -0
  32. package/lib/sequencerecorder.js +11 -0
  33. package/lib/sequencesource.d.ts +7 -1
  34. package/lib/sequencesource.js +89 -47
  35. package/lib/source.d.ts +6 -4
  36. package/lib/version.d.ts +1 -1
  37. package/lib/version.js +1 -1
  38. package/lib/worker-client.js +0 -5
  39. package/lib/worker-imagebitmap.d.ts +5 -0
  40. package/lib/worker-imagebitmap.js +63 -0
  41. package/lib/worker-server.d.ts +0 -5
  42. package/lib/worker-server.js +120 -30
  43. package/lib/workerinterface.d.ts +47 -1
  44. package/lib/zcv.js +117 -135
  45. package/lib/zcv.wasm +0 -0
  46. package/package.json +34 -31
  47. package/umd/fbf9061bc2d2826ecf97c8ba2b51e7a5.wasm +0 -0
  48. package/umd/zappar-cv.js +1 -9
  49. package/umd/zappar-cv.worker.js +1 -1
  50. package/lib/bundleworker-mainthread.d.ts +0 -2
  51. package/lib/bundleworker-mainthread.js +0 -36
  52. package/lib/bundleworker-worker.d.ts +0 -3
  53. package/lib/bundleworker-worker.js +0 -47
  54. package/umd/5dd5cacd27e6e7da828f.bundleworker.js +0 -1
  55. package/umd/a20150d17b0290cadaa6cb9911ab9500.wasm +0 -0
package/lib/gfx.js ADDED
@@ -0,0 +1,4 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.gfx = void 0;
4
+ exports.gfx = `<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" width="250" height="80.408" viewBox="0 0 250 80.408"><defs><filter id="a" x="0" y="1.182" width="250" height="79.226" filterUnits="userSpaceOnUse"><feOffset dy="3" input="SourceAlpha"/><feGaussianBlur stdDeviation="3" result="b"/><feFlood flood-opacity="0.161"/><feComposite operator="in" in2="b"/><feComposite in="SourceGraphic"/></filter><filter id="c" x="6.929" y="0" width="111.975" height="40.988" filterUnits="userSpaceOnUse"><feOffset dy="3" input="SourceAlpha"/><feGaussianBlur stdDeviation="3" result="d"/><feFlood flood-opacity="0.161"/><feComposite operator="in" in2="d"/><feComposite in="SourceGraphic"/></filter></defs><g transform="translate(-26.873 -414.029)"><g transform="translate(44 423.332)"><g transform="translate(0 0.635)" style="isolation:isolate"><g transform="translate(0 0)"><g transform="matrix(1, 0, 0, 1, -17.13, -9.94)" filter="url(#a)"><path d="M55.766,44.97V11.8h8.272v2.6h.089a8.258,8.258,0,0,1,7.046-3.357,11.4,11.4,0,0,1,8.5,3.686,12.667,12.667,0,0,1,3.278,8.622,12.509,12.509,0,0,1-3.25,8.57,11.353,11.353,0,0,1-8.429,3.6,9.533,9.533,0,0,1-7.229-2.95V43.378l-8.269,1.594h0Zm8-21.671a4.958,4.958,0,0,0,5.319,5.15A4.957,4.957,0,0,0,74.4,23.3a5.321,5.321,0,0,0-10.638,0ZM16.105,18.412H2.752V11.038H30.9L14.887,37.16H38.22L0,44.534ZM131.469,38.45a.761.761,0,0,1-.484-.166.582.582,0,0,1-.228-.419l-.553-3.586a12.243,12.243,0,0,1-1.782-.721l-2.789,2.084a.718.718,0,0,1-.5.176.688.688,0,0,1-.495-.214,25.864,25.864,0,0,1-3.264-3.274.754.754,0,0,1,.018-.9c.2-.277.543-.713,1.01-1.3.483-.6.832-1.05,1.068-1.374a9.614,9.614,0,0,1-.811-1.929l-3.62-.527a.631.631,0,0,1-.416-.243.728.728,0,0,1-.159-.458V21.283a.743.743,0,0,1,.159-.448.6.6,0,0,1,.376-.253l3.68-.546a9.511,9.511,0,0,1,.772-1.792c-.523-.733-1.235-1.638-2.117-2.69a.731.731,0,0,1-.2-.467.82.82,0,0,1,.178-.449,19.577,19.577,0,0,1,1.949-2.095c.953-.924,1.582-1.393,1.87-1.393a.861.861,0,0,1,.515.195l2.729,2.084a11.551,11.551,0,0,1,1.8-.741,31.446,31.446,0,0,1,.573-3.623.666.666,0,0,1,.712-.546h4.392a.757.757,0,0,1,.485.166.58.58,0,0,1,.228.419l.553,3.584a12.576,12.576,0,0,1,1.782.722l2.809-2.085a.641.641,0,0,1,.475-.176.776.776,0,0,1,.5.195,27.035,27.035,0,0,1,3.264,3.313.625.625,0,0,1,.139.429.742.742,0,0,1-.159.447c-.2.271-.525.7-1.008,1.3-.467.581-.827,1.043-1.069,1.373a11.8,11.8,0,0,1,.812,1.91l3.62.545a.626.626,0,0,1,.416.244.736.736,0,0,1,.158.457v4.325a.755.755,0,0,1-.158.449.6.6,0,0,1-.4.253l-3.66.545a12.692,12.692,0,0,1-.772,1.774c.465.653,1.177,1.559,2.117,2.688a.751.751,0,0,1,.2.488.648.648,0,0,1-.178.448,19.638,19.638,0,0,1-1.958,2.1c-.947.918-1.573,1.384-1.861,1.384a.976.976,0,0,1-.515-.176l-2.731-2.1a11.361,11.361,0,0,1-1.8.741,31.849,31.849,0,0,1-.573,3.625.668.668,0,0,1-.713.546Zm-1.385-18.491a4.935,4.935,0,0,0,0,7.053,5.12,5.12,0,0,0,7.163,0,4.931,4.931,0,0,0,0-7.053,5.117,5.117,0,0,0-7.163,0Zm69.969,14.868a18.523,18.523,0,0,1-4.932-2.073l1.705-6.1A13.441,13.441,0,0,0,200.779,29a12.14,12.14,0,0,0,4.344.818,4.644,4.644,0,0,0,2.351-.482,1.538,1.538,0,0,0,.806-1.4q0-1.5-3.25-2.242-1.061-.246-1.705-.425a12.335,12.335,0,0,1-5.358-2.589,5.5,5.5,0,0,1-1.671-4.157,6.323,6.323,0,0,1,2.547-5.267,10.993,10.993,0,0,1,6.9-1.972,21.351,21.351,0,0,1,4.552.5,25.439,25.439,0,0,1,4.736,1.557l-1.705,5.491a14.244,14.244,0,0,0-3.319-1.636,10.6,10.6,0,0,0-3.364-.561,4.227,4.227,0,0,0-2.121.426,1.357,1.357,0,0,0-.714,1.232q0,1.211,3.227,2.04.737.2,1.106.291,4.264,1.187,5.935,2.79a5.728,5.728,0,0,1,1.67,4.337A6.781,6.781,0,0,1,213,33.37a11.48,11.48,0,0,1-7.259,2.14A22.577,22.577,0,0,1,200.053,34.827ZM30.568,31.834a12.566,12.566,0,0,1-3.292-8.59,12.567,12.567,0,0,1,3.247-8.568,11.3,11.3,0,0,1,8.423-3.638,9.519,9.519,0,0,1,7.217,3V11.8h8.264V34.735H46.163V32.186h-.09c-1.32,2.071-3.951,3.308-7.038,3.308A11.412,11.412,0,0,1,30.568,31.834Zm5.245-8.545a4.95,4.95,0,0,0,5.312,5.141,4.95,4.95,0,0,0,5.312-5.141A5.023,5.023,0,0,0,41.125,18.1,5.023,5.023,0,0,0,35.813,23.289Zm69.2,11.554-3.527-10.452q-.282-.892-.505-1.8t-.412-1.8a30.821,30.821,0,0,1-.917,3.454l-.047.16L96.1,34.842H90.176l-9.925-23.1h8.632l3.552,10.223q.4,1.12.634,1.955t.377,1.635a23.558,23.558,0,0,1,.917-3.179q.093-.274.141-.411l3.528-10.223h5.056l3.5,10.2.07.183a20.724,20.724,0,0,1,.988,3.43q.141-.732.364-1.532t.647-2.058l3.574-10.223h8.678l-9.971,23.1Zm80.154-.092-8.587-9.841v9.841h-7.888V1.5L176.583,0V22.28l8.272-9.665h9.96L184.584,23.266l11.156,11.485Zm-35.09,0V11.782h8.272v3.761h.091a8.131,8.131,0,0,1,7.546-4.074h1.593V19.3a6.8,6.8,0,0,0-3.5-.761c-4.593,0-5.728,3-5.728,6.941v9.268Z" transform="translate(17.13 15.31)"/></g><g transform="matrix(1, 0, 0, 1, -17.13, -9.94)" filter="url(#c)"><path d="M3.576-12.852v-4.3H6.628a2.867,2.867,0,0,1,1.945.55A2.034,2.034,0,0,1,9.188-15a2.06,2.06,0,0,1-.615,1.608,2.784,2.784,0,0,1-1.945.542ZM2.017-18.464V-6.748H3.576v-4.792H7.153a3.569,3.569,0,0,0,2.683-.9A3.478,3.478,0,0,0,10.747-15a3.45,3.45,0,0,0-.911-2.552,3.638,3.638,0,0,0-2.683-.911Zm11.421,7.483a4.221,4.221,0,0,1,.205-1.37,2.957,2.957,0,0,1,.566-1.009,2.411,2.411,0,0,1,.845-.624A2.515,2.515,0,0,1,16.08-14.2a2.515,2.515,0,0,1,1.026.213,2.411,2.411,0,0,1,.845.624,2.957,2.957,0,0,1,.566,1.009,4.221,4.221,0,0,1,.205,1.37,4.221,4.221,0,0,1-.205,1.37,2.867,2.867,0,0,1-.566,1,2.457,2.457,0,0,1-.845.615,2.515,2.515,0,0,1-1.026.213,2.515,2.515,0,0,1-1.026-.213,2.457,2.457,0,0,1-.845-.615,2.867,2.867,0,0,1-.566-1A4.221,4.221,0,0,1,13.438-10.981Zm-1.477,0a5.573,5.573,0,0,0,.263,1.739,3.97,3.97,0,0,0,.788,1.4,3.623,3.623,0,0,0,1.3.935,4.4,4.4,0,0,0,1.772.336A4.381,4.381,0,0,0,17.86-6.9a3.641,3.641,0,0,0,1.288-.935,3.97,3.97,0,0,0,.788-1.4,5.573,5.573,0,0,0,.263-1.739,5.65,5.65,0,0,0-.263-1.748,3.96,3.96,0,0,0-.788-1.411,3.729,3.729,0,0,0-1.288-.944,4.29,4.29,0,0,0-1.78-.345,4.3,4.3,0,0,0-1.772.345,3.71,3.71,0,0,0-1.3.944,3.96,3.96,0,0,0-.788,1.411A5.65,5.65,0,0,0,11.961-10.981Zm18.28,4.234,2.708-8.484H31.5L29.6-8.29h-.033L27.8-15.231H26.27L24.564-8.29h-.033l-1.92-6.941H21.068l2.724,8.484H25.3l1.707-6.744h.033l1.723,6.744ZM40.169-11.8H35.3a2.905,2.905,0,0,1,.213-.927A2.473,2.473,0,0,1,36-13.492a2.255,2.255,0,0,1,.747-.517,2.429,2.429,0,0,1,.976-.189,2.386,2.386,0,0,1,.96.189,2.415,2.415,0,0,1,.755.509,2.391,2.391,0,0,1,.509.763A2.854,2.854,0,0,1,40.169-11.8Zm1.346,2.363H40.136a1.983,1.983,0,0,1-.747,1.247,2.414,2.414,0,0,1-1.452.41,2.889,2.889,0,0,1-1.2-.23,2.348,2.348,0,0,1-.837-.615,2.4,2.4,0,0,1-.476-.886,3.367,3.367,0,0,1-.131-1.058h6.35a6.754,6.754,0,0,0-.139-1.625,4.7,4.7,0,0,0-.624-1.575,3.716,3.716,0,0,0-1.2-1.19,3.48,3.48,0,0,0-1.879-.468,3.858,3.858,0,0,0-1.6.328,3.76,3.76,0,0,0-1.264.919,4.236,4.236,0,0,0-.829,1.395,5.105,5.105,0,0,0-.3,1.772,6.8,6.8,0,0,0,.287,1.789,3.9,3.9,0,0,0,.763,1.411,3.478,3.478,0,0,0,1.255.919,4.358,4.358,0,0,0,1.764.328,3.851,3.851,0,0,0,2.4-.722A3.39,3.39,0,0,0,41.515-9.439Zm1.526-5.793v8.484h1.395v-3.774a5.8,5.8,0,0,1,.164-1.452,2.784,2.784,0,0,1,.525-1.075,2.251,2.251,0,0,1,.952-.673,3.945,3.945,0,0,1,1.428-.23v-1.477a3.135,3.135,0,0,0-1.871.459,3.779,3.779,0,0,0-1.247,1.526h-.033v-1.789ZM54.15-11.8H49.276a2.905,2.905,0,0,1,.213-.927,2.473,2.473,0,0,1,.492-.763,2.255,2.255,0,0,1,.747-.517A2.429,2.429,0,0,1,51.7-14.2a2.386,2.386,0,0,1,.96.189,2.415,2.415,0,0,1,.755.509,2.391,2.391,0,0,1,.509.763A2.854,2.854,0,0,1,54.15-11.8ZM55.5-9.439H54.117a1.983,1.983,0,0,1-.747,1.247,2.414,2.414,0,0,1-1.452.41,2.889,2.889,0,0,1-1.2-.23,2.348,2.348,0,0,1-.837-.615,2.4,2.4,0,0,1-.476-.886,3.367,3.367,0,0,1-.131-1.058h6.35a6.754,6.754,0,0,0-.139-1.625,4.7,4.7,0,0,0-.624-1.575,3.716,3.716,0,0,0-1.2-1.19,3.48,3.48,0,0,0-1.879-.468,3.858,3.858,0,0,0-1.6.328,3.76,3.76,0,0,0-1.264.919,4.236,4.236,0,0,0-.829,1.395,5.1,5.1,0,0,0-.3,1.772,6.8,6.8,0,0,0,.287,1.789,3.9,3.9,0,0,0,.763,1.411,3.478,3.478,0,0,0,1.255.919,4.358,4.358,0,0,0,1.764.328,3.851,3.851,0,0,0,2.4-.722A3.39,3.39,0,0,0,55.5-9.439Zm2.593-1.477a5.581,5.581,0,0,1,.131-1.214,3.212,3.212,0,0,1,.427-1.05,2.221,2.221,0,0,1,.788-.738,2.4,2.4,0,0,1,1.2-.279,2.589,2.589,0,0,1,1.231.271,2.389,2.389,0,0,1,.829.714,2.985,2.985,0,0,1,.468,1.026,4.889,4.889,0,0,1,.148,1.206,4.919,4.919,0,0,1-.139,1.165,3.053,3.053,0,0,1-.451,1.026,2.4,2.4,0,0,1-.8.73,2.4,2.4,0,0,1-1.2.279,2.491,2.491,0,0,1-1.173-.263,2.332,2.332,0,0,1-.82-.706,3.122,3.122,0,0,1-.476-1.009A4.338,4.338,0,0,1,58.088-10.916Zm6.564,4.168V-18.464H63.257V-14.1h-.033a2.247,2.247,0,0,0-.566-.632,3.216,3.216,0,0,0-.714-.41,3.583,3.583,0,0,0-.755-.222,4.132,4.132,0,0,0-.706-.066,3.861,3.861,0,0,0-1.7.353,3.487,3.487,0,0,0-1.214.952,4.018,4.018,0,0,0-.722,1.4,5.985,5.985,0,0,0-.238,1.707,5.8,5.8,0,0,0,.246,1.707,4.218,4.218,0,0,0,.73,1.411,3.529,3.529,0,0,0,1.214.968,3.827,3.827,0,0,0,1.715.361,4.135,4.135,0,0,0,1.625-.312A2.053,2.053,0,0,0,63.224-7.9h.033v1.149Zm6.761-11.716V-6.748h1.395V-7.88h.033a2.247,2.247,0,0,0,.566.632,2.982,2.982,0,0,0,.714.4,3.874,3.874,0,0,0,.763.213,4.193,4.193,0,0,0,.7.066,3.94,3.94,0,0,0,1.7-.345,3.4,3.4,0,0,0,1.214-.944,4.018,4.018,0,0,0,.722-1.4,5.985,5.985,0,0,0,.238-1.707,5.8,5.8,0,0,0-.246-1.707,4.312,4.312,0,0,0-.73-1.419,3.488,3.488,0,0,0-1.214-.976,3.827,3.827,0,0,0-1.715-.361,4.043,4.043,0,0,0-1.625.32,2.088,2.088,0,0,0-1.083,1.026h-.033v-4.381Zm6.564,7.4a5.581,5.581,0,0,1-.131,1.214,3.212,3.212,0,0,1-.427,1.05,2.243,2.243,0,0,1-.779.738,2.385,2.385,0,0,1-1.206.279A2.589,2.589,0,0,1,74.2-8.052a2.389,2.389,0,0,1-.829-.714,2.985,2.985,0,0,1-.468-1.026A4.889,4.889,0,0,1,72.758-11a4.919,4.919,0,0,1,.139-1.165,3.053,3.053,0,0,1,.451-1.026,2.4,2.4,0,0,1,.8-.73,2.4,2.4,0,0,1,1.2-.279,2.491,2.491,0,0,1,1.173.263,2.332,2.332,0,0,1,.82.706,3.122,3.122,0,0,1,.476,1.009A4.338,4.338,0,0,1,77.976-11.063Zm6.449,5.448a8.871,8.871,0,0,1-.484,1.05,2.871,2.871,0,0,1-.525.7,1.782,1.782,0,0,1-.64.394,2.462,2.462,0,0,1-.812.123,3.721,3.721,0,0,1-.492-.033A2.324,2.324,0,0,1,81-3.5v-1.28a2.342,2.342,0,0,0,.418.139,1.765,1.765,0,0,0,.4.057,1.2,1.2,0,0,0,.714-.205,1.255,1.255,0,0,0,.435-.583L83.539-6.8l-3.364-8.434h1.575L84.228-8.29h.033l2.379-6.941h1.477Z" transform="translate(17.85 28.4)"/></g></g></g><g transform="translate(0 0)"><g transform="translate(0 5.371)"><path d="M11.76,24.456A11.41,11.41,0,0,1,3.292,20.8,12.564,12.564,0,0,1,0,12.206,12.569,12.569,0,0,1,3.246,3.638,11.307,11.307,0,0,1,11.67,0a9.519,9.519,0,0,1,7.217,3V.759h8.264V23.7H18.887V21.148H18.8C17.478,23.219,14.846,24.456,11.76,24.456ZM13.849,7.064A5.023,5.023,0,0,0,8.537,12.25a4.95,4.95,0,0,0,5.312,5.141A4.95,4.95,0,0,0,19.16,12.25,5.023,5.023,0,0,0,13.849,7.064Z" transform="translate(27.277 11.038)" fill="#fff"/><path d="M0,33.931V.758H8.271v2.6H8.36A8.258,8.258,0,0,1,15.406,0,11.4,11.4,0,0,1,23.9,3.686a12.665,12.665,0,0,1,3.278,8.622,12.508,12.508,0,0,1-3.25,8.57,11.351,11.351,0,0,1-8.429,3.6,9.535,9.535,0,0,1-7.229-2.95V32.337L0,33.931ZM13.317,7.07A5.028,5.028,0,0,0,8,12.259a4.957,4.957,0,0,0,5.319,5.15,4.957,4.957,0,0,0,5.319-5.15A5.028,5.028,0,0,0,13.317,7.07Z" transform="translate(55.766 11.041)" fill="#fff"/><path d="M22.836,0l3.5,10.2.07.183a20.748,20.748,0,0,1,.988,3.43q.141-.731.364-1.532t.647-2.058L31.985,0h8.678L30.691,23.1H24.764L21.237,12.647q-.282-.891-.505-1.8t-.412-1.8A30.927,30.927,0,0,1,19.4,12.51l-.047.16L15.851,23.1H9.925L0,0H8.631l3.552,10.223q.4,1.121.635,1.955t.377,1.635a23.522,23.522,0,0,1,.917-3.179q.093-.274.141-.411L17.78,0h5.056" transform="translate(80.251 11.743)" fill="#fff"/><path d="M0,34.75V1.5L7.888,0V22.28l8.272-9.665H26.12L15.889,23.266,27.045,34.75H16.475L7.888,24.91v9.84H0" transform="translate(168.695 0)" fill="#fff"/><path d="M0,21.47l1.705-6.1a13.453,13.453,0,0,0,3.952,2.342A12.145,12.145,0,0,0,10,18.534a4.643,4.643,0,0,0,2.351-.482,1.539,1.539,0,0,0,.807-1.4q0-1.5-3.25-2.241-1.06-.246-1.705-.426A12.325,12.325,0,0,1,2.846,11.4,5.5,5.5,0,0,1,1.176,7.239,6.324,6.324,0,0,1,3.722,1.972,11,11,0,0,1,10.624,0a21.365,21.365,0,0,1,4.551.5,25.45,25.45,0,0,1,4.736,1.558l-1.706,5.49a14.24,14.24,0,0,0-3.319-1.636,10.6,10.6,0,0,0-3.364-.56A4.22,4.22,0,0,0,9.4,5.782a1.356,1.356,0,0,0-.714,1.233q0,1.211,3.227,2.04.737.2,1.106.291,4.264,1.188,5.935,2.79a5.728,5.728,0,0,1,1.671,4.337,6.782,6.782,0,0,1-2.743,5.614,11.478,11.478,0,0,1-7.259,2.14,22.565,22.565,0,0,1-5.693-.683A18.517,18.517,0,0,1,0,21.47" transform="translate(195.121 11.284)" fill="#fff"/><path d="M8.273,4.075h.091A8.13,8.13,0,0,1,15.909,0H17.5V7.834A6.8,6.8,0,0,0,14,7.074c-4.593,0-5.728,3-5.728,6.941v9.268H0V.313H8.273V4.075" transform="translate(150.08 11.469)" fill="#fff"/><path d="M17.392,29.929H13a.76.76,0,0,1-.485-.166.579.579,0,0,1-.227-.419l-.553-3.586a12.3,12.3,0,0,1-1.781-.721L7.163,27.123a.719.719,0,0,1-.495.176.687.687,0,0,1-.495-.214A25.873,25.873,0,0,1,2.91,23.81a.753.753,0,0,1,.019-.9c.2-.277.542-.713,1.009-1.3.483-.6.832-1.05,1.068-1.374a9.591,9.591,0,0,1-.811-1.929l-3.62-.526a.63.63,0,0,1-.416-.243A.729.729,0,0,1,0,17.089V12.762a.746.746,0,0,1,.159-.448.6.6,0,0,1,.376-.253l3.68-.546a9.5,9.5,0,0,1,.772-1.792C4.464,8.991,3.752,8.085,2.87,7.034a.732.732,0,0,1-.2-.468.82.82,0,0,1,.179-.449A19.622,19.622,0,0,1,4.8,4.023C5.751,3.1,6.38,2.631,6.668,2.631a.859.859,0,0,1,.515.194L9.913,4.91a11.536,11.536,0,0,1,1.8-.741A31.372,31.372,0,0,1,12.287.546.665.665,0,0,1,13,0h4.392a.755.755,0,0,1,.485.166.581.581,0,0,1,.229.419l.553,3.584a12.543,12.543,0,0,1,1.781.722L23.25,2.807a.641.641,0,0,1,.475-.176.775.775,0,0,1,.495.194,27.092,27.092,0,0,1,3.264,3.313.625.625,0,0,1,.139.429.745.745,0,0,1-.159.448c-.2.271-.525.695-1.008,1.3-.467.581-.827,1.043-1.069,1.373a11.784,11.784,0,0,1,.812,1.91l3.62.545a.625.625,0,0,1,.416.244.736.736,0,0,1,.157.458v4.325a.754.754,0,0,1-.157.449.606.606,0,0,1-.4.253l-3.66.545a12.628,12.628,0,0,1-.772,1.774c.465.654,1.177,1.559,2.117,2.688a.752.752,0,0,1,.2.488.648.648,0,0,1-.177.448,19.641,19.641,0,0,1-1.959,2.1c-.947.918-1.573,1.384-1.86,1.384a.978.978,0,0,1-.515-.176l-2.731-2.1a11.429,11.429,0,0,1-1.8.741,32.016,32.016,0,0,1-.573,3.624A.668.668,0,0,1,17.392,29.929ZM15.2,9.976a4.938,4.938,0,0,0-3.582,1.463,4.935,4.935,0,0,0,0,7.053,5.12,5.12,0,0,0,7.163,0,4.932,4.932,0,0,0,0-7.053A4.938,4.938,0,0,0,15.2,9.976Z" transform="translate(118.47 8.521)" fill="#fff"/><path d="M0,33.5,16.106,7.374H2.752V0H30.9L14.887,26.122H38.22L0,33.5" transform="translate(0 11.038)" fill="#fff"/></g><path d="M3.576-12.852v-4.3H6.628a2.867,2.867,0,0,1,1.945.55A2.034,2.034,0,0,1,9.188-15a2.06,2.06,0,0,1-.615,1.608,2.784,2.784,0,0,1-1.945.542ZM2.017-18.464V-6.748H3.576v-4.792H7.153a3.569,3.569,0,0,0,2.683-.9A3.478,3.478,0,0,0,10.747-15a3.45,3.45,0,0,0-.911-2.552,3.638,3.638,0,0,0-2.683-.911Zm11.421,7.483a4.221,4.221,0,0,1,.205-1.37,2.957,2.957,0,0,1,.566-1.009,2.411,2.411,0,0,1,.845-.624A2.515,2.515,0,0,1,16.08-14.2a2.515,2.515,0,0,1,1.026.213,2.411,2.411,0,0,1,.845.624,2.957,2.957,0,0,1,.566,1.009,4.221,4.221,0,0,1,.205,1.37,4.221,4.221,0,0,1-.205,1.37,2.867,2.867,0,0,1-.566,1,2.457,2.457,0,0,1-.845.615,2.515,2.515,0,0,1-1.026.213,2.515,2.515,0,0,1-1.026-.213,2.457,2.457,0,0,1-.845-.615,2.867,2.867,0,0,1-.566-1A4.221,4.221,0,0,1,13.438-10.981Zm-1.477,0a5.573,5.573,0,0,0,.263,1.739,3.97,3.97,0,0,0,.788,1.4,3.623,3.623,0,0,0,1.3.935,4.4,4.4,0,0,0,1.772.336A4.381,4.381,0,0,0,17.86-6.9a3.641,3.641,0,0,0,1.288-.935,3.97,3.97,0,0,0,.788-1.4,5.573,5.573,0,0,0,.263-1.739,5.65,5.65,0,0,0-.263-1.748,3.96,3.96,0,0,0-.788-1.411,3.729,3.729,0,0,0-1.288-.944,4.29,4.29,0,0,0-1.78-.345,4.3,4.3,0,0,0-1.772.345,3.71,3.71,0,0,0-1.3.944,3.96,3.96,0,0,0-.788,1.411A5.65,5.65,0,0,0,11.961-10.981Zm18.28,4.234,2.708-8.484H31.5L29.6-8.29h-.033L27.8-15.231H26.27L24.564-8.29h-.033l-1.92-6.941H21.068l2.724,8.484H25.3l1.707-6.744h.033l1.723,6.744ZM40.169-11.8H35.3a2.905,2.905,0,0,1,.213-.927A2.473,2.473,0,0,1,36-13.492a2.255,2.255,0,0,1,.747-.517,2.429,2.429,0,0,1,.976-.189,2.386,2.386,0,0,1,.96.189,2.415,2.415,0,0,1,.755.509,2.391,2.391,0,0,1,.509.763A2.854,2.854,0,0,1,40.169-11.8Zm1.346,2.363H40.136a1.983,1.983,0,0,1-.747,1.247,2.414,2.414,0,0,1-1.452.41,2.889,2.889,0,0,1-1.2-.23,2.348,2.348,0,0,1-.837-.615,2.4,2.4,0,0,1-.476-.886,3.367,3.367,0,0,1-.131-1.058h6.35a6.754,6.754,0,0,0-.139-1.625,4.7,4.7,0,0,0-.624-1.575,3.716,3.716,0,0,0-1.2-1.19,3.48,3.48,0,0,0-1.879-.468,3.858,3.858,0,0,0-1.6.328,3.76,3.76,0,0,0-1.264.919,4.236,4.236,0,0,0-.829,1.395,5.105,5.105,0,0,0-.3,1.772,6.8,6.8,0,0,0,.287,1.789,3.9,3.9,0,0,0,.763,1.411,3.478,3.478,0,0,0,1.255.919,4.358,4.358,0,0,0,1.764.328,3.851,3.851,0,0,0,2.4-.722A3.39,3.39,0,0,0,41.515-9.439Zm1.526-5.793v8.484h1.395v-3.774a5.8,5.8,0,0,1,.164-1.452,2.784,2.784,0,0,1,.525-1.075,2.251,2.251,0,0,1,.952-.673,3.945,3.945,0,0,1,1.428-.23v-1.477a3.135,3.135,0,0,0-1.871.459,3.779,3.779,0,0,0-1.247,1.526h-.033v-1.789ZM54.15-11.8H49.276a2.905,2.905,0,0,1,.213-.927,2.473,2.473,0,0,1,.492-.763,2.255,2.255,0,0,1,.747-.517A2.429,2.429,0,0,1,51.7-14.2a2.386,2.386,0,0,1,.96.189,2.415,2.415,0,0,1,.755.509,2.391,2.391,0,0,1,.509.763A2.854,2.854,0,0,1,54.15-11.8ZM55.5-9.439H54.117a1.983,1.983,0,0,1-.747,1.247,2.414,2.414,0,0,1-1.452.41,2.889,2.889,0,0,1-1.2-.23,2.348,2.348,0,0,1-.837-.615,2.4,2.4,0,0,1-.476-.886,3.367,3.367,0,0,1-.131-1.058h6.35a6.754,6.754,0,0,0-.139-1.625,4.7,4.7,0,0,0-.624-1.575,3.716,3.716,0,0,0-1.2-1.19,3.48,3.48,0,0,0-1.879-.468,3.858,3.858,0,0,0-1.6.328,3.76,3.76,0,0,0-1.264.919,4.236,4.236,0,0,0-.829,1.395,5.1,5.1,0,0,0-.3,1.772,6.8,6.8,0,0,0,.287,1.789,3.9,3.9,0,0,0,.763,1.411,3.478,3.478,0,0,0,1.255.919,4.358,4.358,0,0,0,1.764.328,3.851,3.851,0,0,0,2.4-.722A3.39,3.39,0,0,0,55.5-9.439Zm2.593-1.477a5.581,5.581,0,0,1,.131-1.214,3.212,3.212,0,0,1,.427-1.05,2.221,2.221,0,0,1,.788-.738,2.4,2.4,0,0,1,1.2-.279,2.589,2.589,0,0,1,1.231.271,2.389,2.389,0,0,1,.829.714,2.985,2.985,0,0,1,.468,1.026,4.889,4.889,0,0,1,.148,1.206,4.919,4.919,0,0,1-.139,1.165,3.053,3.053,0,0,1-.451,1.026,2.4,2.4,0,0,1-.8.73,2.4,2.4,0,0,1-1.2.279,2.491,2.491,0,0,1-1.173-.263,2.332,2.332,0,0,1-.82-.706,3.122,3.122,0,0,1-.476-1.009A4.338,4.338,0,0,1,58.088-10.916Zm6.564,4.168V-18.464H63.257V-14.1h-.033a2.247,2.247,0,0,0-.566-.632,3.216,3.216,0,0,0-.714-.41,3.583,3.583,0,0,0-.755-.222,4.132,4.132,0,0,0-.706-.066,3.861,3.861,0,0,0-1.7.353,3.487,3.487,0,0,0-1.214.952,4.018,4.018,0,0,0-.722,1.4,5.985,5.985,0,0,0-.238,1.707,5.8,5.8,0,0,0,.246,1.707,4.218,4.218,0,0,0,.73,1.411,3.529,3.529,0,0,0,1.214.968,3.827,3.827,0,0,0,1.715.361,4.135,4.135,0,0,0,1.625-.312A2.053,2.053,0,0,0,63.224-7.9h.033v1.149Zm6.761-11.716V-6.748h1.395V-7.88h.033a2.247,2.247,0,0,0,.566.632,2.982,2.982,0,0,0,.714.4,3.874,3.874,0,0,0,.763.213,4.193,4.193,0,0,0,.7.066,3.94,3.94,0,0,0,1.7-.345,3.4,3.4,0,0,0,1.214-.944,4.018,4.018,0,0,0,.722-1.4,5.985,5.985,0,0,0,.238-1.707,5.8,5.8,0,0,0-.246-1.707,4.312,4.312,0,0,0-.73-1.419,3.488,3.488,0,0,0-1.214-.976,3.827,3.827,0,0,0-1.715-.361,4.043,4.043,0,0,0-1.625.32,2.088,2.088,0,0,0-1.083,1.026h-.033v-4.381Zm6.564,7.4a5.581,5.581,0,0,1-.131,1.214,3.212,3.212,0,0,1-.427,1.05,2.243,2.243,0,0,1-.779.738,2.385,2.385,0,0,1-1.206.279A2.589,2.589,0,0,1,74.2-8.052a2.389,2.389,0,0,1-.829-.714,2.985,2.985,0,0,1-.468-1.026A4.889,4.889,0,0,1,72.758-11a4.919,4.919,0,0,1,.139-1.165,3.053,3.053,0,0,1,.451-1.026,2.4,2.4,0,0,1,.8-.73,2.4,2.4,0,0,1,1.2-.279,2.491,2.491,0,0,1,1.173.263,2.332,2.332,0,0,1,.82.706,3.122,3.122,0,0,1,.476,1.009A4.338,4.338,0,0,1,77.976-11.063Zm6.449,5.448a8.871,8.871,0,0,1-.484,1.05,2.871,2.871,0,0,1-.525.7,1.782,1.782,0,0,1-.64.394,2.462,2.462,0,0,1-.812.123,3.721,3.721,0,0,1-.492-.033A2.324,2.324,0,0,1,81-3.5v-1.28a2.342,2.342,0,0,0,.418.139,1.765,1.765,0,0,0,.4.057,1.2,1.2,0,0,0,.714-.205,1.255,1.255,0,0,0,.435-.583L83.539-6.8l-3.364-8.434h1.575L84.228-8.29h.033l2.379-6.941h1.477Z" transform="translate(0.722 18.464)" fill="#fff"/></g></g></g></svg>`;
@@ -1,5 +1,5 @@
1
1
  import { zappar_pipeline_t } from "./gen/zappar";
2
- import { Source, CameraFrameInfo } from "./source";
2
+ import { Source } from "./source";
3
3
  import { zappar_html_element_source_t } from "./additional";
4
4
  export declare class HTMLElementSource extends Source {
5
5
  protected _video: HTMLVideoElement | HTMLImageElement;
@@ -10,19 +10,10 @@ export declare class HTMLElementSource extends Source {
10
10
  protected _cameraToScreenRotation: number;
11
11
  private _isUploadFrame;
12
12
  private _currentVideoTexture;
13
- private _computedTransformRotation;
14
- private _computedFrontCameraRotation;
15
- private _cameraUvTransform;
13
+ private _imageProcessor;
16
14
  private _cameraToDeviceTransform;
17
15
  private _cameraToDeviceTransformUserFacing;
18
16
  private _cameraModel;
19
- private _framebufferWidth;
20
- private _framebufferHeight;
21
- private _framebufferId;
22
- private _renderTexture;
23
- private _vertexBuffer;
24
- private _indexBuffer;
25
- private _greyscaleShader;
26
17
  static createVideoElementSource(p: zappar_pipeline_t, element: HTMLVideoElement | HTMLImageElement): zappar_html_element_source_t;
27
18
  static getVideoElementSource(m: zappar_html_element_source_t): HTMLElementSource | undefined;
28
19
  constructor(_video: HTMLVideoElement | HTMLImageElement, _pipeline: zappar_pipeline_t);
@@ -30,15 +21,7 @@ export declare class HTMLElementSource extends Source {
30
21
  destroy(): void;
31
22
  pause(): void;
32
23
  start(): void;
33
- getFrame(currentlyProcessing: boolean): CameraFrameInfo | undefined;
24
+ getFrame(currentlyProcessing: boolean): void;
34
25
  private _processFrame;
35
- private _uploadFrame;
36
- private _readFrame;
37
- private _updateTransforms;
38
- private _getCameraUvTransform;
39
- private _getFramebuffer;
40
- private _getVertexBuffer;
41
- private _getIndexBuffer;
42
- private _getGreyscaleShader;
43
26
  uploadGL(): void;
44
27
  }
@@ -4,9 +4,9 @@ exports.HTMLElementSource = void 0;
4
4
  const pipeline_1 = require("./pipeline");
5
5
  const source_1 = require("./source");
6
6
  const profile_1 = require("./profile");
7
- const shader_1 = require("./shader");
8
7
  const gl_matrix_1 = require("gl-matrix");
9
8
  const loglevel_1 = require("./loglevel");
9
+ const image_process_gl_1 = require("./image-process-gl");
10
10
  let latest = 1;
11
11
  let byId = new Map();
12
12
  class HTMLElementSource extends source_1.Source {
@@ -19,16 +19,9 @@ class HTMLElementSource extends source_1.Source {
19
19
  this._isUserFacing = false;
20
20
  this._cameraToScreenRotation = 0;
21
21
  this._isUploadFrame = true;
22
- this._computedTransformRotation = -1;
23
- this._computedFrontCameraRotation = false;
24
- this._cameraUvTransform = gl_matrix_1.mat4.create();
25
22
  this._cameraToDeviceTransform = gl_matrix_1.mat4.create();
26
23
  this._cameraToDeviceTransformUserFacing = gl_matrix_1.mat4.create();
27
24
  this._cameraModel = new Float32Array([300, 300, 160, 120, 0, 0]);
28
- this._framebufferWidth = 0;
29
- this._framebufferHeight = 0;
30
- this._framebufferId = null;
31
- this._renderTexture = null;
32
25
  gl_matrix_1.mat4.fromScaling(this._cameraToDeviceTransformUserFacing, [-1, 1, -1]);
33
26
  let video = this._video;
34
27
  if (this._video instanceof HTMLVideoElement) {
@@ -52,12 +45,9 @@ class HTMLElementSource extends source_1.Source {
52
45
  return byId.get(m);
53
46
  }
54
47
  _resetGLContext() {
48
+ var _a, _b;
55
49
  this._currentVideoTexture = undefined;
56
- this._framebufferId = null;
57
- this._renderTexture = null;
58
- this._vertexBuffer = undefined;
59
- this._indexBuffer = undefined;
60
- this._greyscaleShader = undefined;
50
+ (_b = (_a = this._imageProcessor) === null || _a === void 0 ? void 0 : _a.resetGLContext) === null || _b === void 0 ? void 0 : _b.call(_a);
61
51
  }
62
52
  destroy() {
63
53
  let p = pipeline_1.Pipeline.get(this._pipeline);
@@ -98,7 +88,11 @@ class HTMLElementSource extends source_1.Source {
98
88
  if (!this._hadFrames)
99
89
  return;
100
90
  try {
101
- return this._processFrame(gl, this._cameraToScreenRotation, currentlyProcessing);
91
+ let info = this._processFrame(gl, this._cameraToScreenRotation, currentlyProcessing);
92
+ if (info) {
93
+ let token = pipeline.registerToken(info);
94
+ pipeline.sendDataToWorker(info.data || new ArrayBuffer(0), token, info.dataWidth, info.dataHeight, info.userFacing, info.cameraToDevice, info.cameraModel);
95
+ }
102
96
  }
103
97
  catch (ex) {
104
98
  console.log("Unable to process frame");
@@ -109,280 +103,42 @@ class HTMLElementSource extends source_1.Source {
109
103
  let pipeline = pipeline_1.Pipeline.get(this._pipeline);
110
104
  if (!pipeline)
111
105
  return undefined;
106
+ if (!this._imageProcessor)
107
+ this._imageProcessor = new image_process_gl_1.ImageProcessGL(gl);
112
108
  if (this._isUploadFrame) {
113
109
  if (!this._currentVideoTexture) {
114
110
  this._currentVideoTexture = pipeline.getVideoTexture();
115
111
  }
116
- this._uploadFrame(rotation, this._isUserFacing);
112
+ if (!this._currentVideoTexture)
113
+ return undefined;
114
+ this._imageProcessor.uploadFrame(this._currentVideoTexture, this._video, rotation, this._isUserFacing);
117
115
  this._isUploadFrame = !this._isUploadFrame;
118
116
  return undefined;
119
117
  }
120
- if (currentlyProcessing)
118
+ if (currentlyProcessing || !this._currentVideoTexture)
121
119
  return undefined;
122
120
  this._isUploadFrame = !this._isUploadFrame;
123
- return this._readFrame(pipeline, gl);
124
- }
125
- _uploadFrame(rotation, fc) {
126
- if (!this._currentVideoTexture)
127
- return;
128
- let pipeline = pipeline_1.Pipeline.get(this._pipeline);
129
- if (!pipeline)
130
- return;
131
- let gl = pipeline.glContext;
132
- if (!gl)
133
- return;
134
- gl.disable(gl.SCISSOR_TEST);
135
- gl.disable(gl.DEPTH_TEST);
136
- gl.disable(gl.BLEND);
137
- gl.disable(gl.CULL_FACE);
138
- gl.pixelStorei(gl.UNPACK_FLIP_Y_WEBGL, false);
139
- gl.activeTexture(gl.TEXTURE0);
140
- gl.bindTexture(gl.TEXTURE_2D, this._currentVideoTexture);
141
- const level = 0;
142
- const internalFormat = gl.RGBA;
143
- const srcFormat = gl.RGBA;
144
- const srcType = gl.UNSIGNED_BYTE;
145
- gl.texImage2D(gl.TEXTURE_2D, level, internalFormat, srcFormat, srcType, this._video);
146
- let videoWidth = 0;
147
- let videoHeight = 0;
148
- if (this._video instanceof HTMLVideoElement) {
149
- videoWidth = this._video.videoWidth;
150
- videoHeight = this._video.videoHeight;
151
- }
152
- else {
153
- videoWidth = this._video.width;
154
- videoHeight = this._video.height;
155
- }
156
- if (videoHeight > videoWidth)
157
- videoHeight = [videoWidth, videoWidth = videoHeight][0];
158
- this._updateTransforms(rotation, fc);
159
- let framebuffer = this._getFramebuffer(gl, profile_1.profile.dataWidth / 4, profile_1.profile.dataHeight);
160
- let vbo = this._getVertexBuffer(gl);
161
- let ibo = this._getIndexBuffer(gl);
162
- let shader = this._getGreyscaleShader(gl);
163
- // // Rendering to the greyscale conversion buffer - bind the framebuffer
164
- gl.bindFramebuffer(gl.FRAMEBUFFER, framebuffer);
165
- gl.viewport(0, 0, this._framebufferWidth, this._framebufferHeight);
166
- // // We'll be replacing all the content - clear is a good hint for this on mobile
167
- gl.clearColor(0.0, 0.0, 0.0, 1.0);
168
- gl.clear(gl.COLOR_BUFFER_BIT);
169
- // // Set up bindings for vertex attributes
170
- gl.bindBuffer(gl.ARRAY_BUFFER, vbo);
171
- gl.vertexAttribPointer(shader.aVertexPositionLoc, 2, gl.FLOAT, false, 4 * 4, 0);
172
- gl.enableVertexAttribArray(shader.aVertexPositionLoc);
173
- gl.vertexAttribPointer(shader.aTextureCoordLoc, 2, gl.FLOAT, false, 4 * 4, 2 * 4);
174
- gl.enableVertexAttribArray(shader.aTextureCoordLoc);
175
- // Bind the index buffer
176
- gl.bindBuffer(gl.ELEMENT_ARRAY_BUFFER, ibo);
177
- // Tell WebGL to use our program when drawing
178
- gl.useProgram(shader.program);
179
- // Specify greyscale width for the correct offsets, and the uv transform
180
- gl.uniform1f(shader.uTexWidthLoc, profile_1.profile.dataWidth);
181
- gl.uniformMatrix4fv(shader.uUvTransformLoc, false, this._cameraUvTransform);
182
- gl.activeTexture(gl.TEXTURE0);
183
- // Bind the texture to texture unit 0
184
- gl.bindTexture(gl.TEXTURE_2D, this._currentVideoTexture);
185
- // Tell the shader we bound the texture to texture unit 0
186
- gl.uniform1i(shader.uSamplerLoc, 0);
187
- // Do the drawing...
188
- gl.drawElements(gl.TRIANGLES, 6, gl.UNSIGNED_SHORT, 0);
189
- // Disable attribute arrays
190
- gl.disableVertexAttribArray(shader.aVertexPositionLoc);
191
- gl.disableVertexAttribArray(shader.aTextureCoordLoc);
192
- gl.bindFramebuffer(gl.FRAMEBUFFER, null);
193
- gl.bindBuffer(gl.ARRAY_BUFFER, null);
194
- gl.bindBuffer(gl.ELEMENT_ARRAY_BUFFER, null);
195
- gl.useProgram(null);
196
- gl.bindTexture(gl.TEXTURE_2D, null);
197
- }
198
- _readFrame(p, gl) {
199
- if (!this._currentVideoTexture)
200
- throw new Error("No video texture");
201
- let tex = this._currentVideoTexture;
202
- this._currentVideoTexture = undefined;
203
121
  let greySize = profile_1.profile.dataWidth * profile_1.profile.dataHeight;
204
- let pixels = p.cameraPixelArrays.pop();
122
+ let pixels = pipeline.cameraPixelArrays.pop();
205
123
  while (pixels) {
206
124
  if (pixels.byteLength === greySize)
207
125
  break;
208
- pixels = p.cameraPixelArrays.pop();
126
+ pixels = pipeline.cameraPixelArrays.pop();
209
127
  }
210
128
  if (!pixels) {
211
129
  pixels = new ArrayBuffer(greySize);
212
130
  }
213
- let pixelsView = new Uint8Array(pixels);
214
- let framebuffer = this._getFramebuffer(gl, profile_1.profile.dataWidth / 4, profile_1.profile.dataHeight);
215
- gl.bindFramebuffer(gl.FRAMEBUFFER, framebuffer);
216
- gl.readPixels(0, 0, this._framebufferWidth, this._framebufferHeight, gl.RGBA, gl.UNSIGNED_BYTE, pixelsView);
217
- gl.bindFramebuffer(gl.FRAMEBUFFER, null);
131
+ let tex = this._currentVideoTexture;
132
+ this._currentVideoTexture = undefined;
218
133
  let focalLength = 300.0 * profile_1.profile.dataWidth / 320.0;
219
134
  this._cameraModel[0] = focalLength;
220
135
  this._cameraModel[1] = focalLength;
221
136
  this._cameraModel[2] = profile_1.profile.dataWidth * 0.5;
222
137
  this._cameraModel[3] = profile_1.profile.dataHeight * 0.5;
223
- return {
224
- uvTransform: this._cameraUvTransform,
225
- data: pixels,
226
- texture: tex,
227
- cameraModel: this._cameraModel,
228
- dataWidth: profile_1.profile.dataWidth,
229
- dataHeight: profile_1.profile.dataHeight,
230
- userFacing: this._computedFrontCameraRotation,
231
- cameraToDevice: this._computedFrontCameraRotation ? this._cameraToDeviceTransformUserFacing : this._cameraToDeviceTransform,
232
- cameraSource: this
233
- };
234
- }
235
- _updateTransforms(rot, fc) {
236
- if (rot == this._computedTransformRotation && fc == this._computedFrontCameraRotation)
237
- return;
238
- this._computedTransformRotation = rot;
239
- this._computedFrontCameraRotation = fc;
240
- this._cameraUvTransform = this._getCameraUvTransform();
241
- }
242
- _getCameraUvTransform() {
243
- switch (this._computedTransformRotation) {
244
- case 270: return new Float32Array([0, 1, 0, 0, -1, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 1]);
245
- case 180: return new Float32Array([-1, 0, 0, 0, 0, -1, 0, 0, 0, 0, 1, 0, 1, 1, 0, 1]);
246
- case 90: return new Float32Array([0, -1, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 1]);
247
- }
248
- return new Float32Array([1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1]);
249
- }
250
- _getFramebuffer(gl, fbWidth, fbHeight) {
251
- if (this._framebufferWidth === fbWidth && this._framebufferHeight === fbHeight && this._framebufferId)
252
- return this._framebufferId;
253
- if (this._framebufferId) {
254
- gl.deleteFramebuffer(this._framebufferId);
255
- this._framebufferId = null;
256
- }
257
- if (this._renderTexture) {
258
- gl.deleteTexture(this._renderTexture);
259
- this._renderTexture = null;
260
- }
261
- this._framebufferId = gl.createFramebuffer();
262
- if (!this._framebufferId)
263
- throw new Error("Unable to create framebuffer");
264
- gl.bindFramebuffer(gl.FRAMEBUFFER, this._framebufferId);
265
- this._renderTexture = gl.createTexture();
266
- if (!this._renderTexture)
267
- throw new Error("Unable to create render texture");
268
- gl.activeTexture(gl.TEXTURE0);
269
- gl.bindTexture(gl.TEXTURE_2D, this._renderTexture);
270
- gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, fbWidth, fbHeight, 0, gl.RGBA, gl.UNSIGNED_BYTE, null);
271
- gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE);
272
- gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE);
273
- gl.texParameterf(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR);
274
- gl.framebufferTexture2D(gl.FRAMEBUFFER, gl.COLOR_ATTACHMENT0, gl.TEXTURE_2D, this._renderTexture, 0);
275
- let fbStatus = gl.checkFramebufferStatus(gl.FRAMEBUFFER);
276
- if (fbStatus !== gl.FRAMEBUFFER_COMPLETE)
277
- throw new Error("Framebuffer not complete: " + fbStatus.toString());
278
- this._framebufferWidth = fbWidth;
279
- this._framebufferHeight = fbHeight;
280
- gl.bindTexture(gl.TEXTURE_2D, null);
281
- gl.bindFramebuffer(gl.FRAMEBUFFER, null);
282
- return this._framebufferId;
283
- }
284
- _getVertexBuffer(gl) {
285
- if (this._vertexBuffer)
286
- return this._vertexBuffer;
287
- this._vertexBuffer = gl.createBuffer();
288
- if (!this._vertexBuffer)
289
- throw new Error("Unable to create vertex buffer");
290
- gl.bindBuffer(gl.ARRAY_BUFFER, this._vertexBuffer);
291
- let buffer = new Float32Array([-1.0, -1.0, 0.0, 0.0,
292
- -1.0, 1.0, 0.0, 1.0,
293
- 1.0, 1.0, 1.0, 1.0,
294
- 1.0, -1.0, 1.0, 0.0]);
295
- gl.bufferData(gl.ARRAY_BUFFER, buffer, gl.STATIC_DRAW);
296
- return this._vertexBuffer;
297
- }
298
- _getIndexBuffer(gl) {
299
- if (this._indexBuffer)
300
- return this._indexBuffer;
301
- this._indexBuffer = gl.createBuffer();
302
- if (!this._indexBuffer)
303
- throw new Error("Unable to create index buffer");
304
- gl.bindBuffer(gl.ELEMENT_ARRAY_BUFFER, this._indexBuffer);
305
- let buffer = new Uint16Array([0, 1, 2, 0, 2, 3]);
306
- gl.bufferData(gl.ELEMENT_ARRAY_BUFFER, buffer, gl.STATIC_DRAW);
307
- return this._indexBuffer;
308
- }
309
- _getGreyscaleShader(gl) {
310
- if (this._greyscaleShader)
311
- return this._greyscaleShader;
312
- let prog = gl.createProgram();
313
- if (!prog)
314
- throw new Error("Unable to create program");
315
- let vertexShader = shader_1.compileShader(gl, gl.VERTEX_SHADER, greyscaleVsSource);
316
- let fragmentShader = shader_1.compileShader(gl, gl.FRAGMENT_SHADER, greyscaleFsSource);
317
- gl.attachShader(prog, vertexShader);
318
- gl.attachShader(prog, fragmentShader);
319
- shader_1.linkProgram(gl, prog);
320
- let uTexWidthLoc = gl.getUniformLocation(prog, "uTexWidth");
321
- if (!uTexWidthLoc)
322
- throw new Error("Unable to get uniform location uTexWidth");
323
- let uUvTransformLoc = gl.getUniformLocation(prog, "uUvTransform");
324
- if (!uUvTransformLoc)
325
- throw new Error("Unable to get uniform location uUvTransform");
326
- let uSamplerLoc = gl.getUniformLocation(prog, "uSampler");
327
- if (!uSamplerLoc)
328
- throw new Error("Unable to get uniform location uSampler");
329
- this._greyscaleShader = {
330
- program: prog,
331
- aVertexPositionLoc: gl.getAttribLocation(prog, "aVertexPosition"),
332
- aTextureCoordLoc: gl.getAttribLocation(prog, "aTextureCoord"),
333
- uTexWidthLoc: uTexWidthLoc,
334
- uUvTransformLoc: uUvTransformLoc,
335
- uSamplerLoc: uSamplerLoc
336
- };
337
- return this._greyscaleShader;
138
+ return Object.assign(Object.assign({}, this._imageProcessor.readFrame(tex, pixels)), { cameraModel: this._cameraModel, cameraSource: this, cameraToDevice: this._isUserFacing ? this._cameraToDeviceTransformUserFacing : this._cameraToDeviceTransform });
338
139
  }
339
140
  uploadGL() {
340
141
  // No-op as already uploaded
341
142
  }
342
143
  }
343
144
  exports.HTMLElementSource = HTMLElementSource;
344
- let greyscaleVsSource = `
345
- attribute vec4 aVertexPosition;
346
- attribute vec2 aTextureCoord;
347
-
348
- varying highp vec2 vTextureCoord1;
349
- varying highp vec2 vTextureCoord2;
350
- varying highp vec2 vTextureCoord3;
351
- varying highp vec2 vTextureCoord4;
352
-
353
- uniform float uTexWidth;
354
- uniform mat4 uUvTransform;
355
-
356
- void main(void) {
357
- highp vec2 offset1 = vec2(1.5 / uTexWidth, 0);
358
- highp vec2 offset2 = vec2(0.5 / uTexWidth, 0);
359
-
360
- gl_Position = aVertexPosition;
361
- vTextureCoord1 = (uUvTransform * vec4(aTextureCoord - offset1, 0, 1)).xy;
362
- vTextureCoord2 = (uUvTransform * vec4(aTextureCoord - offset2, 0, 1)).xy;
363
- vTextureCoord3 = (uUvTransform * vec4(aTextureCoord + offset2, 0, 1)).xy;
364
- vTextureCoord4 = (uUvTransform * vec4(aTextureCoord + offset1, 0, 1)).xy;
365
- }
366
- `;
367
- // Fragment shader program
368
- let greyscaleFsSource = `
369
- varying highp vec2 vTextureCoord1;
370
- varying highp vec2 vTextureCoord2;
371
- varying highp vec2 vTextureCoord3;
372
- varying highp vec2 vTextureCoord4;
373
-
374
- uniform sampler2D uSampler;
375
-
376
- const lowp vec3 colorWeights = vec3(77.0 / 256.0, 150.0 / 256.0, 29.0 / 256.0);
377
-
378
- void main(void) {
379
- lowp vec4 outpx;
380
-
381
- outpx.r = dot(colorWeights, texture2D(uSampler, vTextureCoord1).xyz);
382
- outpx.g = dot(colorWeights, texture2D(uSampler, vTextureCoord2).xyz);
383
- outpx.b = dot(colorWeights, texture2D(uSampler, vTextureCoord3).xyz);
384
- outpx.a = dot(colorWeights, texture2D(uSampler, vTextureCoord4).xyz);
385
-
386
- gl_FragColor = outpx;
387
- }
388
- `;
@@ -0,0 +1,37 @@
1
+ export interface ProcessedFrame {
2
+ data?: ArrayBuffer;
3
+ texture: WebGLTexture | undefined;
4
+ dataWidth: number;
5
+ dataHeight: number;
6
+ uvTransform?: Float32Array;
7
+ userFacing: boolean;
8
+ }
9
+ export declare class ImageProcessGL {
10
+ private _gl;
11
+ protected _isPaused: boolean;
12
+ protected _hadFrames: boolean;
13
+ protected _isUserFacing: boolean;
14
+ protected _cameraToScreenRotation: number;
15
+ private _isUploadFrame;
16
+ private _computedTransformRotation;
17
+ private _computedFrontCameraRotation;
18
+ private _cameraUvTransform;
19
+ private _framebufferWidth;
20
+ private _framebufferHeight;
21
+ private _framebufferId;
22
+ private _renderTexture;
23
+ private _vertexBuffer;
24
+ private _indexBuffer;
25
+ private _greyscaleShader;
26
+ constructor(_gl: WebGLRenderingContext);
27
+ resetGLContext(): void;
28
+ destroy(): void;
29
+ uploadFrame(texture: WebGLTexture, img: HTMLVideoElement | HTMLImageElement | ImageBitmap, rotation: number, fc: boolean): void;
30
+ readFrame(texture: WebGLTexture, pixels: ArrayBuffer): ProcessedFrame;
31
+ private _updateTransforms;
32
+ private _getCameraUvTransform;
33
+ private _getFramebuffer;
34
+ private _getVertexBuffer;
35
+ private _getIndexBuffer;
36
+ private _getGreyscaleShader;
37
+ }