@zappar/zappar-cv 0.3.13 → 0.4.0-beta.10

package/lib/html-element-source.js

@@ -4,9 +4,9 @@ exports.HTMLElementSource = void 0;
 const pipeline_1 = require("./pipeline");
 const source_1 = require("./source");
 const profile_1 = require("./profile");
-const shader_1 = require("./shader");
 const gl_matrix_1 = require("gl-matrix");
 const loglevel_1 = require("./loglevel");
+const image_process_gl_1 = require("./image-process-gl");
 let latest = 1;
 let byId = new Map();
 class HTMLElementSource extends source_1.Source {
@@ -19,14 +19,10 @@ class HTMLElementSource extends source_1.Source {
         this._isUserFacing = false;
         this._cameraToScreenRotation = 0;
         this._isUploadFrame = true;
-        this._computedTransformRotation = -1;
-        this._computedFrontCameraRotation = false;
-        this._cameraUvTransform = gl_matrix_1.mat4.create();
-        this._cameraVertexTransform = [1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1];
-        this._framebufferWidth = 0;
-        this._framebufferHeight = 0;
-        this._framebufferId = null;
-        this._renderTexture = null;
+        this._cameraToDeviceTransform = gl_matrix_1.mat4.create();
+        this._cameraToDeviceTransformUserFacing = gl_matrix_1.mat4.create();
+        this._cameraModel = new Float32Array([300, 300, 160, 120, 0, 0]);
+        gl_matrix_1.mat4.fromScaling(this._cameraToDeviceTransformUserFacing, [-1, 1, -1]);
         let video = this._video;
         if (this._video instanceof HTMLVideoElement) {
             video.addEventListener("loadedmetadata", () => { this._hadFrames = true; });
@@ -49,12 +45,9 @@ class HTMLElementSource extends source_1.Source {
         return byId.get(m);
     }
     _resetGLContext() {
+        var _a, _b;
         this._currentVideoTexture = undefined;
-        this._framebufferId = null;
-        this._renderTexture = null;
-        this._vertexBuffer = undefined;
-        this._indexBuffer = undefined;
-        this._greyscaleShader = undefined;
+        (_b = (_a = this._imageProcessor) === null || _a === void 0 ? void 0 : _a.resetGLContext) === null || _b === void 0 ? void 0 : _b.call(_a);
     }
     destroy() {
         let p = pipeline_1.Pipeline.get(this._pipeline);
@@ -95,7 +88,11 @@ class HTMLElementSource extends source_1.Source {
         if (!this._hadFrames)
             return;
         try {
-            return this._processFrame(gl, this._cameraToScreenRotation, currentlyProcessing);
+            let info = this._processFrame(gl, this._cameraToScreenRotation, currentlyProcessing);
+            if (info) {
+                let token = pipeline.registerToken(info);
+                pipeline.sendDataToWorker(info.data || new ArrayBuffer(0), token, info.dataWidth, info.dataHeight, info.userFacing, info.cameraToDevice, info.cameraModel);
+            }
         }
         catch (ex) {
             console.log("Unable to process frame");
@@ -106,298 +103,42 @@ class HTMLElementSource extends source_1.Source {
         let pipeline = pipeline_1.Pipeline.get(this._pipeline);
         if (!pipeline)
             return undefined;
+        if (!this._imageProcessor)
+            this._imageProcessor = new image_process_gl_1.ImageProcessGL(gl);
         if (this._isUploadFrame) {
             if (!this._currentVideoTexture) {
                 this._currentVideoTexture = pipeline.getVideoTexture();
             }
-            this._uploadFrame(rotation, this._isUserFacing);
+            if (!this._currentVideoTexture)
+                return undefined;
+            this._imageProcessor.uploadFrame(this._currentVideoTexture, this._video, rotation, this._isUserFacing);
             this._isUploadFrame = !this._isUploadFrame;
             return undefined;
         }
-        if (currentlyProcessing)
+        if (currentlyProcessing || !this._currentVideoTexture)
             return undefined;
         this._isUploadFrame = !this._isUploadFrame;
-        return this._readFrame(pipeline, gl);
-    }
-    _uploadFrame(rotation, fc) {
-        if (!this._currentVideoTexture)
-            return;
-        let pipeline = pipeline_1.Pipeline.get(this._pipeline);
-        if (!pipeline)
-            return;
-        let gl = pipeline.glContext;
-        if (!gl)
-            return;
-        gl.disable(gl.SCISSOR_TEST);
-        gl.disable(gl.DEPTH_TEST);
-        gl.disable(gl.BLEND);
-        gl.disable(gl.CULL_FACE);
-        gl.pixelStorei(gl.UNPACK_FLIP_Y_WEBGL, false);
-        gl.activeTexture(gl.TEXTURE0);
-        gl.bindTexture(gl.TEXTURE_2D, this._currentVideoTexture);
-        const level = 0;
-        const internalFormat = gl.RGBA;
-        const srcFormat = gl.RGBA;
-        const srcType = gl.UNSIGNED_BYTE;
-        gl.texImage2D(gl.TEXTURE_2D, level, internalFormat, srcFormat, srcType, this._video);
-        let videoWidth = 0;
-        let videoHeight = 0;
-        if (this._video instanceof HTMLVideoElement) {
-            videoWidth = this._video.videoWidth;
-            videoHeight = this._video.videoHeight;
-        }
-        else {
-            videoWidth = this._video.width;
-            videoHeight = this._video.height;
-        }
-        if (videoHeight > videoWidth)
-            videoHeight = [videoWidth, videoWidth = videoHeight][0];
-        this._updateTransforms(rotation, fc);
-        let framebuffer = this._getFramebuffer(gl, profile_1.profile.dataWidth / 4, profile_1.profile.dataHeight);
-        let vbo = this._getVertexBuffer(gl);
-        let ibo = this._getIndexBuffer(gl);
-        let shader = this._getGreyscaleShader(gl);
-        //     // Rendering to the greyscale conversion buffer - bind the framebuffer
-        gl.bindFramebuffer(gl.FRAMEBUFFER, framebuffer);
-        gl.viewport(0, 0, this._framebufferWidth, this._framebufferHeight);
-        //     // We'll be replacing all the content - clear is a good hint for this on mobile
-        gl.clearColor(0.0, 0.0, 0.0, 1.0);
-        gl.clear(gl.COLOR_BUFFER_BIT);
-        //     // Set up bindings for vertex attributes
-        gl.bindBuffer(gl.ARRAY_BUFFER, vbo);
-        gl.vertexAttribPointer(shader.aVertexPositionLoc, 2, gl.FLOAT, false, 4 * 4, 0);
-        gl.enableVertexAttribArray(shader.aVertexPositionLoc);
-        gl.vertexAttribPointer(shader.aTextureCoordLoc, 2, gl.FLOAT, false, 4 * 4, 2 * 4);
-        gl.enableVertexAttribArray(shader.aTextureCoordLoc);
-        // Bind the index buffer
-        gl.bindBuffer(gl.ELEMENT_ARRAY_BUFFER, ibo);
-        // Tell WebGL to use our program when drawing
-        gl.useProgram(shader.program);
-        // Specify greyscale width for the correct offsets, and the uv transform
-        gl.uniform1f(shader.uTexWidthLoc, profile_1.profile.dataWidth);
-        gl.uniformMatrix4fv(shader.uUvTransformLoc, false, this._cameraUvTransform);
-        gl.activeTexture(gl.TEXTURE0);
-        // Bind the texture to texture unit 0
-        gl.bindTexture(gl.TEXTURE_2D, this._currentVideoTexture);
-        // Tell the shader we bound the texture to texture unit 0
-        gl.uniform1i(shader.uSamplerLoc, 0);
-        // Do the drawing...
-        gl.drawElements(gl.TRIANGLES, 6, gl.UNSIGNED_SHORT, 0);
-        // Disable attribute arrays
-        gl.disableVertexAttribArray(shader.aVertexPositionLoc);
-        gl.disableVertexAttribArray(shader.aTextureCoordLoc);
-        gl.bindFramebuffer(gl.FRAMEBUFFER, null);
-        gl.bindBuffer(gl.ARRAY_BUFFER, null);
-        gl.bindBuffer(gl.ELEMENT_ARRAY_BUFFER, null);
-        gl.useProgram(null);
-        gl.bindTexture(gl.TEXTURE_2D, null);
-    }
-    _readFrame(p, gl) {
-        if (!this._currentVideoTexture)
-            throw new Error("No video texture");
-        let tex = this._currentVideoTexture;
-        this._currentVideoTexture = undefined;
         let greySize = profile_1.profile.dataWidth * profile_1.profile.dataHeight;
-        let pixels = p.cameraPixelArrays.pop();
+        let pixels = pipeline.cameraPixelArrays.pop();
         while (pixels) {
             if (pixels.byteLength === greySize)
                 break;
-            pixels = p.cameraPixelArrays.pop();
+            pixels = pipeline.cameraPixelArrays.pop();
         }
         if (!pixels) {
             pixels = new ArrayBuffer(greySize);
         }
-        let pixelsView = new Uint8Array(pixels);
-        let framebuffer = this._getFramebuffer(gl, profile_1.profile.dataWidth / 4, profile_1.profile.dataHeight);
-        gl.bindFramebuffer(gl.FRAMEBUFFER, framebuffer);
-        gl.readPixels(0, 0, this._framebufferWidth, this._framebufferHeight, gl.RGBA, gl.UNSIGNED_BYTE, pixelsView);
-        gl.bindFramebuffer(gl.FRAMEBUFFER, null);
-        return {
-            uvTransform: this._cameraUvTransform,
-            data: pixels,
-            texture: tex,
-            dataWidth: profile_1.profile.dataWidth,
-            dataHeight: profile_1.profile.dataHeight,
-            userFacing: this._computedFrontCameraRotation
-        };
-    }
-    _updateTransforms(rot, fc) {
-        if (rot == this._computedTransformRotation && fc == this._computedFrontCameraRotation)
-            return;
-        this._computedTransformRotation = rot;
-        this._computedFrontCameraRotation = fc;
-        this._cameraUvTransform = this._getCameraUvTransform();
-        this._cameraVertexTransform = this._getCameraVertexTransform();
-    }
-    _getCameraUvTransform() {
-        switch (this._computedTransformRotation) {
-            case 270: return new Float32Array([0, 1, 0, 0, -1, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 1]);
-            case 180: return new Float32Array([-1, 0, 0, 0, 0, -1, 0, 0, 0, 0, 1, 0, 1, 1, 0, 1]);
-            case 90: return new Float32Array([0, -1, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 1]);
-        }
-        return new Float32Array([1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1]);
-    }
-    _getCameraVertexTransform() {
-        let identity = [1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1];
-        if (!this._computedFrontCameraRotation)
-            return identity;
-        // It's a little odd there's only one special case here.
-        // This is because the camera rotation should really depend on
-        // isFrontCamera as well; we should aim to rotate the camera to
-        // process it in sensor-native orientation and then rotate and
-        // flip for rendering based on the screen orientation.
-        // For now I've kept with the rotation values calculated for the
-        // rear camera and correct the rendering of the front camera here
-        // for all rotations (tested on iPad which allows inverse
-        // portrait).
-        // TODO: Figure this out correctly. Probably need to do this to
-        // correctly use accelerometer / gyro data alongside vision data
-        // regardless of orientation
-        switch (this._computedTransformRotation) {
-            case 0:
-            case 90:
-            case 180:
-                identity[0] = -1;
-                break;
-            case 270:
-                identity[5] = -1;
-                break;
-        }
-        return identity;
-    }
-    _getFramebuffer(gl, fbWidth, fbHeight) {
-        if (this._framebufferWidth === fbWidth && this._framebufferHeight === fbHeight && this._framebufferId)
-            return this._framebufferId;
-        if (this._framebufferId) {
-            gl.deleteFramebuffer(this._framebufferId);
-            this._framebufferId = null;
-        }
-        if (this._renderTexture) {
-            gl.deleteTexture(this._renderTexture);
-            this._renderTexture = null;
-        }
-        this._framebufferId = gl.createFramebuffer();
-        if (!this._framebufferId)
-            throw new Error("Unable to create framebuffer");
-        gl.bindFramebuffer(gl.FRAMEBUFFER, this._framebufferId);
-        this._renderTexture = gl.createTexture();
-        if (!this._renderTexture)
-            throw new Error("Unable to create render texture");
-        gl.activeTexture(gl.TEXTURE0);
-        gl.bindTexture(gl.TEXTURE_2D, this._renderTexture);
-        gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, fbWidth, fbHeight, 0, gl.RGBA, gl.UNSIGNED_BYTE, null);
-        gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE);
-        gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE);
-        gl.texParameterf(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR);
-        gl.framebufferTexture2D(gl.FRAMEBUFFER, gl.COLOR_ATTACHMENT0, gl.TEXTURE_2D, this._renderTexture, 0);
-        let fbStatus = gl.checkFramebufferStatus(gl.FRAMEBUFFER);
-        if (fbStatus !== gl.FRAMEBUFFER_COMPLETE)
-            throw new Error("Framebuffer not complete: " + fbStatus.toString());
-        this._framebufferWidth = fbWidth;
-        this._framebufferHeight = fbHeight;
-        gl.bindTexture(gl.TEXTURE_2D, null);
-        gl.bindFramebuffer(gl.FRAMEBUFFER, null);
-        return this._framebufferId;
-    }
-    _getVertexBuffer(gl) {
-        if (this._vertexBuffer)
-            return this._vertexBuffer;
-        this._vertexBuffer = gl.createBuffer();
-        if (!this._vertexBuffer)
-            throw new Error("Unable to create vertex buffer");
-        gl.bindBuffer(gl.ARRAY_BUFFER, this._vertexBuffer);
-        let buffer = new Float32Array([-1.0, -1.0, 0.0, 0.0,
-            -1.0, 1.0, 0.0, 1.0,
-            1.0, 1.0, 1.0, 1.0,
-            1.0, -1.0, 1.0, 0.0]);
-        gl.bufferData(gl.ARRAY_BUFFER, buffer, gl.STATIC_DRAW);
-        return this._vertexBuffer;
-    }
-    _getIndexBuffer(gl) {
-        if (this._indexBuffer)
-            return this._indexBuffer;
-        this._indexBuffer = gl.createBuffer();
-        if (!this._indexBuffer)
-            throw new Error("Unable to create index buffer");
-        gl.bindBuffer(gl.ELEMENT_ARRAY_BUFFER, this._indexBuffer);
-        let buffer = new Uint16Array([0, 1, 2, 0, 2, 3]);
-        gl.bufferData(gl.ELEMENT_ARRAY_BUFFER, buffer, gl.STATIC_DRAW);
-        return this._indexBuffer;
+        let tex = this._currentVideoTexture;
+        this._currentVideoTexture = undefined;
+        let focalLength = 300.0 * profile_1.profile.dataWidth / 320.0;
+        this._cameraModel[0] = focalLength;
+        this._cameraModel[1] = focalLength;
+        this._cameraModel[2] = profile_1.profile.dataWidth * 0.5;
+        this._cameraModel[3] = profile_1.profile.dataHeight * 0.5;
+        return Object.assign(Object.assign({}, this._imageProcessor.readFrame(tex, pixels)), { cameraModel: this._cameraModel, cameraSource: this, cameraToDevice: this._isUserFacing ? this._cameraToDeviceTransformUserFacing : this._cameraToDeviceTransform });
     }
-    _getGreyscaleShader(gl) {
-        if (this._greyscaleShader)
-            return this._greyscaleShader;
-        let prog = gl.createProgram();
-        if (!prog)
-            throw new Error("Unable to create program");
-        let vertexShader = shader_1.compileShader(gl, gl.VERTEX_SHADER, greyscaleVsSource);
-        let fragmentShader = shader_1.compileShader(gl, gl.FRAGMENT_SHADER, greyscaleFsSource);
-        gl.attachShader(prog, vertexShader);
-        gl.attachShader(prog, fragmentShader);
-        shader_1.linkProgram(gl, prog);
-        let uTexWidthLoc = gl.getUniformLocation(prog, "uTexWidth");
-        if (!uTexWidthLoc)
-            throw new Error("Unable to get uniform location uTexWidth");
-        let uUvTransformLoc = gl.getUniformLocation(prog, "uUvTransform");
-        if (!uUvTransformLoc)
-            throw new Error("Unable to get uniform location uUvTransform");
-        let uSamplerLoc = gl.getUniformLocation(prog, "uSampler");
-        if (!uSamplerLoc)
-            throw new Error("Unable to get uniform location uSampler");
-        this._greyscaleShader = {
-            program: prog,
-            aVertexPositionLoc: gl.getAttribLocation(prog, "aVertexPosition"),
-            aTextureCoordLoc: gl.getAttribLocation(prog, "aTextureCoord"),
-            uTexWidthLoc: uTexWidthLoc,
-            uUvTransformLoc: uUvTransformLoc,
-            uSamplerLoc: uSamplerLoc
-        };
-        return this._greyscaleShader;
+    uploadGL() {
+        // No-op as already uploaded
     }
 }
 exports.HTMLElementSource = HTMLElementSource;
-let greyscaleVsSource = `
-    attribute vec4 aVertexPosition;
-    attribute vec2 aTextureCoord;
-
-    varying highp vec2 vTextureCoord1;
-    varying highp vec2 vTextureCoord2;
-    varying highp vec2 vTextureCoord3;
-    varying highp vec2 vTextureCoord4;
-
-    uniform float uTexWidth;
-	uniform mat4 uUvTransform;
-
-    void main(void) {
-      highp vec2 offset1 = vec2(1.5 / uTexWidth, 0);
-      highp vec2 offset2 = vec2(0.5 / uTexWidth, 0);
-
-      gl_Position = aVertexPosition;
-      vTextureCoord1 = (uUvTransform * vec4(aTextureCoord - offset1, 0, 1)).xy;
-      vTextureCoord2 = (uUvTransform * vec4(aTextureCoord - offset2, 0, 1)).xy;
-      vTextureCoord3 = (uUvTransform * vec4(aTextureCoord + offset2, 0, 1)).xy;
-      vTextureCoord4 = (uUvTransform * vec4(aTextureCoord + offset1, 0, 1)).xy;
-    }
-`;
-// Fragment shader program
-let greyscaleFsSource = `
-  varying highp vec2 vTextureCoord1;
-  varying highp vec2 vTextureCoord2;
-  varying highp vec2 vTextureCoord3;
-  varying highp vec2 vTextureCoord4;
-
-  uniform sampler2D uSampler;
-
-  const lowp vec3 colorWeights = vec3(77.0 / 256.0, 150.0 / 256.0, 29.0 / 256.0);
-
-  void main(void) {
-    lowp vec4 outpx;
-
-    outpx.r = dot(colorWeights, texture2D(uSampler, vTextureCoord1).xyz);
-    outpx.g = dot(colorWeights, texture2D(uSampler, vTextureCoord2).xyz);
-    outpx.b = dot(colorWeights, texture2D(uSampler, vTextureCoord3).xyz);
-    outpx.a = dot(colorWeights, texture2D(uSampler, vTextureCoord4).xyz);
-
-    gl_FragColor = outpx;
-  }
-`;

package/lib/image-process-gl.d.ts

@@ -0,0 +1,37 @@
+export interface ProcessedFrame {
+    data?: ArrayBuffer;
+    texture: WebGLTexture | undefined;
+    dataWidth: number;
+    dataHeight: number;
+    uvTransform?: Float32Array;
+    userFacing: boolean;
+}
+export declare class ImageProcessGL {
+    private _gl;
+    protected _isPaused: boolean;
+    protected _hadFrames: boolean;
+    protected _isUserFacing: boolean;
+    protected _cameraToScreenRotation: number;
+    private _isUploadFrame;
+    private _computedTransformRotation;
+    private _computedFrontCameraRotation;
+    private _cameraUvTransform;
+    private _framebufferWidth;
+    private _framebufferHeight;
+    private _framebufferId;
+    private _renderTexture;
+    private _vertexBuffer;
+    private _indexBuffer;
+    private _greyscaleShader;
+    constructor(_gl: WebGLRenderingContext);
+    resetGLContext(): void;
+    destroy(): void;
+    uploadFrame(texture: WebGLTexture, img: HTMLVideoElement | HTMLImageElement | ImageBitmap, rotation: number, fc: boolean): void;
+    readFrame(texture: WebGLTexture, pixels: ArrayBuffer): ProcessedFrame;
+    private _updateTransforms;
+    private _getCameraUvTransform;
+    private _getFramebuffer;
+    private _getVertexBuffer;
+    private _getIndexBuffer;
+    private _getGreyscaleShader;
+}