@zappar/zappar-cv 3.0.1-beta.8 → 3.1.0-beta.2

package/lib/yuv-conversion-gl.js

@@ -0,0 +1,416 @@
+import { GLStateManager } from "./gl-state-manager";
+import { compileShader, linkProgram } from "./shader";
+import { zcout } from "./loglevel";
+import { getSharedTracer } from "./tracing/sharedtracer";
+// Some no-op tracing helpers for concise code and good dead code removal in bundlers
+let tracer = null;
+let traceStart = (name) => { };
+let traceEnd = () => { };
+if (!(typeof Z_TRACING)) {
+    tracer = getSharedTracer();
+    traceStart = (name) => { tracer.start(name); };
+    traceEnd = () => { tracer.end(); };
+}
+export var UvLayout;
+(function (UvLayout) {
+    UvLayout[UvLayout["INTERLEAVED_UV"] = 0] = "INTERLEAVED_UV";
+    UvLayout[UvLayout["INTERLEAVED_VU"] = 1] = "INTERLEAVED_VU";
+    UvLayout[UvLayout["PLANAR_UV"] = 2] = "PLANAR_UV";
+})(UvLayout || (UvLayout = {}));
+export class YUVConversionGL {
+    constructor(_gl) {
+        this._gl = _gl;
+        this._yTexture = null;
+        this._uvTexture = null;
+        this._uvTextureSinglePlane = false;
+        this._vTexture = null;
+        this._framebufferId = null;
+        this._vao = null;
+        this._conversionShaders = [];
+        this._isWebGL2 = false;
+        this._useVao = false;
+        this._isWebGL2 = _gl.getParameter(_gl.VERSION).indexOf("WebGL 2") >= 0;
+        this._useVao = true;
+        if (!this._isWebGL2) {
+            this._vaoExtension = this._gl.getExtension("OES_vertex_array_object");
+            if (this._vaoExtension == null) {
+                // We'll have to query and reset the vertex attribute state manually :(
+                this._useVao = false;
+                this._instancedArraysExtension = this._gl.getExtension("ANGLE_instanced_arrays");
+            }
+        }
+        const gl2 = _gl; // Must check _isWebGL2 before use
+        this._oneComponentInternalFormat = this._isWebGL2 ? gl2.R8 : _gl.ALPHA;
+        this._oneComponentFormat = this._isWebGL2 ? gl2.RED : _gl.ALPHA;
+        this._twoComponentInternalFormat = this._isWebGL2 ? gl2.RG8 : _gl.LUMINANCE_ALPHA;
+        this._twoComponentFormat = this._isWebGL2 ? gl2.RG : _gl.LUMINANCE_ALPHA;
+    }
+    resetGLContext() {
+        this._yTexture = null;
+        this._uvTexture = null;
+        this._vTexture = null;
+        this._framebufferId = null;
+        this._vao = null;
+        this._vertexBuffer = undefined;
+        this._conversionShaders = [];
+    }
+    destroy() {
+        this.resetGLContext();
+    }
+    yuvToTexture(y, uv, uvLayout, destination) {
+        const gl = this._gl;
+        const gl2 = gl; // Must check _isWebGL2 before use
+        const glStateManager = GLStateManager.get(gl);
+        glStateManager.push();
+        const reenableScissorTest = gl.isEnabled(gl.SCISSOR_TEST);
+        const reenableDepthTest = gl.isEnabled(gl.DEPTH_TEST);
+        const reenableBlend = gl.isEnabled(gl.BLEND);
+        const reenableCullFace = gl.isEnabled(gl.CULL_FACE);
+        const reenableStencilTest = gl.isEnabled(gl.STENCIL_TEST);
+        const previousUnpackFlip = gl.getParameter(gl.UNPACK_FLIP_Y_WEBGL);
+        const previousActiveTexture = gl.getParameter(gl.ACTIVE_TEXTURE);
+        const previousProgram = gl.getParameter(gl.CURRENT_PROGRAM);
+        const previousBoundFramebuffer = gl.getParameter(gl.FRAMEBUFFER_BINDING);
+        const previousBoundArrayBuffer = gl.getParameter(gl.ARRAY_BUFFER_BINDING);
+        let previousVAO = null;
+        if (this._isWebGL2) {
+            previousVAO = gl2.getParameter(gl2.VERTEX_ARRAY_BINDING);
+        }
+        else if (this._vaoExtension) {
+            previousVAO = gl.getParameter(this._vaoExtension.VERTEX_ARRAY_BINDING_OES);
+        }
+        gl.activeTexture(gl.TEXTURE0);
+        const previousBoundTexture0 = gl.getParameter(gl.TEXTURE_BINDING_2D);
+        // We've stored all the GL state we will need to restore, so this is the
+        // first moment we can initialize our WebGL objects safely
+        if (!this._framebufferId)
+            this._prepareContext();
+        let isPlanarUv = (uvLayout === UvLayout.PLANAR_UV && !!uv.dataVPlane);
+        this._prepareTextures(y, uv, isPlanarUv);
+        // Get the appropriate shader for the uv layout
+        let conversionShader = this._conversionShaders[uvLayout];
+        if (!conversionShader) {
+            traceStart("YUV: Prepare conversion shader");
+            this._prepareConversionShader(uvLayout);
+            conversionShader = this._conversionShaders[uvLayout];
+            traceEnd();
+        }
+        if (!conversionShader || !this._vertexBuffer || !this._yTexture || !this._uvTexture || (isPlanarUv && !this._vTexture)) {
+            // Didn't fully initialize our WebGL objects, bail out but reset the few bits we've changed
+            gl.bindTexture(gl.TEXTURE_2D, previousBoundTexture0);
+            gl.activeTexture(previousActiveTexture);
+            gl.bindBuffer(gl.ARRAY_BUFFER, previousBoundArrayBuffer);
+            if (this._isWebGL2) {
+                gl2.bindVertexArray(previousVAO);
+            }
+            else if (this._vaoExtension) {
+                this._vaoExtension.bindVertexArrayOES(previousVAO);
+            }
+            gl.useProgram(previousProgram);
+            glStateManager.pop();
+            return;
+        }
+        const aPositionLoc = conversionShader.aPositionLoc;
+        let prevPosAttribState = undefined;
+        if (!this._useVao) {
+            // Capture previous vertex attribute state for the location we're using for positions
+            traceStart('getVertexAttribState');
+            prevPosAttribState = this._getVertexAttribState(gl, aPositionLoc);
+            traceEnd();
+        }
+        // Global state
+        gl.disable(gl.SCISSOR_TEST);
+        gl.disable(gl.DEPTH_TEST);
+        gl.disable(gl.BLEND);
+        gl.disable(gl.CULL_FACE);
+        gl.disable(gl.STENCIL_TEST);
+        gl.pixelStorei(gl.UNPACK_FLIP_Y_WEBGL, false);
+        // Upload the Y texture data
+        gl.bindTexture(gl.TEXTURE_2D, this._yTexture.texture);
+        gl.texSubImage2D(gl.TEXTURE_2D, 0, 0, 0, y.width, y.height, this._yTexture.format, gl.UNSIGNED_BYTE, y.data);
+        // Upload UV plane
+        gl.activeTexture(gl.TEXTURE1);
+        const previousBoundTexture1 = gl.getParameter(gl.TEXTURE_BINDING_2D);
+        gl.bindTexture(gl.TEXTURE_2D, this._uvTexture.texture);
+        gl.texSubImage2D(gl.TEXTURE_2D, 0, 0, 0, uv.width, uv.height, this._uvTexture.format, gl.UNSIGNED_BYTE, uv.data);
+        let previousBoundTexture2;
+        if (isPlanarUv) {
+            gl.activeTexture(gl.TEXTURE2);
+            previousBoundTexture2 = gl.getParameter(gl.TEXTURE_BINDING_2D);
+            gl.bindTexture(gl.TEXTURE_2D, this._vTexture.texture);
+            gl.texSubImage2D(gl.TEXTURE_2D, 0, 0, 0, uv.width, uv.height, this._vTexture.format, gl.UNSIGNED_BYTE, uv.dataVPlane);
+        }
+        // Bind framebuffer and set the destination texture (caller must ensure its size matches the Y data)
+        gl.bindFramebuffer(gl.FRAMEBUFFER, this._framebufferId);
+        gl.framebufferTexture2D(gl.FRAMEBUFFER, gl.COLOR_ATTACHMENT0, gl.TEXTURE_2D, destination, 0);
+        gl.viewport(0, 0, y.width, y.height);
+        // We'll be replacing all the content - clear is a good hint for this on mobile
+        gl.clear(gl.COLOR_BUFFER_BIT);
+        // Set up bindings for vertex attributes
+        if (this._useVao) {
+            if (this._isWebGL2) {
+                gl2.bindVertexArray(this._vao);
+            }
+            else if (this._vaoExtension) {
+                this._vaoExtension.bindVertexArrayOES(this._vao);
+            }
+        }
+        else {
+            gl.bindBuffer(gl.ARRAY_BUFFER, this._vertexBuffer);
+            gl.vertexAttribPointer(aPositionLoc, 2, gl.FLOAT, false, 0, 0);
+            if (prevPosAttribState && prevPosAttribState.divisor != 0) {
+                if (this._isWebGL2) {
+                    gl2.vertexAttribDivisor(aPositionLoc, 0);
+                }
+                else if (this._instancedArraysExtension) {
+                    this._instancedArraysExtension.vertexAttribDivisorANGLE(aPositionLoc, 0);
+                }
+            }
+            gl.enableVertexAttribArray(aPositionLoc);
+        }
+        // Tell WebGL to use our program when drawing
+        gl.useProgram(conversionShader.program);
+        // Do the drawing...
+        gl.drawArrays(gl.TRIANGLE_STRIP, 0, 4);
+        // Remove the destination texture from the framebuffer
+        gl.framebufferTexture2D(gl.FRAMEBUFFER, gl.COLOR_ATTACHMENT0, gl.TEXTURE_2D, null, 0);
+        // Reset the state
+        if (reenableBlend)
+            gl.enable(gl.BLEND);
+        if (reenableCullFace)
+            gl.enable(gl.CULL_FACE);
+        if (reenableDepthTest)
+            gl.enable(gl.DEPTH_TEST);
+        if (reenableScissorTest)
+            gl.enable(gl.SCISSOR_TEST);
+        if (reenableStencilTest)
+            gl.enable(gl.STENCIL_TEST);
+        gl.pixelStorei(gl.UNPACK_FLIP_Y_WEBGL, previousUnpackFlip);
+        gl.bindFramebuffer(gl.FRAMEBUFFER, previousBoundFramebuffer);
+        if (isPlanarUv) {
+            gl.bindTexture(gl.TEXTURE_2D, previousBoundTexture2);
+            gl.activeTexture(gl.TEXTURE1);
+        }
+        gl.bindTexture(gl.TEXTURE_2D, previousBoundTexture1);
+        gl.activeTexture(gl.TEXTURE0);
+        gl.bindTexture(gl.TEXTURE_2D, previousBoundTexture0);
+        gl.activeTexture(previousActiveTexture);
+        if (this._useVao) {
+            if (this._isWebGL2) {
+                gl2.bindVertexArray(previousVAO);
+            }
+            else if (this._vaoExtension) {
+                this._vaoExtension.bindVertexArrayOES(previousVAO);
+            }
+        }
+        else if (prevPosAttribState) {
+            gl.bindBuffer(gl.ARRAY_BUFFER, prevPosAttribState.bufferBinding);
+            gl.vertexAttribPointer(aPositionLoc, prevPosAttribState.size, prevPosAttribState.type, prevPosAttribState.normalized, prevPosAttribState.stride, prevPosAttribState.offset);
+            if (prevPosAttribState.divisor != 0) {
+                if (this._isWebGL2) {
+                    gl2.vertexAttribDivisor(aPositionLoc, prevPosAttribState.divisor);
+                }
+                else if (this._instancedArraysExtension) {
+                    this._instancedArraysExtension.vertexAttribDivisorANGLE(aPositionLoc, prevPosAttribState.divisor);
+                }
+            }
+            if (!prevPosAttribState.enabled)
+                gl.disableVertexAttribArray(aPositionLoc);
+        }
+        gl.bindBuffer(gl.ARRAY_BUFFER, previousBoundArrayBuffer);
+        gl.useProgram(previousProgram);
+        glStateManager.pop();
+    }
+    _getVertexAttribState(gl, index) {
+        let divisor = 0;
+        if (this._isWebGL2) {
+            traceStart("getWebGL2Divisor");
+            divisor = gl.getVertexAttrib(index, gl.VERTEX_ATTRIB_ARRAY_DIVISOR);
+            traceEnd();
+        }
+        else if (this._instancedArraysExtension) {
+            traceStart("getExtDivisor");
+            divisor = gl.getVertexAttrib(index, this._instancedArraysExtension.VERTEX_ATTRIB_ARRAY_DIVISOR_ANGLE);
+            traceEnd();
+        }
+        return {
+            size: gl.getVertexAttrib(index, gl.VERTEX_ATTRIB_ARRAY_SIZE),
+            type: gl.getVertexAttrib(index, gl.VERTEX_ATTRIB_ARRAY_TYPE),
+            normalized: gl.getVertexAttrib(index, gl.VERTEX_ATTRIB_ARRAY_NORMALIZED),
+            stride: gl.getVertexAttrib(index, gl.VERTEX_ATTRIB_ARRAY_STRIDE),
+            offset: gl.getVertexAttribOffset(index, gl.VERTEX_ATTRIB_ARRAY_POINTER),
+            bufferBinding: gl.getVertexAttrib(index, gl.VERTEX_ATTRIB_ARRAY_BUFFER_BINDING),
+            enabled: gl.getVertexAttrib(index, gl.VERTEX_ATTRIB_ARRAY_ENABLED),
+            divisor
+        };
+    }
+    _prepareContext() {
+        traceStart('YUV: prepareContext');
+        const gl = this._gl;
+        this._framebufferId = gl.createFramebuffer();
+        let vertexData = new Float32Array([
+            -1.0, -1.0,
+            -1.0, 1.0,
+            1.0, -1.0,
+            1.0, 1.0
+        ]);
+        this._vertexBuffer = gl.createBuffer();
+        gl.bindBuffer(gl.ARRAY_BUFFER, this._vertexBuffer);
+        gl.bufferData(gl.ARRAY_BUFFER, vertexData, gl.STATIC_DRAW);
+        if (this._useVao) {
+            if (this._isWebGL2) {
+                const gl2 = gl;
+                this._vao = gl2.createVertexArray();
+                gl2.bindVertexArray(this._vao);
+            }
+            else if (this._vaoExtension) {
+                this._vao = this._vaoExtension.createVertexArrayOES();
+                this._vaoExtension.bindVertexArrayOES(this._vao);
+            }
+            const aPositionLoc = 0; // Ensured with bindAttribLocation
+            // _vertexBuffer is already bound to gl.ARRAY_BUFFER
+            gl.vertexAttribPointer(aPositionLoc, 2, gl.FLOAT, false, 0, 0);
+            gl.enableVertexAttribArray(aPositionLoc);
+        }
+        traceEnd();
+    }
+    _prepareConversionShader(uvLayout) {
+        const gl = this._gl;
+        const isPlanar = (uvLayout === UvLayout.PLANAR_UV);
+        let program = gl.createProgram();
+        if (!program)
+            throw new Error("Couldn't create program");
+        let vertexShader = compileShader(gl, gl.VERTEX_SHADER, conversionVsSource);
+        const singleChannel = this._isWebGL2 ? 'r' : 'a';
+        const twoChannel0 = 'r';
+        const twoChannel1 = this._isWebGL2 ? 'g' : 'a';
+        let fragmentSource;
+        switch (uvLayout) {
+            case UvLayout.INTERLEAVED_UV:
+                fragmentSource = conversionFsSource(singleChannel, twoChannel0, twoChannel1);
+                break;
+            case UvLayout.INTERLEAVED_VU:
+                fragmentSource = conversionFsSource(singleChannel, twoChannel1, twoChannel0);
+                break;
+            case UvLayout.PLANAR_UV:
+                fragmentSource = conversionFsSourcePlanar(singleChannel);
+                break;
+        }
+        let fragmentShader = compileShader(gl, gl.FRAGMENT_SHADER, fragmentSource);
+        gl.attachShader(program, vertexShader);
+        gl.attachShader(program, fragmentShader);
+        gl.bindAttribLocation(program, 0, "aPosition");
+        linkProgram(gl, program);
+        let uYSamplerLoc = gl.getUniformLocation(program, "uYSampler");
+        let uUVSamplerLoc = (!isPlanar ? gl.getUniformLocation(program, "uUVSampler") : null);
+        let uUSamplerLoc = (isPlanar ? gl.getUniformLocation(program, "uUSampler") : null);
+        let uVSamplerLoc = (isPlanar ? gl.getUniformLocation(program, "uVSampler") : null);
+        let err = gl.getError();
+        if (err != gl.NO_ERROR)
+            zcout(`YUVConversion: Error before setting uniforms: 0x${err.toString(16)}`);
+        gl.useProgram(program);
+        gl.uniform1i(uYSamplerLoc, 0);
+        gl.uniform1i(isPlanar ? uUSamplerLoc : uUVSamplerLoc, 1);
+        if (isPlanar)
+            gl.uniform1i(uVSamplerLoc, 2);
+        err = gl.getError();
+        if (err != gl.NO_ERROR)
+            zcout(`YUVConversion: Error after setting uniforms: 0x${err.toString(16)}`);
+        this._conversionShaders[uvLayout] = {
+            program,
+            aPositionLoc: gl.getAttribLocation(program, "aPosition")
+        };
+    }
+    _prepareTextures(y, uv, isPlanarUv) {
+        const gl = this._gl;
+        if (!this._yTexture || this._yTexture.width != y.width || this._yTexture.height != y.height) {
+            traceStart("YUV: Creating Y texture");
+            if (this._yTexture)
+                gl.deleteTexture(this._yTexture.texture);
+            this._yTexture = this._createSizedTexture(y.width, y.height, this._oneComponentInternalFormat, this._oneComponentFormat);
+            traceEnd();
+        }
+        if (!this._uvTexture || this._uvTexture.width != uv.width || this._uvTexture.height != uv.height ||
+            this._uvTextureSinglePlane != isPlanarUv) {
+            traceStart("YUV: Creating UV texture");
+            if (this._uvTexture)
+                gl.deleteTexture(this._uvTexture.texture);
+            const internalFormat = isPlanarUv ? this._oneComponentInternalFormat : this._twoComponentInternalFormat;
+            const format = isPlanarUv ? this._oneComponentFormat : this._twoComponentFormat;
+            this._uvTexture = this._createSizedTexture(uv.width, uv.height, internalFormat, format);
+            this._uvTextureSinglePlane = isPlanarUv;
+            traceEnd();
+        }
+        if (!isPlanarUv)
+            return;
+        if (!this._vTexture || this._vTexture.width != uv.width || this._vTexture.height != uv.height) {
+            traceStart("YUV: Creating V texture");
+            if (this._vTexture)
+                gl.deleteTexture(this._vTexture.texture);
+            this._vTexture = this._createSizedTexture(uv.width, uv.height, this._oneComponentInternalFormat, this._oneComponentFormat);
+            traceEnd();
+        }
+    }
+    _createSizedTexture(width, height, internalFormat, format) {
+        const gl = this._gl;
+        const texture = gl.createTexture();
+        if (!texture)
+            return null;
+        gl.bindTexture(gl.TEXTURE_2D, texture);
+        gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE);
+        gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE);
+        gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR);
+        if (this._isWebGL2) {
+            gl.texStorage2D(gl.TEXTURE_2D, 1, internalFormat, width, height);
+        }
+        else {
+            gl.texImage2D(gl.TEXTURE_2D, 0, internalFormat, width, height, 0, format, gl.UNSIGNED_BYTE, null);
+        }
+        return { texture, width, height, format, internalFormat };
+    }
+}
+let conversionVsSource = `
+    attribute vec4 aPosition;
+    varying highp vec2 vUV;
+
+    void main(void) {
+      gl_Position = aPosition;
+      vUV = 0.5 * (aPosition.xy + vec2(1.0, 1.0));
+    }
+`;
+// Fragment shader program
+let conversionFsSource = (yChannel, uChannel, vChannel) => `
+  varying highp vec2 vUV;
+
+  uniform sampler2D uYSampler;
+  uniform sampler2D uUVSampler;
+
+  void main(void) {
+    mediump vec4 uv = texture2D(uUVSampler, vUV);
+    mediump vec3 ycbcr = vec3(texture2D(uYSampler, vUV).${yChannel}, uv.${uChannel} - 0.5, uv.${vChannel} - 0.5);
+    const mediump mat3 ycbcrToRgb = mat3(1.0000,  1.0000, 1.0000,
+                                         0.0000, -0.3441, 1.7720,
+                                         1.4020, -0.7141, 0.0000);
+
+    gl_FragColor = vec4(ycbcrToRgb * ycbcr, 1.0);
+  }
+`;
+let conversionFsSourcePlanar = (channel) => `
+  varying highp vec2 vUV;
+
+  uniform sampler2D uYSampler;
+  uniform sampler2D uUSampler;
+  uniform sampler2D uVSampler;
+
+  void main(void) {
+    mediump vec3 ycbcr = vec3(
+        texture2D(uYSampler, vUV).${channel},
+        texture2D(uUSampler, vUV).${channel} - 0.5,
+        texture2D(uVSampler, vUV).${channel} - 0.5);
+    const mediump mat3 ycbcrToRgb = mat3(1.0000,  1.0000, 1.0000,
+                                         0.0000, -0.3441, 1.7720,
+                                         1.4020, -0.7141, 0.0000);
+
+    gl_FragColor = vec4(ycbcrToRgb * ycbcr, 1.0);
+  }
+`;