@zappar/zappar-cv 0.3.14 → 0.3.17

package/CHANGELOG.md

@@ -1,5 +1,23 @@
 # Changelog
 
+## [0.3.17] - 2022-07-21
+
+### Fixed
+
+- Preserve `TextureAttribDivisor`, `VertexAttribDivisor` and `STENCIL_TEST` WebGL state.
+
+## [0.3.16] - 2021-02-07
+
+### Fixed
+
+- External WebGL state is preserved during `process_gl` calls
+
+## [0.3.15] - 2021-01-25
+
+### Fixed
+
+- Issues on iPad Pro
+
 ## [0.3.14] - 2021-10-25
 
 ### Changed

package/README.md

@@ -18,8 +18,8 @@ npm i @zappar/zappar-cv
 
 You can use our CDN from within your HTML:
 ```
-<script src="https://libs.zappar.com/zappar-cv/0.3.14/zappar-cv.js"></script>
+<script src="https://libs.zappar.com/zappar-cv/0.3.17/zappar-cv.js"></script>
 ```
 
 Or you can download and host our standalone JavaScript bundle:
-[https://libs.zappar.com/zappar-cv/0.3.14/zappar-cv.zip](https://libs.zappar.com/zappar-cv/0.3.14/zappar-cv.zip)
+[https://libs.zappar.com/zappar-cv/0.3.17/zappar-cv.zip](https://libs.zappar.com/zappar-cv/0.3.17/zappar-cv.zip)

package/lib/gl-state-manager.d.ts

@@ -0,0 +1,9 @@
+export declare class GLStateManager {
+    private _gl;
+    static get(gl: WebGLRenderingContext): GLStateManager;
+    private _viewports;
+    private _underlyingViewport;
+    private constructor();
+    push(): void;
+    pop(): void;
+}

package/lib/gl-state-manager.js

@@ -0,0 +1,35 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.GLStateManager = void 0;
+const managers = new Map();
+class GLStateManager {
+    constructor(_gl) {
+        this._gl = _gl;
+        this._viewports = [];
+        this._underlyingViewport = this._gl.viewport;
+        this._viewports.push(this._gl.getParameter(this._gl.VIEWPORT));
+        this._gl.viewport = (x, y, width, height) => {
+            this._viewports[this._viewports.length - 1] = [x, y, width, height];
+            this._underlyingViewport.call(this._gl, x, y, width, height);
+        };
+    }
+    static get(gl) {
+        let existing = managers.get(gl);
+        if (!existing) {
+            existing = new GLStateManager(gl);
+            managers.set(gl, existing);
+        }
+        return existing;
+    }
+    push() {
+        this._viewports.push(this._viewports[this._viewports.length - 1]);
+    }
+    pop() {
+        const current = this._viewports.pop();
+        const prev = this._viewports[this._viewports.length - 1];
+        if (!current || current[0] !== prev[0] || current[1] !== prev[1] || current[2] !== prev[2] || current[3] !== prev[3]) {
+            this._underlyingViewport.call(this._gl, prev[0], prev[1], prev[2], prev[3]);
+        }
+    }
+}
+exports.GLStateManager = GLStateManager;

package/lib/html-element-source.d.ts

@@ -30,6 +30,8 @@ export declare class HTMLElementSource extends Source {
     start(): void;
     getFrame(currentlyProcessing: boolean): CameraFrameInfo | undefined;
     private _processFrame;
+    private _instancedArraysExtension;
+    private _instancedArraysExtensionSet;
     private _uploadFrame;
     private _readFrame;
     private _updateTransforms;

package/lib/html-element-source.js

@@ -7,6 +7,7 @@ const profile_1 = require("./profile");
 const shader_1 = require("./shader");
 const gl_matrix_1 = require("gl-matrix");
 const loglevel_1 = require("./loglevel");
+const gl_state_manager_1 = require("./gl-state-manager");
 let latest = 1;
 let byId = new Map();
 class HTMLElementSource extends source_1.Source {
@@ -27,6 +28,7 @@ class HTMLElementSource extends source_1.Source {
         this._framebufferHeight = 0;
         this._framebufferId = null;
         this._renderTexture = null;
+        this._instancedArraysExtensionSet = false;
         let video = this._video;
         if (this._video instanceof HTMLVideoElement) {
             video.addEventListener("loadedmetadata", () => { this._hadFrames = true; });
@@ -55,6 +57,7 @@ class HTMLElementSource extends source_1.Source {
         this._vertexBuffer = undefined;
         this._indexBuffer = undefined;
         this._greyscaleShader = undefined;
+        this._instancedArraysExtensionSet = false;
     }
     destroy() {
         let p = pipeline_1.Pipeline.get(this._pipeline);
@@ -125,15 +128,34 @@ class HTMLElementSource extends source_1.Source {
         let pipeline = pipeline_1.Pipeline.get(this._pipeline);
         if (!pipeline)
             return;
-        let gl = pipeline.glContext;
+        let { glContext: gl, isWebGL2 } = pipeline;
         if (!gl)
             return;
+        if (!isWebGL2 && !this._instancedArraysExtensionSet) {
+            this._instancedArraysExtension = gl.getExtension("ANGLE_instanced_arrays");
+            this._instancedArraysExtensionSet = true;
+        }
+        const glStateManager = gl_state_manager_1.GLStateManager.get(gl);
+        glStateManager.push();
+        const reenableScissorTest = gl.isEnabled(gl.SCISSOR_TEST);
+        const reenableDepthTest = gl.isEnabled(gl.DEPTH_TEST);
+        const reenableBlend = gl.isEnabled(gl.BLEND);
+        const reenableCullFace = gl.isEnabled(gl.CULL_FACE);
+        const reenableStencilTest = gl.isEnabled(gl.STENCIL_TEST);
+        const previousActiveTexture = gl.getParameter(gl.ACTIVE_TEXTURE);
+        const previousUnpackFlip = gl.getParameter(gl.UNPACK_FLIP_Y_WEBGL);
+        const previousProgram = gl.getParameter(gl.CURRENT_PROGRAM);
+        gl.activeTexture(gl.TEXTURE0);
+        const previousBoundTexture = gl.getParameter(gl.TEXTURE_BINDING_2D);
+        const previousBoundFramebuffer = gl.getParameter(gl.FRAMEBUFFER_BINDING);
+        const previousBoundArrayBuffer = gl.getParameter(gl.ARRAY_BUFFER_BINDING);
+        const previousBoundElementArrayBuffer = gl.getParameter(gl.ELEMENT_ARRAY_BUFFER_BINDING);
         gl.disable(gl.SCISSOR_TEST);
         gl.disable(gl.DEPTH_TEST);
         gl.disable(gl.BLEND);
         gl.disable(gl.CULL_FACE);
+        gl.disable(gl.STENCIL_TEST);
         gl.pixelStorei(gl.UNPACK_FLIP_Y_WEBGL, false);
-        gl.activeTexture(gl.TEXTURE0);
         gl.bindTexture(gl.TEXTURE_2D, this._currentVideoTexture);
         const level = 0;
         const internalFormat = gl.RGBA;
@@ -157,13 +179,43 @@ class HTMLElementSource extends source_1.Source {
         let vbo = this._getVertexBuffer(gl);
         let ibo = this._getIndexBuffer(gl);
         let shader = this._getGreyscaleShader(gl);
-        //     // Rendering to the greyscale conversion buffer - bind the framebuffer
+        const previousVertexAttribSize = gl.getVertexAttrib(shader.aVertexPositionLoc, gl.VERTEX_ATTRIB_ARRAY_SIZE);
+        const previousVertexAttribType = gl.getVertexAttrib(shader.aVertexPositionLoc, gl.VERTEX_ATTRIB_ARRAY_TYPE);
+        const previousVertexAttribNormalized = gl.getVertexAttrib(shader.aVertexPositionLoc, gl.VERTEX_ATTRIB_ARRAY_NORMALIZED);
+        const previousVertexAttribStride = gl.getVertexAttrib(shader.aVertexPositionLoc, gl.VERTEX_ATTRIB_ARRAY_STRIDE);
+        const previousVertexAttribOffset = gl.getVertexAttribOffset(shader.aVertexPositionLoc, gl.VERTEX_ATTRIB_ARRAY_POINTER);
+        const previousVertexAttribEnabled = gl.getVertexAttrib(shader.aVertexPositionLoc, gl.VERTEX_ATTRIB_ARRAY_ENABLED);
+        const previousVertexAttribBufferBinding = gl.getVertexAttrib(shader.aVertexPositionLoc, gl.VERTEX_ATTRIB_ARRAY_BUFFER_BINDING);
+        const previousTextureAttribSize = gl.getVertexAttrib(shader.aTextureCoordLoc, gl.VERTEX_ATTRIB_ARRAY_SIZE);
+        const previousTextureAttribType = gl.getVertexAttrib(shader.aTextureCoordLoc, gl.VERTEX_ATTRIB_ARRAY_TYPE);
+        const previousTextureAttribNormalized = gl.getVertexAttrib(shader.aTextureCoordLoc, gl.VERTEX_ATTRIB_ARRAY_NORMALIZED);
+        const previousTextureAttribStride = gl.getVertexAttrib(shader.aTextureCoordLoc, gl.VERTEX_ATTRIB_ARRAY_STRIDE);
+        const previousTextureAttribOffset = gl.getVertexAttribOffset(shader.aTextureCoordLoc, gl.VERTEX_ATTRIB_ARRAY_POINTER);
+        const previousTextureAttribEnabled = gl.getVertexAttrib(shader.aTextureCoordLoc, gl.VERTEX_ATTRIB_ARRAY_ENABLED);
+        const previousTextureAttribBufferBinding = gl.getVertexAttrib(shader.aTextureCoordLoc, gl.VERTEX_ATTRIB_ARRAY_BUFFER_BINDING);
+        let previousVertexAttribDivisor = 0;
+        let previousTextureAttribDivisor = 0;
+        if (isWebGL2) {
+            previousVertexAttribDivisor = gl.getVertexAttrib(shader.aVertexPositionLoc, gl.VERTEX_ATTRIB_ARRAY_DIVISOR);
+            previousTextureAttribDivisor = gl.getVertexAttrib(shader.aTextureCoordLoc, gl.VERTEX_ATTRIB_ARRAY_DIVISOR);
+            gl.vertexAttribDivisor(shader.aVertexPositionLoc, 0);
+            gl.vertexAttribDivisor(shader.aTextureCoordLoc, 0);
+        }
+        else if (this._instancedArraysExtension) {
+            previousVertexAttribDivisor = gl.getVertexAttrib(shader.aVertexPositionLoc, this._instancedArraysExtension.VERTEX_ATTRIB_ARRAY_DIVISOR_ANGLE);
+            previousTextureAttribDivisor = gl.getVertexAttrib(shader.aTextureCoordLoc, this._instancedArraysExtension.VERTEX_ATTRIB_ARRAY_DIVISOR_ANGLE);
+            this._instancedArraysExtension.vertexAttribDivisorANGLE(shader.aVertexPositionLoc, 0);
+            this._instancedArraysExtension.vertexAttribDivisorANGLE(shader.aTextureCoordLoc, 0);
+        }
+        // Rendering to the greyscale conversion buffer - bind the framebuffer
         gl.bindFramebuffer(gl.FRAMEBUFFER, framebuffer);
+        // const t_before = performance.now();
+        // const viewport = gl.getParameter(gl.VIEWPORT);
+        // console.log("Viewport time", performance.now() - t_before, viewport);
         gl.viewport(0, 0, this._framebufferWidth, this._framebufferHeight);
-        //     // We'll be replacing all the content - clear is a good hint for this on mobile
-        gl.clearColor(0.0, 0.0, 0.0, 1.0);
+        // We'll be replacing all the content - clear is a good hint for this on mobile
         gl.clear(gl.COLOR_BUFFER_BIT);
-        //     // Set up bindings for vertex attributes
+        // Set up bindings for vertex attributes
         gl.bindBuffer(gl.ARRAY_BUFFER, vbo);
         gl.vertexAttribPointer(shader.aVertexPositionLoc, 2, gl.FLOAT, false, 4 * 4, 0);
         gl.enableVertexAttribArray(shader.aVertexPositionLoc);
@@ -183,14 +235,38 @@ class HTMLElementSource extends source_1.Source {
         gl.uniform1i(shader.uSamplerLoc, 0);
         // Do the drawing...
         gl.drawElements(gl.TRIANGLES, 6, gl.UNSIGNED_SHORT, 0);
-        // Disable attribute arrays
-        gl.disableVertexAttribArray(shader.aVertexPositionLoc);
-        gl.disableVertexAttribArray(shader.aTextureCoordLoc);
-        gl.bindFramebuffer(gl.FRAMEBUFFER, null);
-        gl.bindBuffer(gl.ARRAY_BUFFER, null);
-        gl.bindBuffer(gl.ELEMENT_ARRAY_BUFFER, null);
-        gl.useProgram(null);
-        gl.bindTexture(gl.TEXTURE_2D, null);
+        gl.bindBuffer(gl.ARRAY_BUFFER, previousVertexAttribBufferBinding);
+        gl.vertexAttribPointer(shader.aVertexPositionLoc, previousVertexAttribSize, previousVertexAttribType, previousVertexAttribNormalized, previousVertexAttribStride, previousVertexAttribOffset);
+        gl.bindBuffer(gl.ARRAY_BUFFER, previousTextureAttribBufferBinding);
+        gl.vertexAttribPointer(shader.aTextureCoordLoc, previousTextureAttribSize, previousTextureAttribType, previousTextureAttribNormalized, previousTextureAttribStride, previousTextureAttribOffset);
+        gl.bindBuffer(gl.ARRAY_BUFFER, previousBoundArrayBuffer);
+        gl.bindBuffer(gl.ELEMENT_ARRAY_BUFFER, previousBoundElementArrayBuffer);
+        if (!previousVertexAttribEnabled)
+            gl.disableVertexAttribArray(shader.aVertexPositionLoc);
+        if (!previousTextureAttribEnabled)
+            gl.disableVertexAttribArray(shader.aTextureCoordLoc);
+        if (isWebGL2) {
+            gl.vertexAttribDivisor(shader.aVertexPositionLoc, previousVertexAttribDivisor);
+            gl.vertexAttribDivisor(shader.aTextureCoordLoc, previousTextureAttribDivisor);
+        }
+        else if (this._instancedArraysExtension) {
+            this._instancedArraysExtension.vertexAttribDivisorANGLE(shader.aVertexPositionLoc, previousVertexAttribDivisor);
+            this._instancedArraysExtension.vertexAttribDivisorANGLE(shader.aTextureCoordLoc, previousTextureAttribDivisor);
+        }
+        gl.bindFramebuffer(gl.FRAMEBUFFER, previousBoundFramebuffer);
+        gl.useProgram(previousProgram);
+        gl.bindTexture(gl.TEXTURE_2D, previousBoundTexture);
+        gl.activeTexture(previousActiveTexture);
+        gl.pixelStorei(gl.UNPACK_FLIP_Y_WEBGL, previousUnpackFlip);
+        if (reenableBlend)
+            gl.enable(gl.BLEND);
+        if (reenableCullFace)
+            gl.enable(gl.CULL_FACE);
+        if (reenableDepthTest)
+            gl.enable(gl.DEPTH_TEST);
+        if (reenableScissorTest)
+            gl.enable(gl.SCISSOR_TEST);
+        glStateManager.pop();
     }
     _readFrame(p, gl) {
         if (!this._currentVideoTexture)
@@ -208,10 +284,11 @@ class HTMLElementSource extends source_1.Source {
             pixels = new ArrayBuffer(greySize);
         }
         let pixelsView = new Uint8Array(pixels);
+        const previousBoundFramebuffer = gl.getParameter(gl.FRAMEBUFFER_BINDING);
         let framebuffer = this._getFramebuffer(gl, profile_1.profile.dataWidth / 4, profile_1.profile.dataHeight);
         gl.bindFramebuffer(gl.FRAMEBUFFER, framebuffer);
         gl.readPixels(0, 0, this._framebufferWidth, this._framebufferHeight, gl.RGBA, gl.UNSIGNED_BYTE, pixelsView);
-        gl.bindFramebuffer(gl.FRAMEBUFFER, null);
+        gl.bindFramebuffer(gl.FRAMEBUFFER, previousBoundFramebuffer);
         return {
             uvTransform: this._cameraUvTransform,
             data: pixels,

package/lib/pipeline.d.ts

@@ -15,15 +15,17 @@ export declare class Pipeline {
     private cameraTokens;
     private nextCameraToken;
     private tokensInFlight;
-    glContext: WebGLRenderingContext | undefined;
+    glContext: WebGLRenderingContext | WebGL2RenderingContext | undefined;
     private videoTextures;
     cameraPixelArrays: ArrayBuffer[];
     private _cameraDraw;
     private _faceDraw;
     private _faceProjectDraw;
+    private _isWebGL2;
     onGLContextReset: Event;
     static create(client: zappar_cwrap, mgr: MsgManager): zappar_pipeline_t;
     static get(p: zappar_pipeline_t): Pipeline | undefined;
+    get isWebGL2(): boolean;
     private constructor();
     frameUpdate(client: zappar_client): void;
     cameraTokenReturn(tokenId: number, pixelArray: ArrayBuffer): void;

package/lib/pipeline.js

@@ -21,6 +21,7 @@ class Pipeline {
         this.tokensInFlight = 0;
         this.videoTextures = [];
         this.cameraPixelArrays = [];
+        this._isWebGL2 = false;
         this.onGLContextReset = new event_1.Event();
     }
     static create(client, mgr) {
@@ -31,6 +32,9 @@ class Pipeline {
     static get(p) {
         return byId.get(p);
     }
+    get isWebGL2() {
+        return this._isWebGL2;
+    }
     frameUpdate(client) {
         for (let msg of this.pendingMessages) {
             client.processMessages(msg);
@@ -106,6 +110,7 @@ class Pipeline {
     glContextSet(gl, texturePool) {
         this.glContextLost();
         this.glContext = gl;
+        this._isWebGL2 = gl.getParameter(gl.VERSION).indexOf("WebGL 2") >= 0;
         texturePool = texturePool || [];
         for (let i = 0; i < 4; i++) {
             let tex = texturePool[i] || gl.createTexture();

package/lib/profile.js

@@ -26,43 +26,41 @@ let engine = (agent.getEngine().name || "unknown").toLowerCase();
 if (engine === "webkit" && os !== "ios") {
     exports.profile.deviceMotionMutliplier = 1.0;
     if (window.orientation !== undefined) {
-        // iPad
-        iDevice();
+        // iPad. The version number isn't reliable as it masquerades as MacOS
+        iDevice("15.0");
     }
 }
 if (engine === "webkit" && os === "ios") {
     exports.profile.deviceMotionMutliplier = 1.0;
-    iDevice();
+    const version = agent.getOS().version || "15.0";
+    iDevice(version);
 }
-function iDevice() {
-    let version = agent.getOS().version;
-    if (version) {
-        let versionParts = version.split(".");
-        if (versionParts.length >= 2) {
-            const majorVersion = parseInt(versionParts[0]);
-            const minorVersion = parseInt(versionParts[1]);
-            if (majorVersion < 11 ||
-                (majorVersion === 11 && minorVersion < 3)) {
-                exports.profile.blacklisted = true;
-            }
-            if (majorVersion < 12 ||
-                (majorVersion === 12 && minorVersion < 2)) {
-                exports.profile.videoElementInDOM = true;
-            }
-            if ((majorVersion === 12 && minorVersion >= 2) || (majorVersion >= 13))
-                exports.profile.showGyroPermissionsWarningIfNecessary = true;
-            if (majorVersion >= 13) {
-                exports.profile.showSafariPermissionsResetIfNecessary = true;
-            }
-            if (((majorVersion >= 12 && minorVersion > 1) || (majorVersion >= 13)) &&
-                navigator.mediaDevices &&
-                navigator.mediaDevices.getSupportedConstraints &&
-                navigator.mediaDevices.getSupportedConstraints().frameRate) {
-                exports.profile.requestHighFrameRate = true;
-                // Avoid bug where iOS letterboxes 16:9 into 4:3 for high fps
-                exports.profile.videoHeight = 360;
-                exports.profile.dataHeight = 180;
-            }
+function iDevice(version) {
+    let versionParts = version.split(".");
+    if (versionParts.length >= 2) {
+        const majorVersion = parseInt(versionParts[0]);
+        const minorVersion = parseInt(versionParts[1]);
+        if (majorVersion < 11 ||
+            (majorVersion === 11 && minorVersion < 3)) {
+            exports.profile.blacklisted = true;
+        }
+        if (majorVersion < 12 ||
+            (majorVersion === 12 && minorVersion < 2)) {
+            exports.profile.videoElementInDOM = true;
+        }
+        if ((majorVersion === 12 && minorVersion >= 2) || (majorVersion >= 13))
+            exports.profile.showGyroPermissionsWarningIfNecessary = true;
+        if (majorVersion >= 13) {
+            exports.profile.showSafariPermissionsResetIfNecessary = true;
+        }
+        if (((majorVersion >= 12 && minorVersion > 1) || (majorVersion >= 13)) &&
+            navigator.mediaDevices &&
+            navigator.mediaDevices.getSupportedConstraints &&
+            navigator.mediaDevices.getSupportedConstraints().frameRate) {
+            exports.profile.requestHighFrameRate = true;
+            // Avoid bug where iOS letterboxes 16:9 into 4:3 for high fps
+            exports.profile.videoHeight = 360;
+            exports.profile.dataHeight = 180;
         }
     }
 }

package/lib/version.d.ts

@@ -1 +1 @@
-export declare const VERSION = "0.3.14";
+export declare const VERSION = "0.3.17";

package/lib/version.js

@@ -1,4 +1,4 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.VERSION = void 0;
-exports.VERSION = "0.3.14";
+exports.VERSION = "0.3.17";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@zappar/zappar-cv",
-  "version": "0.3.14",
+  "version": "0.3.17",
   "description": "Zappar's core computer vision library, supporting image, face and world tracking.",
   "main": "lib/index.js",
   "types": "lib/index.d.ts",