@zappar/zappar-cv 0.4.0-beta.1 → 0.4.0-beta.10

package/lib/image-process-gl.js

@@ -0,0 +1,298 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ImageProcessGL = void 0;
+const gl_matrix_1 = require("gl-matrix");
+const gl_state_manager_1 = require("./gl-state-manager");
+const profile_1 = require("./profile");
+const shader_1 = require("./shader");
+class ImageProcessGL {
+    constructor(_gl) {
+        this._gl = _gl;
+        this._isPaused = true;
+        this._hadFrames = false;
+        this._isUserFacing = false;
+        this._cameraToScreenRotation = 0;
+        this._isUploadFrame = true;
+        this._computedTransformRotation = -1;
+        this._computedFrontCameraRotation = false;
+        this._cameraUvTransform = gl_matrix_1.mat4.create();
+        this._framebufferWidth = 0;
+        this._framebufferHeight = 0;
+        this._framebufferId = null;
+        this._renderTexture = null;
+    }
+    resetGLContext() {
+        this._framebufferId = null;
+        this._renderTexture = null;
+        this._vertexBuffer = undefined;
+        this._indexBuffer = undefined;
+        this._greyscaleShader = undefined;
+    }
+    destroy() {
+        this.resetGLContext();
+    }
+    uploadFrame(texture, img, rotation, fc) {
+        let gl = this._gl;
+        const glStateManager = gl_state_manager_1.GLStateManager.get(gl);
+        glStateManager.push();
+        const reenableScissorTest = gl.isEnabled(gl.SCISSOR_TEST);
+        const reenableDepthTest = gl.isEnabled(gl.DEPTH_TEST);
+        const reenableBlend = gl.isEnabled(gl.BLEND);
+        const reenableCullFace = gl.isEnabled(gl.CULL_FACE);
+        const previousActiveTexture = gl.getParameter(gl.ACTIVE_TEXTURE);
+        const previousUnpackFlip = gl.getParameter(gl.UNPACK_FLIP_Y_WEBGL);
+        const previousProgram = gl.getParameter(gl.CURRENT_PROGRAM);
+        gl.activeTexture(gl.TEXTURE0);
+        const previousBoundTexture = gl.getParameter(gl.TEXTURE_BINDING_2D);
+        const previousBoundFramebuffer = gl.getParameter(gl.FRAMEBUFFER_BINDING);
+        const previousBoundArrayBuffer = gl.getParameter(gl.ARRAY_BUFFER_BINDING);
+        const previousBoundElementArrayBuffer = gl.getParameter(gl.ELEMENT_ARRAY_BUFFER_BINDING);
+        gl.disable(gl.SCISSOR_TEST);
+        gl.disable(gl.DEPTH_TEST);
+        gl.disable(gl.BLEND);
+        gl.disable(gl.CULL_FACE);
+        gl.pixelStorei(gl.UNPACK_FLIP_Y_WEBGL, false);
+        gl.bindTexture(gl.TEXTURE_2D, texture);
+        const level = 0;
+        const internalFormat = gl.RGBA;
+        const srcFormat = gl.RGBA;
+        const srcType = gl.UNSIGNED_BYTE;
+        gl.texImage2D(gl.TEXTURE_2D, level, internalFormat, srcFormat, srcType, img);
+        let videoWidth = 0;
+        let videoHeight = 0;
+        if (typeof HTMLVideoElement !== "undefined" && img instanceof HTMLVideoElement) {
+            videoWidth = img.videoWidth;
+            videoHeight = img.videoHeight;
+        }
+        else {
+            videoWidth = img.width;
+            videoHeight = img.height;
+        }
+        if (videoHeight > videoWidth)
+            videoHeight = [videoWidth, videoWidth = videoHeight][0];
+        this._updateTransforms(rotation, fc);
+        let framebuffer = this._getFramebuffer(gl, profile_1.profile.dataWidth / 4, profile_1.profile.dataHeight);
+        let vbo = this._getVertexBuffer(gl);
+        let ibo = this._getIndexBuffer(gl);
+        let shader = this._getGreyscaleShader(gl);
+        const previousVertexAttribSize = gl.getVertexAttrib(shader.aVertexPositionLoc, gl.VERTEX_ATTRIB_ARRAY_SIZE);
+        const previousVertexAttribType = gl.getVertexAttrib(shader.aVertexPositionLoc, gl.VERTEX_ATTRIB_ARRAY_TYPE);
+        const previousVertexAttribNormalized = gl.getVertexAttrib(shader.aVertexPositionLoc, gl.VERTEX_ATTRIB_ARRAY_NORMALIZED);
+        const previousVertexAttribStride = gl.getVertexAttrib(shader.aVertexPositionLoc, gl.VERTEX_ATTRIB_ARRAY_STRIDE);
+        const previousVertexAttribOffset = gl.getVertexAttribOffset(shader.aVertexPositionLoc, gl.VERTEX_ATTRIB_ARRAY_POINTER);
+        const previousTextureAttribSize = gl.getVertexAttrib(shader.aTextureCoordLoc, gl.VERTEX_ATTRIB_ARRAY_SIZE);
+        const previousTextureAttribType = gl.getVertexAttrib(shader.aTextureCoordLoc, gl.VERTEX_ATTRIB_ARRAY_TYPE);
+        const previousTextureAttribNormalized = gl.getVertexAttrib(shader.aTextureCoordLoc, gl.VERTEX_ATTRIB_ARRAY_NORMALIZED);
+        const previousTextureAttribStride = gl.getVertexAttrib(shader.aTextureCoordLoc, gl.VERTEX_ATTRIB_ARRAY_STRIDE);
+        const previousTextureAttribOffset = gl.getVertexAttribOffset(shader.aTextureCoordLoc, gl.VERTEX_ATTRIB_ARRAY_POINTER);
+        // Rendering to the greyscale conversion buffer - bind the framebuffer
+        gl.bindFramebuffer(gl.FRAMEBUFFER, framebuffer);
+        gl.viewport(0, 0, this._framebufferWidth, this._framebufferHeight);
+        // We'll be replacing all the content - clear is a good hint for this on mobile
+        gl.clear(gl.COLOR_BUFFER_BIT);
+        // Set up bindings for vertex attributes
+        gl.bindBuffer(gl.ARRAY_BUFFER, vbo);
+        gl.vertexAttribPointer(shader.aVertexPositionLoc, 2, gl.FLOAT, false, 4 * 4, 0);
+        gl.enableVertexAttribArray(shader.aVertexPositionLoc);
+        gl.vertexAttribPointer(shader.aTextureCoordLoc, 2, gl.FLOAT, false, 4 * 4, 2 * 4);
+        gl.enableVertexAttribArray(shader.aTextureCoordLoc);
+        // Bind the index buffer
+        gl.bindBuffer(gl.ELEMENT_ARRAY_BUFFER, ibo);
+        // Tell WebGL to use our program when drawing
+        gl.useProgram(shader.program);
+        // Specify greyscale width for the correct offsets, and the uv transform
+        gl.uniform1f(shader.uTexWidthLoc, profile_1.profile.dataWidth);
+        gl.uniformMatrix4fv(shader.uUvTransformLoc, false, this._cameraUvTransform);
+        gl.activeTexture(gl.TEXTURE0);
+        // Bind the texture to texture unit 0
+        gl.bindTexture(gl.TEXTURE_2D, texture);
+        // Tell the shader we bound the texture to texture unit 0
+        gl.uniform1i(shader.uSamplerLoc, 0);
+        // Do the drawing...
+        gl.drawElements(gl.TRIANGLES, 6, gl.UNSIGNED_SHORT, 0);
+        gl.vertexAttribPointer(shader.aVertexPositionLoc, previousVertexAttribSize, previousVertexAttribType, previousVertexAttribNormalized, previousVertexAttribStride, previousVertexAttribOffset);
+        gl.vertexAttribPointer(shader.aTextureCoordLoc, previousTextureAttribSize, previousTextureAttribType, previousTextureAttribNormalized, previousTextureAttribStride, previousTextureAttribOffset);
+        gl.bindFramebuffer(gl.FRAMEBUFFER, previousBoundFramebuffer);
+        gl.bindBuffer(gl.ARRAY_BUFFER, previousBoundArrayBuffer);
+        gl.bindBuffer(gl.ELEMENT_ARRAY_BUFFER, previousBoundElementArrayBuffer);
+        gl.useProgram(previousProgram);
+        gl.bindTexture(gl.TEXTURE_2D, previousBoundTexture);
+        gl.activeTexture(previousActiveTexture);
+        gl.pixelStorei(gl.UNPACK_FLIP_Y_WEBGL, previousUnpackFlip);
+        if (reenableBlend)
+            gl.enable(gl.BLEND);
+        if (reenableCullFace)
+            gl.enable(gl.CULL_FACE);
+        if (reenableDepthTest)
+            gl.enable(gl.DEPTH_TEST);
+        if (reenableScissorTest)
+            gl.enable(gl.SCISSOR_TEST);
+    }
+    readFrame(texture, pixels) {
+        let gl = this._gl;
+        let pixelsView = new Uint8Array(pixels);
+        let framebuffer = this._getFramebuffer(gl, profile_1.profile.dataWidth / 4, profile_1.profile.dataHeight);
+        const previousBoundFramebuffer = gl.getParameter(gl.FRAMEBUFFER_BINDING);
+        gl.bindFramebuffer(gl.FRAMEBUFFER, framebuffer);
+        gl.readPixels(0, 0, this._framebufferWidth, this._framebufferHeight, gl.RGBA, gl.UNSIGNED_BYTE, pixelsView);
+        gl.bindFramebuffer(gl.FRAMEBUFFER, previousBoundFramebuffer);
+        return {
+            uvTransform: this._cameraUvTransform,
+            data: pixels,
+            texture,
+            dataWidth: profile_1.profile.dataWidth,
+            dataHeight: profile_1.profile.dataHeight,
+            userFacing: this._computedFrontCameraRotation
+        };
+    }
+    _updateTransforms(rot, fc) {
+        if (rot == this._computedTransformRotation && fc == this._computedFrontCameraRotation)
+            return;
+        this._computedTransformRotation = rot;
+        this._computedFrontCameraRotation = fc;
+        this._cameraUvTransform = this._getCameraUvTransform();
+    }
+    _getCameraUvTransform() {
+        switch (this._computedTransformRotation) {
+            case 270: return new Float32Array([0, 1, 0, 0, -1, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 1]);
+            case 180: return new Float32Array([-1, 0, 0, 0, 0, -1, 0, 0, 0, 0, 1, 0, 1, 1, 0, 1]);
+            case 90: return new Float32Array([0, -1, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 1]);
+        }
+        return new Float32Array([1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1]);
+    }
+    _getFramebuffer(gl, fbWidth, fbHeight) {
+        if (this._framebufferWidth === fbWidth && this._framebufferHeight === fbHeight && this._framebufferId)
+            return this._framebufferId;
+        if (this._framebufferId) {
+            gl.deleteFramebuffer(this._framebufferId);
+            this._framebufferId = null;
+        }
+        if (this._renderTexture) {
+            gl.deleteTexture(this._renderTexture);
+            this._renderTexture = null;
+        }
+        this._framebufferId = gl.createFramebuffer();
+        if (!this._framebufferId)
+            throw new Error("Unable to create framebuffer");
+        gl.bindFramebuffer(gl.FRAMEBUFFER, this._framebufferId);
+        this._renderTexture = gl.createTexture();
+        if (!this._renderTexture)
+            throw new Error("Unable to create render texture");
+        gl.activeTexture(gl.TEXTURE0);
+        gl.bindTexture(gl.TEXTURE_2D, this._renderTexture);
+        gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, fbWidth, fbHeight, 0, gl.RGBA, gl.UNSIGNED_BYTE, null);
+        gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE);
+        gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE);
+        gl.texParameterf(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR);
+        gl.framebufferTexture2D(gl.FRAMEBUFFER, gl.COLOR_ATTACHMENT0, gl.TEXTURE_2D, this._renderTexture, 0);
+        let fbStatus = gl.checkFramebufferStatus(gl.FRAMEBUFFER);
+        if (fbStatus !== gl.FRAMEBUFFER_COMPLETE)
+            throw new Error("Framebuffer not complete: " + fbStatus.toString());
+        this._framebufferWidth = fbWidth;
+        this._framebufferHeight = fbHeight;
+        gl.bindTexture(gl.TEXTURE_2D, null);
+        gl.bindFramebuffer(gl.FRAMEBUFFER, null);
+        return this._framebufferId;
+    }
+    _getVertexBuffer(gl) {
+        if (this._vertexBuffer)
+            return this._vertexBuffer;
+        this._vertexBuffer = gl.createBuffer();
+        if (!this._vertexBuffer)
+            throw new Error("Unable to create vertex buffer");
+        gl.bindBuffer(gl.ARRAY_BUFFER, this._vertexBuffer);
+        let buffer = new Float32Array([-1.0, -1.0, 0.0, 0.0,
+            -1.0, 1.0, 0.0, 1.0,
+            1.0, 1.0, 1.0, 1.0,
+            1.0, -1.0, 1.0, 0.0]);
+        gl.bufferData(gl.ARRAY_BUFFER, buffer, gl.STATIC_DRAW);
+        return this._vertexBuffer;
+    }
+    _getIndexBuffer(gl) {
+        if (this._indexBuffer)
+            return this._indexBuffer;
+        this._indexBuffer = gl.createBuffer();
+        if (!this._indexBuffer)
+            throw new Error("Unable to create index buffer");
+        gl.bindBuffer(gl.ELEMENT_ARRAY_BUFFER, this._indexBuffer);
+        let buffer = new Uint16Array([0, 1, 2, 0, 2, 3]);
+        gl.bufferData(gl.ELEMENT_ARRAY_BUFFER, buffer, gl.STATIC_DRAW);
+        return this._indexBuffer;
+    }
+    _getGreyscaleShader(gl) {
+        if (this._greyscaleShader)
+            return this._greyscaleShader;
+        let prog = gl.createProgram();
+        if (!prog)
+            throw new Error("Unable to create program");
+        let vertexShader = shader_1.compileShader(gl, gl.VERTEX_SHADER, greyscaleVsSource);
+        let fragmentShader = shader_1.compileShader(gl, gl.FRAGMENT_SHADER, greyscaleFsSource);
+        gl.attachShader(prog, vertexShader);
+        gl.attachShader(prog, fragmentShader);
+        shader_1.linkProgram(gl, prog);
+        let uTexWidthLoc = gl.getUniformLocation(prog, "uTexWidth");
+        if (!uTexWidthLoc)
+            throw new Error("Unable to get uniform location uTexWidth");
+        let uUvTransformLoc = gl.getUniformLocation(prog, "uUvTransform");
+        if (!uUvTransformLoc)
+            throw new Error("Unable to get uniform location uUvTransform");
+        let uSamplerLoc = gl.getUniformLocation(prog, "uSampler");
+        if (!uSamplerLoc)
+            throw new Error("Unable to get uniform location uSampler");
+        this._greyscaleShader = {
+            program: prog,
+            aVertexPositionLoc: gl.getAttribLocation(prog, "aVertexPosition"),
+            aTextureCoordLoc: gl.getAttribLocation(prog, "aTextureCoord"),
+            uTexWidthLoc: uTexWidthLoc,
+            uUvTransformLoc: uUvTransformLoc,
+            uSamplerLoc: uSamplerLoc
+        };
+        return this._greyscaleShader;
+    }
+}
+exports.ImageProcessGL = ImageProcessGL;
+let greyscaleVsSource = `
+    attribute vec4 aVertexPosition;
+    attribute vec2 aTextureCoord;
+
+    varying highp vec2 vTextureCoord1;
+    varying highp vec2 vTextureCoord2;
+    varying highp vec2 vTextureCoord3;
+    varying highp vec2 vTextureCoord4;
+
+    uniform float uTexWidth;
+	uniform mat4 uUvTransform;
+
+    void main(void) {
+      highp vec2 offset1 = vec2(1.5 / uTexWidth, 0);
+      highp vec2 offset2 = vec2(0.5 / uTexWidth, 0);
+
+      gl_Position = aVertexPosition;
+      vTextureCoord1 = (uUvTransform * vec4(aTextureCoord - offset1, 0, 1)).xy;
+      vTextureCoord2 = (uUvTransform * vec4(aTextureCoord - offset2, 0, 1)).xy;
+      vTextureCoord3 = (uUvTransform * vec4(aTextureCoord + offset2, 0, 1)).xy;
+      vTextureCoord4 = (uUvTransform * vec4(aTextureCoord + offset1, 0, 1)).xy;
+    }
+`;
+// Fragment shader program
+let greyscaleFsSource = `
+  varying highp vec2 vTextureCoord1;
+  varying highp vec2 vTextureCoord2;
+  varying highp vec2 vTextureCoord3;
+  varying highp vec2 vTextureCoord4;
+
+  uniform sampler2D uSampler;
+
+  const lowp vec3 colorWeights = vec3(77.0 / 256.0, 150.0 / 256.0, 29.0 / 256.0);
+
+  void main(void) {
+    lowp vec4 outpx;
+
+    outpx.r = dot(colorWeights, texture2D(uSampler, vTextureCoord1).xyz);
+    outpx.g = dot(colorWeights, texture2D(uSampler, vTextureCoord2).xyz);
+    outpx.b = dot(colorWeights, texture2D(uSampler, vTextureCoord3).xyz);
+    outpx.a = dot(colorWeights, texture2D(uSampler, vTextureCoord4).xyz);
+
+    gl_FragColor = outpx;
+  }
+`;

package/lib/imagebitmap-camera-source.d.ts

@@ -0,0 +1,36 @@
+import { zappar_camera_source_t, zappar_pipeline_t } from "./gen/zappar";
+import { CameraFrameInfo, Source } from "./source";
+export declare class ImageBitmapCameraSource extends Source {
+    private _impl;
+    private _pipeline;
+    private _deviceId;
+    static USER_DEFAULT_DEVICE_ID: string;
+    static DEFAULT_DEVICE_ID: string;
+    private _currentStream;
+    private _activeDeviceId;
+    private _isPaused;
+    private _isUserFacing;
+    private _hadFrames;
+    private _canvas;
+    private _context;
+    private _lastFrameTime;
+    private _video;
+    private _cameraToDeviceTransform;
+    private _cameraToDeviceTransformUserFacing;
+    private _cameraModel;
+    constructor(_impl: zappar_camera_source_t, _pipeline: zappar_pipeline_t, _deviceId: string);
+    destroy(): void;
+    private _stop;
+    pause(): void;
+    start(): void;
+    private _getConstraints;
+    getFrame(currentlyProcessing: boolean): void;
+    private _getUserMedia;
+    private _syncCamera;
+    private _hasStartedOrientation;
+    private _deviceMotionListener;
+    private _startDeviceOrientation;
+    private _startDeviceMotion;
+    private _stopDeviceMotion;
+    uploadGL(info: CameraFrameInfo): void;
+}

package/lib/imagebitmap-camera-source.js

@@ -0,0 +1,277 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ImageBitmapCameraSource = void 0;
+const profile_1 = require("./profile");
+const pipeline_1 = require("./pipeline");
+const source_1 = require("./source");
+const cameramodel_1 = require("./cameramodel");
+const loglevel_1 = require("./loglevel");
+const camera_source_map_1 = require("./camera-source-map");
+const gl_matrix_1 = require("gl-matrix");
+class ImageBitmapCameraSource extends source_1.Source {
+    constructor(_impl, _pipeline, _deviceId) {
+        super();
+        this._impl = _impl;
+        this._pipeline = _pipeline;
+        this._deviceId = _deviceId;
+        this._currentStream = null;
+        this._activeDeviceId = null;
+        this._isPaused = true;
+        this._isUserFacing = false;
+        this._hadFrames = false;
+        this._canvas = new OffscreenCanvas(640, 480);
+        this._context = this._canvas.getContext("2d");
+        this._lastFrameTime = -1;
+        this._video = document.createElement("video");
+        this._cameraToDeviceTransform = gl_matrix_1.mat4.create();
+        this._cameraToDeviceTransformUserFacing = gl_matrix_1.mat4.create();
+        this._cameraModel = new Float32Array([300, 300, 160, 120, 0, 0]);
+        this._hasStartedOrientation = false;
+        this._deviceMotionListener = (ev) => {
+            let pipeline = pipeline_1.Pipeline.get(this._pipeline);
+            if (!pipeline)
+                return;
+            let timeStamp = (ev.timeStamp !== undefined && ev.timeStamp !== null) ? ev.timeStamp : performance.now();
+            if (ev.accelerationIncludingGravity !== null &&
+                ev.accelerationIncludingGravity.x !== null &&
+                ev.accelerationIncludingGravity.y !== null &&
+                ev.accelerationIncludingGravity.z !== null) {
+                pipeline.motionAccelerometerSubmit(timeStamp, ev.accelerationIncludingGravity.x * profile_1.profile.deviceMotionMutliplier, ev.accelerationIncludingGravity.y * profile_1.profile.deviceMotionMutliplier, ev.accelerationIncludingGravity.z * profile_1.profile.deviceMotionMutliplier);
+            }
+            if (ev.rotationRate !== null &&
+                ev.rotationRate.alpha !== null &&
+                ev.rotationRate.beta !== null &&
+                ev.rotationRate.gamma !== null && !this._hasStartedOrientation) {
+                ev.timeStamp;
+                pipeline.motionRotationRateSubmit(timeStamp, ev.rotationRate.alpha * Math.PI / -180.0, ev.rotationRate.beta * Math.PI / -180.0, ev.rotationRate.gamma * Math.PI / -180.0);
+            }
+            else if (!this._hasStartedOrientation) {
+                this._startDeviceOrientation();
+            }
+        };
+        loglevel_1.zcout("Using ImageBitmap camera source");
+        gl_matrix_1.mat4.fromScaling(this._cameraToDeviceTransformUserFacing, [-1, 1, -1]);
+        this._video.muted = true;
+        this._video.setAttribute('playsinline', '');
+        this._video.setAttribute('webkit-playsinline', '');
+        if (profile_1.profile.videoElementInDOM) {
+            this._video.style.width = "0px";
+            this._video.style.height = "0px";
+            document.body.appendChild(this._video);
+        }
+        this._video.addEventListener("loadedmetadata", () => { this._hadFrames = true; });
+    }
+    destroy() {
+        this.pause();
+        camera_source_map_1.deleteCameraSource(this._impl);
+    }
+    _stop() {
+        if (!this._currentStream)
+            return;
+        let tracks = this._currentStream.getTracks();
+        tracks.forEach(t => t.stop());
+        this._currentStream = null;
+    }
+    pause() {
+        this._isPaused = true;
+        let p = pipeline_1.Pipeline.get(this._pipeline);
+        if (p && p.currentCameraSource === this)
+            p.currentCameraSource = undefined;
+        this._stopDeviceMotion();
+        this._syncCamera();
+    }
+    start() {
+        var _a;
+        let p = pipeline_1.Pipeline.get(this._pipeline);
+        if (p && p.currentCameraSource !== this) {
+            (_a = p.currentCameraSource) === null || _a === void 0 ? void 0 : _a.pause();
+            p.currentCameraSource = this;
+        }
+        this._isPaused = false;
+        this._startDeviceMotion();
+        this._syncCamera();
+    }
+    _getConstraints() {
+        return __awaiter(this, void 0, void 0, function* () {
+            let deviceId;
+            let facingMode;
+            if (this._deviceId !== ImageBitmapCameraSource.DEFAULT_DEVICE_ID &&
+                this._deviceId !== ImageBitmapCameraSource.USER_DEFAULT_DEVICE_ID) {
+                // Custom device
+                deviceId = this._deviceId;
+            }
+            else {
+                facingMode = (this._deviceId === ImageBitmapCameraSource.DEFAULT_DEVICE_ID) ? "environment" : "user";
+            }
+            let constraints = {
+                audio: false,
+                video: {
+                    facingMode: facingMode,
+                    width: profile_1.profile.videoWidth,
+                    height: profile_1.profile.videoHeight,
+                    frameRate: profile_1.profile.requestHighFrameRate ? 60 : undefined,
+                    deviceId: deviceId
+                }
+            };
+            if (deviceId)
+                return constraints;
+            if (!navigator.mediaDevices || !navigator.mediaDevices.enumerateDevices)
+                return constraints;
+            let devices = yield navigator.mediaDevices.enumerateDevices();
+            let hasHadCapabilities = false;
+            devices = devices.filter(val => {
+                // Remove non-video devices
+                if (val.kind !== "videoinput")
+                    return false;
+                // If the media info object contains capabilities, use it to filter to the correct facing cameras
+                if (val.getCapabilities) {
+                    hasHadCapabilities = true;
+                    let capabilities = val.getCapabilities();
+                    if (capabilities && capabilities.facingMode && capabilities.facingMode.indexOf(facingMode === "user" ? "user" : "environment") < 0)
+                        return false;
+                }
+                return true;
+            });
+            // If none of the devices had capability info, or we have no devices left, fall back to the standard constraints
+            if (!hasHadCapabilities || devices.length === 0) {
+                return constraints;
+            }
+            if (typeof constraints.video === "object") {
+                loglevel_1.zcout("choosing device ID", devices[devices.length - 1].deviceId);
+                constraints.video.deviceId = devices[devices.length - 1].deviceId;
+            }
+            return constraints;
+        });
+    }
+    getFrame(currentlyProcessing) {
+        var _a, _b;
+        if (!this._context)
+            return;
+        if (!this._hadFrames)
+            return;
+        if (currentlyProcessing)
+            return;
+        let currentTime = performance.now();
+        if (currentTime < (this._lastFrameTime + 25))
+            return;
+        this._lastFrameTime = currentTime;
+        if (this._canvas.width !== this._video.videoWidth)
+            this._canvas.width = this._video.videoWidth;
+        if (this._canvas.height !== this._video.videoHeight)
+            this._canvas.height = this._video.videoHeight;
+        this._context.drawImage(this._video, 0, 0);
+        const imageBitmap = this._canvas.transferToImageBitmap();
+        let rotation = cameramodel_1.cameraRotationForScreenOrientation(false);
+        let pipeline = pipeline_1.Pipeline.get(this._pipeline);
+        if (!pipeline)
+            return;
+        let focalLength = 300.0 * profile_1.profile.dataWidth / 320.0;
+        this._cameraModel[0] = focalLength;
+        this._cameraModel[1] = focalLength;
+        this._cameraModel[2] = profile_1.profile.dataWidth * 0.5;
+        this._cameraModel[3] = profile_1.profile.dataHeight * 0.5;
+        let token = pipeline.registerToken({
+            dataWidth: this._canvas.width,
+            dataHeight: this._canvas.height,
+            texture: undefined,
+            userFacing: this._isUserFacing,
+            cameraSource: this,
+            cameraModel: this._cameraModel.slice(),
+            cameraToDevice: this._isUserFacing ? this._cameraToDeviceTransformUserFacing : this._cameraToDeviceTransform
+        });
+        (_b = (_a = pipeline_1.Pipeline.get(this._pipeline)) === null || _a === void 0 ? void 0 : _a.sendImageBitmapToWorker) === null || _b === void 0 ? void 0 : _b.call(_a, imageBitmap, rotation, this._isUserFacing, token, this._cameraModel, this._isUserFacing ? this._cameraToDeviceTransformUserFacing : this._cameraToDeviceTransform);
+        return;
+    }
+    _getUserMedia() {
+        return __awaiter(this, void 0, void 0, function* () {
+            let constraints = yield this._getConstraints();
+            if (navigator.mediaDevices && navigator.mediaDevices.getUserMedia)
+                return yield navigator.mediaDevices.getUserMedia(constraints);
+            return yield new Promise((resolve, reject) => {
+                navigator.getUserMedia(constraints, resolve, reject);
+            });
+        });
+    }
+    _syncCamera() {
+        return __awaiter(this, void 0, void 0, function* () {
+            if (this._currentStream && this._isPaused) {
+                this._stop();
+                return;
+            }
+            if (this._currentStream && this._activeDeviceId !== this._deviceId)
+                this._stop();
+            if (!this._isPaused) {
+                this._activeDeviceId = this._deviceId;
+                this._currentStream = yield this._getUserMedia();
+                if (this._isPaused) {
+                    yield this._syncCamera();
+                    return;
+                }
+                this._isUserFacing = false;
+                if (this._currentStream) {
+                    let videoTracks = this._currentStream.getVideoTracks();
+                    if (videoTracks.length > 0) {
+                        this._isUserFacing = videoTracks[0].getSettings().facingMode === "user";
+                    }
+                }
+                this._video.src = "";
+                this._video.loop = false;
+                this._video.muted = true;
+                this._video.srcObject = this._currentStream;
+                this._video.play();
+            }
+        });
+    }
+    _startDeviceOrientation() {
+        if (this._hasStartedOrientation)
+            return;
+        this._hasStartedOrientation = true;
+        window.addEventListener("deviceorientation", (ev) => {
+            let pipeline = pipeline_1.Pipeline.get(this._pipeline);
+            if (!pipeline)
+                return;
+            let timeStamp = (ev.timeStamp !== undefined && ev.timeStamp !== null) ? ev.timeStamp : performance.now();
+            if (ev.alpha === null || ev.beta === null || ev.gamma === null)
+                return;
+            pipeline.motionAttitudeSubmit(timeStamp, ev.alpha, ev.beta, ev.gamma);
+        });
+    }
+    _startDeviceMotion() {
+        window.addEventListener("devicemotion", this._deviceMotionListener, false);
+    }
+    _stopDeviceMotion() {
+        window.removeEventListener("devicemotion", this._deviceMotionListener);
+    }
+    uploadGL(info) {
+        const pipeline = pipeline_1.Pipeline.get(this._pipeline);
+        const gl = pipeline === null || pipeline === void 0 ? void 0 : pipeline.glContext;
+        if (!info ||
+            info.texture ||
+            !info.frame ||
+            !pipeline ||
+            !gl)
+            return;
+        let texture = pipeline.getVideoTexture();
+        if (!texture)
+            return;
+        gl.bindTexture(gl.TEXTURE_2D, texture);
+        gl.pixelStorei(gl.UNPACK_FLIP_Y_WEBGL, false);
+        gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, info.frame);
+        gl.bindTexture(gl.TEXTURE_2D, null);
+        info.texture = texture;
+        info.frame.close();
+        delete info.frame;
+    }
+}
+exports.ImageBitmapCameraSource = ImageBitmapCameraSource;
+ImageBitmapCameraSource.USER_DEFAULT_DEVICE_ID = "Simulated User Default Device ID: a908df7f-5661-4d20-b227-a1c15d2fdb4b";
+ImageBitmapCameraSource.DEFAULT_DEVICE_ID = "Simulated Default Device ID: a908df7f-5661-4d20-b227-a1c15d2fdb4b";

package/lib/imagetracker.d.ts

@@ -0,0 +1,28 @@
+import { zappar_image_tracker_t, zappar_pipeline_t } from "./gen/zappar";
+import { zappar_cwrap } from "./gen/zappar-native";
+interface PreviewInfo {
+    compressed: Uint8Array;
+    mimeType: string;
+}
+interface ParsedTargetInfo {
+    preview?: PreviewInfo;
+    physicalScaleFactor: number;
+    topRadius: number;
+    bottomRadius: number;
+    sideLength: number;
+}
+export declare class ImageTracker {
+    private _client;
+    private _impl;
+    private _targets;
+    static create(pipeline: zappar_pipeline_t, client: zappar_cwrap): zappar_image_tracker_t;
+    static get(p: zappar_image_tracker_t): ImageTracker | undefined;
+    private constructor();
+    destroy(): void;
+    loadFromMemory(data: ArrayBuffer): void;
+    targetCount(): number;
+    getTargetInfo(i: number): ParsedTargetInfo;
+    private _parseOdle;
+    getDecodedPreview(i: number): HTMLImageElement;
+}
+export {};