@zappar/zappar-cv 2.0.0-beta.5 → 2.0.0-beta.9

package/CHANGELOG.md

@@ -1,5 +1,11 @@
 # Changelog
 
+## [0.3.17] - 2022-07-21
+
+### Fixed
+
+- Preserve `TextureAttribDivisor`, `VertexAttribDivisor` and `STENCIL_TEST` WebGL state.
+
 ## [0.3.16] - 2021-02-07
 
 ### Fixed

package/README.md

@@ -18,8 +18,8 @@ npm i @zappar/zappar-cv
 
 You can use our CDN from within your HTML:
 ```
-<script src="https://libs.zappar.com/zappar-cv/2.0.0-beta.5/zappar-cv.js"></script>
+<script src="https://libs.zappar.com/zappar-cv/2.0.0-beta.9/zappar-cv.js"></script>
 ```
 
 Or you can download and host our standalone JavaScript bundle:
-[https://libs.zappar.com/zappar-cv/2.0.0-beta.5/zappar-cv.zip](https://libs.zappar.com/zappar-cv/2.0.0-beta.5/zappar-cv.zip)
+[https://libs.zappar.com/zappar-cv/2.0.0-beta.9/zappar-cv.zip](https://libs.zappar.com/zappar-cv/2.0.0-beta.9/zappar-cv.zip)

package/lib/additional.d.ts

@@ -6,6 +6,7 @@ export interface Additional {
     pipeline_gl_context_set(pipeline: zappar_pipeline_t, gl: WebGLRenderingContext, texturePool?: WebGLTexture[]): void;
     pipeline_gl_context_lost(pipeline: zappar_pipeline_t): void;
     pipeline_draw_face(pipeline: zappar_pipeline_t, projectionMatrix: Float32Array, cameraMatrix: Float32Array, targetMatrix: Float32Array, o: zappar_face_mesh_t): void;
+    pipeline_draw_image_target_preview(pipeline: zappar_pipeline_t, projectionMatrix: Float32Array, cameraMatrix: Float32Array, targetMatrix: Float32Array, o: zappar_image_tracker_t, indx: number): void;
     pipeline_draw_face_project(pipeline: zappar_pipeline_t, drawwMatrix: Float32Array, vertices: Float32Array, uvMatrix: Float32Array, uvs: Float32Array, indices: Uint16Array, texture: WebGLTexture): void;
     draw_plane(gl: WebGLRenderingContext, projectionMatrix: Float32Array, cameraMatrix: Float32Array, targetMatrix: Float32Array, texture: string): void;
     image_tracker_target_image(o: zappar_image_tracker_t, indx: number): HTMLImageElement | undefined;

package/lib/drawplane.js

@@ -63,6 +63,7 @@ export function drawPlane(gl, projectionMatrix, cameraMatrix, targetMatrix, text
     gl.enableVertexAttribArray(shader.attr_textureCoord);
     gl.drawArrays(gl.TRIANGLE_STRIP, 0, 4);
     gl.disableVertexAttribArray(shader.attr_position);
+    gl.disableVertexAttribArray(shader.attr_textureCoord);
     gl.bindBuffer(gl.ARRAY_BUFFER, null);
 }
 function generateLocalMatrix() {

package/lib/drawpreviewmesh.d.ts

@@ -0,0 +1,16 @@
+import { ImageTracker } from "./imagetracker";
+export declare class PreviewMeshDraw {
+    private _gl;
+    private _vbo;
+    private _uvbo;
+    private _ibo;
+    private _lastIndices;
+    private _shader;
+    constructor(_gl: WebGLRenderingContext);
+    dispose(): void;
+    private _generateIBO;
+    private _generateVBO;
+    private _generateUVBO;
+    draw(matrix: Float32Array, tracker: ImageTracker, indx: number): void;
+    private _getShader;
+}

package/lib/drawpreviewmesh.js

@@ -0,0 +1,186 @@
+import { compileShader, linkProgram } from "./shader";
+export class PreviewMeshDraw {
+    constructor(_gl) {
+        this._gl = _gl;
+    }
+    dispose() {
+        if (this._vbo)
+            this._gl.deleteBuffer(this._vbo);
+        if (this._uvbo)
+            this._gl.deleteBuffer(this._uvbo);
+        if (this._ibo)
+            this._gl.deleteBuffer(this._ibo);
+        if (this._shader)
+            this._gl.deleteProgram(this._shader.prog);
+        this._vbo = undefined;
+        this._uvbo = undefined;
+        this._ibo = undefined;
+        this._shader = undefined;
+    }
+    _generateIBO(indices, gl) {
+        if (this._ibo && this._lastIndices === indices)
+            return this._ibo;
+        this._lastIndices = indices;
+        if (!this._ibo)
+            this._ibo = gl.createBuffer();
+        if (!this._ibo)
+            throw new Error("Unable to create buffer object");
+        gl.bindBuffer(gl.ELEMENT_ARRAY_BUFFER, this._ibo);
+        gl.bufferData(gl.ELEMENT_ARRAY_BUFFER, indices, gl.STATIC_DRAW);
+        gl.bindBuffer(gl.ELEMENT_ARRAY_BUFFER, null);
+        return this._ibo;
+    }
+    _generateVBO(face, gl) {
+        if (!this._vbo)
+            this._vbo = gl.createBuffer();
+        if (!this._vbo)
+            throw new Error("Unable to create buffer object");
+        gl.bindBuffer(gl.ARRAY_BUFFER, this._vbo);
+        gl.bufferData(gl.ARRAY_BUFFER, face, gl.STATIC_DRAW);
+        gl.bindBuffer(gl.ARRAY_BUFFER, null);
+        return this._vbo;
+    }
+    _generateUVBO(face, gl) {
+        if (!this._uvbo)
+            this._uvbo = gl.createBuffer();
+        if (!this._uvbo)
+            throw new Error("Unable to create buffer object");
+        gl.bindBuffer(gl.ARRAY_BUFFER, this._uvbo);
+        gl.bufferData(gl.ARRAY_BUFFER, face, gl.STATIC_DRAW);
+        gl.bindBuffer(gl.ARRAY_BUFFER, null);
+        return this._uvbo;
+    }
+    draw(matrix, tracker, indx) {
+        var _a;
+        const o = tracker.getPreviewMesh(indx);
+        const image = (_a = tracker.getTargetInfo(indx).preview) === null || _a === void 0 ? void 0 : _a.image;
+        if (!o || !image)
+            return;
+        if (!image.complete)
+            return;
+        let gl = this._gl;
+        let shader = this._getShader(gl);
+        let v = this._generateVBO(o.vertices, gl);
+        let n = this._generateUVBO(o.uvs, gl);
+        let i = this._generateIBO(o.indices, gl);
+        gl.enable(gl.DEPTH_TEST);
+        gl.enable(gl.CULL_FACE);
+        gl.useProgram(shader.prog);
+        gl.uniformMatrix4fv(shader.unif_matrix, false, matrix);
+        gl.activeTexture(gl.TEXTURE0);
+        gl.bindTexture(gl.TEXTURE_2D, loadTexture(gl, image));
+        gl.uniform1i(shader.unif_skinSampler, 0);
+        gl.bindBuffer(gl.ARRAY_BUFFER, v);
+        gl.vertexAttribPointer(shader.attr_position, 3, gl.FLOAT, false, 0, 0);
+        gl.enableVertexAttribArray(shader.attr_position);
+        gl.bindBuffer(gl.ARRAY_BUFFER, n);
+        gl.vertexAttribPointer(shader.attr_textureCoord, 2, gl.FLOAT, false, 0, 0);
+        gl.enableVertexAttribArray(shader.attr_textureCoord);
+        gl.bindBuffer(gl.ELEMENT_ARRAY_BUFFER, i);
+        gl.drawElements(gl.TRIANGLES, o.indices.length, gl.UNSIGNED_SHORT, 0);
+        gl.disableVertexAttribArray(shader.attr_position);
+        gl.disableVertexAttribArray(shader.attr_textureCoord);
+        gl.bindBuffer(gl.ARRAY_BUFFER, null);
+        gl.bindBuffer(gl.ELEMENT_ARRAY_BUFFER, null);
+    }
+    _getShader(gl) {
+        if (this._shader)
+            return this._shader;
+        let prog = gl.createProgram();
+        if (!prog)
+            throw new Error("Unable to create program");
+        let vertexShader = compileShader(gl, gl.VERTEX_SHADER, vertexShaderSrc);
+        let fragmentShader = compileShader(gl, gl.FRAGMENT_SHADER, fragmentShaderSrc);
+        gl.attachShader(prog, vertexShader);
+        gl.attachShader(prog, fragmentShader);
+        linkProgram(gl, prog);
+        let unif_matrix = gl.getUniformLocation(prog, "matrix");
+        if (!unif_matrix)
+            throw new Error("Unable to get uniform location mattrix");
+        let unif_skinSampler = gl.getUniformLocation(prog, "skinSampler");
+        if (!unif_skinSampler)
+            throw new Error("Unable to get uniform location skinSampler");
+        this._shader = {
+            prog,
+            unif_matrix,
+            unif_skinSampler,
+            attr_position: gl.getAttribLocation(prog, "position"),
+            attr_textureCoord: gl.getAttribLocation(prog, "textureCoord")
+        };
+        return this._shader;
+    }
+}
+const texturesByElement = new Map();
+function loadTexture(gl, elm) {
+    let existing = texturesByElement.get(elm);
+    if (existing)
+        return existing;
+    existing = gl.createTexture() || undefined;
+    if (!existing)
+        throw new Error("Unable to create texture");
+    texturesByElement.set(elm, existing);
+    gl.bindTexture(gl.TEXTURE_2D, existing);
+    gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.LINEAR);
+    gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR);
+    gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE);
+    gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE);
+    // Because images have to be download over the internet
+    // they might take a moment until they are ready.
+    // Until then put a single pixel in the texture so we can
+    // use it immediately. When the image has finished downloading
+    // we'll update the texture with the contents of the image.
+    const level = 0;
+    const internalFormat = gl.RGBA;
+    const srcFormat = gl.RGBA;
+    const srcType = gl.UNSIGNED_BYTE;
+    gl.pixelStorei(gl.UNPACK_FLIP_Y_WEBGL, true);
+    gl.texImage2D(gl.TEXTURE_2D, level, internalFormat, srcFormat, srcType, elm);
+    elm.addEventListener("load", () => {
+        if (!existing)
+            return;
+        gl.bindTexture(gl.TEXTURE_2D, existing);
+        const level = 0;
+        const internalFormat = gl.RGBA;
+        const srcFormat = gl.RGBA;
+        const srcType = gl.UNSIGNED_BYTE;
+        gl.pixelStorei(gl.UNPACK_FLIP_Y_WEBGL, true);
+        gl.texImage2D(gl.TEXTURE_2D, level, internalFormat, srcFormat, srcType, elm);
+    });
+    return existing;
+}
+let vertexShaderSrc = `
+#ifndef GL_ES
+#define highp
+#define mediump
+#define lowp
+#endif
+
+uniform mat4 matrix;
+attribute vec4 position;
+attribute vec2 textureCoord;
+
+varying highp vec2 vTextureCoord;
+
+void main()
+{
+    gl_Position = matrix * position;
+    vTextureCoord = textureCoord;
+}`;
+let fragmentShaderSrc = `
+#define highp mediump
+#ifdef GL_ES
+    // define default precision for float, vec, mat.
+    precision highp float;
+#else
+#define highp
+#define mediump
+#define lowp
+#endif
+
+varying highp vec2 vTextureCoord;
+uniform sampler2D skinSampler;
+
+void main()
+{
+    gl_FragColor = texture2D(skinSampler, vTextureCoord);
+}`;

package/lib/gen/zappar.d.ts

@@ -39,6 +39,10 @@ export declare type zappar_instant_world_tracker_t = number & {
 export interface zappar {
     loaded(): boolean;
     camera_default_device_id(userFacing: boolean): string;
+    camera_count(): number;
+    camera_id(indx: number): string;
+    camera_name(indx: number): string;
+    camera_user_facing(indx: number): boolean;
     projection_matrix_from_camera_model(model: Float32Array, renderWidth: number, renderHeight: number): Float32Array;
     projection_matrix_from_camera_model_ext(model: Float32Array, renderWidth: number, renderHeight: number, zNear: number, zFar: number): Float32Array;
     log_level(): log_level_t;
@@ -105,6 +109,14 @@ export interface zappar {
     image_tracker_target_preview_compressed(o: zappar_image_tracker_t, indx: number): Uint8Array;
     image_tracker_target_preview_compressed_size(o: zappar_image_tracker_t, indx: number): number;
     image_tracker_target_preview_compressed_mimetype(o: zappar_image_tracker_t, indx: number): string;
+    image_tracker_target_preview_mesh_indices(o: zappar_image_tracker_t, indx: number): Uint16Array;
+    image_tracker_target_preview_mesh_indices_size(o: zappar_image_tracker_t, indx: number): number;
+    image_tracker_target_preview_mesh_vertices(o: zappar_image_tracker_t, indx: number): Float32Array;
+    image_tracker_target_preview_mesh_vertices_size(o: zappar_image_tracker_t, indx: number): number;
+    image_tracker_target_preview_mesh_normals(o: zappar_image_tracker_t, indx: number): Float32Array;
+    image_tracker_target_preview_mesh_normals_size(o: zappar_image_tracker_t, indx: number): number;
+    image_tracker_target_preview_mesh_uvs(o: zappar_image_tracker_t, indx: number): Float32Array;
+    image_tracker_target_preview_mesh_uvs_size(o: zappar_image_tracker_t, indx: number): number;
     image_tracker_enabled(o: zappar_image_tracker_t): boolean;
     image_tracker_enabled_set(o: zappar_image_tracker_t, enabled: boolean): void;
     image_tracker_anchor_count(o: zappar_image_tracker_t): number;

package/lib/image-process-gl.d.ts

@@ -23,7 +23,9 @@ export declare class ImageProcessGL {
     private _vertexBuffer;
     private _indexBuffer;
     private _greyscaleShader;
-    constructor(_gl: WebGLRenderingContext);
+    private _isWebGL2;
+    private _instancedArraysExtension;
+    constructor(_gl: WebGL2RenderingContext | WebGLRenderingContext);
     resetGLContext(): void;
     destroy(): void;
     uploadFrame(texture: WebGLTexture, img: HTMLVideoElement | HTMLImageElement | ImageBitmap, rotation: number, fc: boolean): void;

package/lib/image-process-gl.js

@@ -17,6 +17,11 @@ export class ImageProcessGL {
         this._framebufferHeight = 0;
         this._framebufferId = null;
         this._renderTexture = null;
+        this._isWebGL2 = false;
+        this._isWebGL2 = _gl.getParameter(_gl.VERSION).indexOf("WebGL 2") >= 0;
+        if (!this._isWebGL2) {
+            this._instancedArraysExtension = this._gl.getExtension("ANGLE_instanced_arrays");
+        }
     }
     resetGLContext() {
         this._framebufferId = null;
@@ -36,6 +41,7 @@ export class ImageProcessGL {
         const reenableDepthTest = gl.isEnabled(gl.DEPTH_TEST);
         const reenableBlend = gl.isEnabled(gl.BLEND);
         const reenableCullFace = gl.isEnabled(gl.CULL_FACE);
+        const reenableStencilTest = gl.isEnabled(gl.STENCIL_TEST);
         const previousActiveTexture = gl.getParameter(gl.ACTIVE_TEXTURE);
         const previousUnpackFlip = gl.getParameter(gl.UNPACK_FLIP_Y_WEBGL);
         const previousProgram = gl.getParameter(gl.CURRENT_PROGRAM);
@@ -48,6 +54,7 @@ export class ImageProcessGL {
         gl.disable(gl.DEPTH_TEST);
         gl.disable(gl.BLEND);
         gl.disable(gl.CULL_FACE);
+        gl.disable(gl.STENCIL_TEST);
         gl.pixelStorei(gl.UNPACK_FLIP_Y_WEBGL, false);
         gl.bindTexture(gl.TEXTURE_2D, texture);
         const level = 0;
@@ -84,8 +91,22 @@ export class ImageProcessGL {
         const previousTextureAttribNormalized = gl.getVertexAttrib(shader.aTextureCoordLoc, gl.VERTEX_ATTRIB_ARRAY_NORMALIZED);
         const previousTextureAttribStride = gl.getVertexAttrib(shader.aTextureCoordLoc, gl.VERTEX_ATTRIB_ARRAY_STRIDE);
         const previousTextureAttribOffset = gl.getVertexAttribOffset(shader.aTextureCoordLoc, gl.VERTEX_ATTRIB_ARRAY_POINTER);
-        const previousTextureAttribEnabled = gl.getVertexAttrib(shader.aTextureCoordLoc, gl.VERTEX_ATTRIB_ARRAY_ENABLED);
         const previousTextureAttribBufferBinding = gl.getVertexAttrib(shader.aTextureCoordLoc, gl.VERTEX_ATTRIB_ARRAY_BUFFER_BINDING);
+        const previousTextureAttribEnabled = gl.getVertexAttrib(shader.aTextureCoordLoc, gl.VERTEX_ATTRIB_ARRAY_ENABLED);
+        let previousVertexAttribDivisor = 0;
+        let previousTextureAttribDivisor = 0;
+        if (this._isWebGL2) {
+            previousVertexAttribDivisor = gl.getVertexAttrib(shader.aVertexPositionLoc, gl.VERTEX_ATTRIB_ARRAY_DIVISOR);
+            previousTextureAttribDivisor = gl.getVertexAttrib(shader.aTextureCoordLoc, gl.VERTEX_ATTRIB_ARRAY_DIVISOR);
+            gl.vertexAttribDivisor(shader.aVertexPositionLoc, 0);
+            gl.vertexAttribDivisor(shader.aTextureCoordLoc, 0);
+        }
+        else if (this._instancedArraysExtension) {
+            previousVertexAttribDivisor = gl.getVertexAttrib(shader.aVertexPositionLoc, this._instancedArraysExtension.VERTEX_ATTRIB_ARRAY_DIVISOR_ANGLE);
+            previousTextureAttribDivisor = gl.getVertexAttrib(shader.aTextureCoordLoc, this._instancedArraysExtension.VERTEX_ATTRIB_ARRAY_DIVISOR_ANGLE);
+            this._instancedArraysExtension.vertexAttribDivisorANGLE(shader.aVertexPositionLoc, 0);
+            this._instancedArraysExtension.vertexAttribDivisorANGLE(shader.aTextureCoordLoc, 0);
+        }
         // Rendering to the greyscale conversion buffer - bind the framebuffer
         gl.bindFramebuffer(gl.FRAMEBUFFER, framebuffer);
         gl.viewport(0, 0, this._framebufferWidth, this._framebufferHeight);
@@ -121,6 +142,14 @@ export class ImageProcessGL {
             gl.disableVertexAttribArray(shader.aVertexPositionLoc);
         if (!previousTextureAttribEnabled)
             gl.disableVertexAttribArray(shader.aTextureCoordLoc);
+        if (this._isWebGL2) {
+            gl.vertexAttribDivisor(shader.aVertexPositionLoc, previousVertexAttribDivisor);
+            gl.vertexAttribDivisor(shader.aTextureCoordLoc, previousTextureAttribDivisor);
+        }
+        else if (this._instancedArraysExtension) {
+            this._instancedArraysExtension.vertexAttribDivisorANGLE(shader.aVertexPositionLoc, previousVertexAttribDivisor);
+            this._instancedArraysExtension.vertexAttribDivisorANGLE(shader.aTextureCoordLoc, previousTextureAttribDivisor);
+        }
         gl.bindFramebuffer(gl.FRAMEBUFFER, previousBoundFramebuffer);
         gl.useProgram(previousProgram);
         gl.bindTexture(gl.TEXTURE_2D, previousBoundTexture);
@@ -135,6 +164,8 @@ export class ImageProcessGL {
             gl.enable(gl.DEPTH_TEST);
         if (reenableScissorTest)
             gl.enable(gl.SCISSOR_TEST);
+        if (reenableStencilTest)
+            gl.enable(gl.STENCIL_TEST);
     }
     readFrame(texture, pixels) {
         let gl = this._gl;

package/lib/imagetracker-previewmesh.js

@@ -12,11 +12,12 @@ function planar(info) {
     const aspectRatio = info.trainedWidth / info.trainedHeight;
     if (isNaN(aspectRatio))
         return defaultMesh();
+    const scaling = info.physicalScaleFactor > 0 ? info.physicalScaleFactor : 1;
     const vertices = new Float32Array([
-        -1.0 * aspectRatio, -1, 0,
-        -1.0 * aspectRatio, 1, 0,
-        aspectRatio, 1, 0,
-        aspectRatio, -1, 0
+        -1.0 * aspectRatio * scaling, -1 * scaling, 0,
+        -1.0 * aspectRatio * scaling, scaling, 0,
+        aspectRatio * scaling, scaling, 0,
+        aspectRatio * scaling, -1 * scaling, 0
     ]);
     const indices = new Uint16Array([0, 2, 1, 0, 3, 2]);
     const uvs = new Float32Array([
@@ -42,7 +43,8 @@ function defaultMesh() {
     };
 }
 function cylindrical(info) {
-    return generalConical(info, 2, false, 0, 0, 0, vec2.create(), info.trainedWidth / info.trainedHeight);
+    const wrap_amount = 2 * info.trainedWidth / (info.trainedHeight * info.topRadius);
+    return generalConical(info, 2, false, 0, 0, 0, vec2.create(), info.trainedWidth / info.trainedHeight, wrap_amount, info.physicalScaleFactor);
 }
 function conical(info) {
     const radius_diff = info.topRadius - info.bottomRadius;
@@ -131,26 +133,31 @@ function conical(info) {
     vec2.subtract(bottom_from_center, bottom_corner, rotation_center);
     const top_2d_radius = vec2.length(top_from_center);
     const bottom_2d_radius = vec2.length(bottom_from_center);
+    let max_angle = 2 * Math.abs(Math.atan(top_from_center[0] / top_from_center[1]));
+    if (wide)
+        max_angle = (2 * Math.PI) - max_angle;
+    let theta_3d = (top_2d_radius * max_angle) / info.topRadius;
     let theta = Math.abs(Math.atan(top_from_center[0] / top_from_center[1]));
     if (wide)
         theta = Math.PI - theta;
-    return generalConical(info, height_3d, flip, theta, bottom_2d_radius, top_2d_radius, rotation_center, aspect_ratio);
+    return generalConical(info, height_3d, flip, theta, bottom_2d_radius, top_2d_radius, rotation_center, aspect_ratio, theta_3d, info.physicalScaleFactor);
 }
-function generalConical(info, height_3d, flip, theta, bottom_2d_radius, top_2d_radius, rotation_center, aspect_ratio) {
+function generalConical(info, height_3d, flip, theta, bottom_2d_radius, top_2d_radius, rotation_center, aspect_ratio, wrap_amount, psf) {
     if (isNaN(aspect_ratio))
         aspect_ratio = 1;
     const vertices = [];
     const uvs = [];
-    const scale_factor = 2.0 / height_3d;
+    const physical_scale = psf > 0 ? psf : 1;
+    const scale_factor = physical_scale * 2.0 / height_3d;
     const subdivisons = 64;
     for (let s = 0; s <= subdivisons; ++s) {
-        const angle = s * 2.0 * Math.PI / subdivisons;
+        const angle = (s * wrap_amount / subdivisons) + (((2 * Math.PI) - wrap_amount) / 2);
         const bx = info.bottomRadius * Math.sin(angle) * scale_factor;
         const bz = info.bottomRadius * Math.cos(angle) * scale_factor;
         const tx = info.topRadius * Math.sin(angle) * scale_factor;
         const tz = info.topRadius * Math.cos(angle) * scale_factor;
-        const btm = -1;
-        const top = 1;
+        const btm = -1 * physical_scale;
+        const top = physical_scale;
         if (flip) {
             vertices.push(bx, btm, bz);
             vertices.push(tx, top, tz);

package/lib/index.d.ts

@@ -3,4 +3,4 @@ import { Additional } from "./additional";
 import { Options } from "./options";
 export declare type Zappar = zappar & Additional;
 export declare function initialize(opts?: Options): Zappar;
-export { zappar_image_tracker_t, zappar_instant_world_tracker_t, zappar_barcode_finder_t, zappar_face_tracker_t, zappar_face_landmark_t, barcode_format_t, face_landmark_name_t, instant_world_tracker_transform_orientation_t, log_level_t, zappar_face_mesh_t, zappar_pipeline_t, zappar_camera_source_t, zappar_sequence_source_t } from "./gen/zappar";
+export { zappar_image_tracker_t, zappar_instant_world_tracker_t, zappar_barcode_finder_t, zappar_face_tracker_t, zappar_face_landmark_t, barcode_format_t, face_landmark_name_t, instant_world_tracker_transform_orientation_t, log_level_t, zappar_face_mesh_t, zappar_pipeline_t, zappar_camera_source_t, zappar_sequence_source_t, image_target_type_t } from "./gen/zappar";

package/lib/index.js

@@ -4,4 +4,4 @@ export function initialize(opts) {
     console.log(`Zappar CV v${VERSION}`);
     return nativeInitialize(opts);
 }
-export { barcode_format_t, face_landmark_name_t, instant_world_tracker_transform_orientation_t, log_level_t } from "./gen/zappar";
+export { barcode_format_t, face_landmark_name_t, instant_world_tracker_transform_orientation_t, log_level_t, image_target_type_t } from "./gen/zappar";

package/lib/native.js

@@ -118,7 +118,7 @@ export function initialize(opts) {
             }
         }
     });
-    client = Object.assign(Object.assign({}, c.impl), { loaded: () => loaded, camera_default_device_id: userFacing => userFacing ? CameraSource.USER_DEFAULT_DEVICE_ID : CameraSource.DEFAULT_DEVICE_ID, camera_source_create: (p, deviceId) => createCameraSource(p, deviceId), camera_source_destroy: cam => { var _a; return (_a = getCameraSource(cam)) === null || _a === void 0 ? void 0 : _a.destroy(); }, camera_source_pause: cam => { var _a; return (_a = getCameraSource(cam)) === null || _a === void 0 ? void 0 : _a.pause(); }, camera_source_start: cam => { var _a; return (_a = getCameraSource(cam)) === null || _a === void 0 ? void 0 : _a.start(); }, pipeline_create: () => Pipeline.create(c.impl, messageManager), pipeline_frame_update: (p) => { var _a; return (_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.frameUpdate(c); }, pipeline_camera_frame_draw_gl: (pipeline, screenWidth, screenHeight, mirror) => {
+    client = Object.assign(Object.assign({}, c.impl), { loaded: () => loaded, camera_default_device_id: userFacing => userFacing ? CameraSource.USER_DEFAULT_DEVICE_ID : CameraSource.DEFAULT_DEVICE_ID, camera_source_create: (p, deviceId) => createCameraSource(p, deviceId), camera_source_destroy: cam => { var _a; return (_a = getCameraSource(cam)) === null || _a === void 0 ? void 0 : _a.destroy(); }, camera_source_pause: cam => { var _a; return (_a = getCameraSource(cam)) === null || _a === void 0 ? void 0 : _a.pause(); }, camera_source_start: cam => { var _a; return (_a = getCameraSource(cam)) === null || _a === void 0 ? void 0 : _a.start(); }, camera_count: () => 2, camera_id: indx => indx === 0 ? CameraSource.DEFAULT_DEVICE_ID : CameraSource.USER_DEFAULT_DEVICE_ID, camera_name: indx => indx === 0 ? "Rear-facing Camera" : "User-facing Camera", camera_user_facing: indx => indx !== 0, pipeline_create: () => Pipeline.create(c.impl, messageManager), pipeline_frame_update: (p) => { var _a; return (_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.frameUpdate(c); }, pipeline_camera_frame_draw_gl: (pipeline, screenWidth, screenHeight, mirror) => {
             var _a;
             (_a = Pipeline.get(pipeline)) === null || _a === void 0 ? void 0 : _a.cameraFrameDrawGL(screenWidth, screenHeight, mirror);
         }, draw_plane: (gl, projectionMatrix, cameraMatrix, targetMatrix, texture) => {
@@ -134,6 +134,14 @@ export function initialize(opts) {
         }, pipeline_draw_face_project: (p, matrix, vertices, uvMatrix, uvs, indices, texture) => {
             var _a;
             (_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.drawFaceProject(matrix, vertices, uvMatrix, uvs, indices, texture);
+        }, pipeline_draw_image_target_preview: (p, projectionMatrix, cameraMatrix, targetMatrix, o, indx) => {
+            var _a;
+            let obj = ImageTracker.get(o);
+            if (!obj) {
+                zcwarn("image tracker not found");
+                return;
+            }
+            (_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.drawImageTargetPreview(projectionMatrix, cameraMatrix, targetMatrix, indx, obj);
         }, projection_matrix_from_camera_model: projectionMatrix, projection_matrix_from_camera_model_ext: projectionMatrix, pipeline_process_gl: p => { var _a; return (_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.processGL(); }, pipeline_gl_context_set: (p, gl, texturePool) => { var _a; return (_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.glContextSet(gl, texturePool); }, pipeline_gl_context_lost: (p) => { var _a; return (_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.glContextLost(); }, pipeline_camera_frame_upload_gl: p => { var _a; return (_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.uploadGL(); }, pipeline_camera_frame_texture_gl: p => { var _a; return (_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.cameraFrameTexture(); }, pipeline_camera_frame_texture_matrix: (p, sw, sh, mirror) => { var _a; return ((_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.cameraFrameTextureMatrix(sw, sh, mirror)) || mat4.create(); }, pipeline_camera_frame_user_facing: p => { var _a; return ((_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.cameraFrameUserFacing()) || false; }, pipeline_camera_pose_default: () => mat4.create(), pipeline_camera_pose_with_attitude: (p, mirror) => { var _a; return ((_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.cameraPoseWithAttitude(mirror)) || mat4.create(); }, pipeline_camera_pose_with_origin: (p, o) => { let res = mat4.create(); mat4.invert(res, o); return res; }, pipeline_sequence_record_clear: p => { var _a; return (_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.sequenceRecordClear(); }, pipeline_sequence_record_start: (p, expectedFrames) => { var _a; return (_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.sequenceRecordStart(expectedFrames); }, pipeline_sequence_record_stop: p => { var _a; return (_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.sequenceRecordStop(); }, pipeline_sequence_record_device_attitude_matrices_set: (p, v) => { var _a; return (_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.sequenceRecordDeviceAttitudeMatrices(v); }, pipeline_sequence_record_data: p => { var _a; return ((_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.sequenceRecordData()) || new Uint8Array(0); }, pipeline_sequence_record_data_size: p => { var _a; return ((_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.sequenceRecordData().byteLength) || 0; }, instant_world_tracker_anchor_pose_camera_relative: (o, mirror) => {
             let res = applyScreenCounterRotation(undefined, c.impl.instant_world_tracker_anchor_pose_raw(o));
             if (mirror) {
@@ -265,6 +273,62 @@ export function initialize(opts) {
             }
             mat4.multiply(res, cameraPose, res);
             return res;
+        }, image_tracker_target_preview_mesh_indices: (o, indx) => {
+            let obj = ImageTracker.get(o);
+            if (!obj) {
+                zcwarn("attempting to call image_tracker_target_preview_mesh_indices on a destroyed zappar_image_tracker_t");
+                return new Uint16Array();
+            }
+            return obj.getPreviewMesh(indx).indices;
+        }, image_tracker_target_preview_mesh_vertices: (o, indx) => {
+            let obj = ImageTracker.get(o);
+            if (!obj) {
+                zcwarn("attempting to call image_tracker_target_preview_mesh_vertices on a destroyed zappar_image_tracker_t");
+                return new Float32Array();
+            }
+            return obj.getPreviewMesh(indx).vertices;
+        }, image_tracker_target_preview_mesh_uvs: (o, indx) => {
+            let obj = ImageTracker.get(o);
+            if (!obj) {
+                zcwarn("attempting to call image_tracker_target_preview_mesh_uvs on a destroyed zappar_image_tracker_t");
+                return new Float32Array();
+            }
+            return obj.getPreviewMesh(indx).uvs;
+        }, image_tracker_target_preview_mesh_normals: (o, indx) => {
+            let obj = ImageTracker.get(o);
+            if (!obj) {
+                zcwarn("attempting to call image_tracker_target_preview_mesh_normals on a destroyed zappar_image_tracker_t");
+                return new Float32Array();
+            }
+            return obj.getPreviewMesh(indx).normals;
+        }, image_tracker_target_preview_mesh_indices_size: (o, indx) => {
+            let obj = ImageTracker.get(o);
+            if (!obj) {
+                zcwarn("attempting to call image_tracker_target_preview_mesh_indices_size on a destroyed zappar_image_tracker_t");
+                return 0;
+            }
+            return obj.getPreviewMesh(indx).indices.length;
+        }, image_tracker_target_preview_mesh_vertices_size: (o, indx) => {
+            let obj = ImageTracker.get(o);
+            if (!obj) {
+                zcwarn("attempting to call image_tracker_target_preview_mesh_vertices_size on a destroyed zappar_image_tracker_t");
+                return 0;
+            }
+            return obj.getPreviewMesh(indx).vertices.length;
+        }, image_tracker_target_preview_mesh_uvs_size: (o, indx) => {
+            let obj = ImageTracker.get(o);
+            if (!obj) {
+                zcwarn("attempting to call image_tracker_target_preview_mesh_uvs_size on a destroyed zappar_image_tracker_t");
+                return 0;
+            }
+            return obj.getPreviewMesh(indx).uvs.length;
+        }, image_tracker_target_preview_mesh_normals_size: (o, indx) => {
+            let obj = ImageTracker.get(o);
+            if (!obj) {
+                zcwarn("attempting to call image_tracker_target_preview_mesh_normals_size on a destroyed zappar_image_tracker_t");
+                return 0;
+            }
+            return obj.getPreviewMesh(indx).normals.length;
         }, face_tracker_anchor_pose_camera_relative: (o, indx, mirror) => {
             let res = applyScreenCounterRotation(undefined, c.impl.face_tracker_anchor_pose_raw(o, indx));
             if (mirror) {

package/lib/pipeline.d.ts

@@ -7,6 +7,7 @@ import { Source, CameraFrameInfo } from "./source";
 import { CameraFrameReturnS2C, ImageBitmapS2C, VideoFrameS2C } from "./workerinterface";
 import { FaceMesh } from "./facemesh";
 import { Event } from "./event";
+import { ImageTracker } from "./imagetracker";
 export declare class Pipeline {
     private _client;
     private _impl;
@@ -21,6 +22,7 @@ export declare class Pipeline {
     cameraPixelArrays: ArrayBuffer[];
     private _cameraDraw;
     private _faceDraw;
+    private _imageTargetPreviewDraw;
     private _faceProjectDraw;
     private _sequenceRecorder;
     private _sequenceRecordDeviceAttitudeMatrices;
@@ -44,6 +46,7 @@ export declare class Pipeline {
     glContextLost(): void;
     glContextSet(gl: WebGLRenderingContext, texturePool?: WebGLTexture[]): void;
     drawFace(projectionMatrix: Float32Array, cameraMatrix: Float32Array, targetMatrix: Float32Array, o: FaceMesh): void;
+    drawImageTargetPreview(projectionMatrix: Float32Array, cameraMatrix: Float32Array, targetMatrix: Float32Array, indx: number, o: ImageTracker): void;
     drawFaceProject(matrix: Float32Array, vertices: Float32Array, uvMatrix: Float32Array, uvs: Float32Array, indices: Uint16Array, texture: WebGLTexture): void;
     cameraFrameTexture(): WebGLTexture | undefined;
     cameraFrameTextureMatrix(sw: number, sh: number, mirror: boolean): Float32Array;

package/lib/pipeline.js

@@ -8,6 +8,7 @@ import { Event } from "./event";
 import { zcerr } from "./loglevel";
 import { SequenceRecorder } from "./sequencerecorder";
 import { getCameraSource } from "./camera-source-map";
+import { PreviewMeshDraw } from "./drawpreviewmesh";
 let byId = new Map();
 let identity = mat4.create();
 export class Pipeline {
@@ -121,10 +122,13 @@ export class Pipeline {
             this._cameraDraw.dispose();
         if (this._faceDraw)
             this._faceDraw.dispose();
+        if (this._imageTargetPreviewDraw)
+            this._imageTargetPreviewDraw.dispose();
         if (this._faceProjectDraw)
             this._faceProjectDraw.dispose();
         delete this._cameraDraw;
         delete this._faceDraw;
+        delete this._imageTargetPreviewDraw;
         delete this._faceProjectDraw;
         disposeDrawPlane();
         this.onGLContextReset.emit();
@@ -166,6 +170,16 @@ export class Pipeline {
         mat4.multiply(mat, mat, targetMatrix);
         this._faceDraw.drawFace(mat, o);
     }
+    drawImageTargetPreview(projectionMatrix, cameraMatrix, targetMatrix, indx, o) {
+        if (!this.glContext)
+            return;
+        if (!this._imageTargetPreviewDraw)
+            this._imageTargetPreviewDraw = new PreviewMeshDraw(this.glContext);
+        let mat = mat4.create();
+        mat4.multiply(mat, projectionMatrix, cameraMatrix);
+        mat4.multiply(mat, mat, targetMatrix);
+        this._imageTargetPreviewDraw.draw(mat, o, indx);
+    }
     drawFaceProject(matrix, vertices, uvMatrix, uvs, indices, texture) {
         if (!this.glContext)
             return;

package/lib/version.d.ts

@@ -1 +1 @@
-export declare const VERSION = "2.0.0-beta.5";
+export declare const VERSION = "2.0.0-beta.9";

package/lib/version.js

@@ -1 +1 @@
-export const VERSION = "2.0.0-beta.5";
+export const VERSION = "2.0.0-beta.9";

package/lib/worker-client.js

@@ -15,7 +15,7 @@ export function launchWorker(worker) {
             worker = new Worker(new URL("./worker", import.meta.url), { type: 'module' });
         worker.postMessage({
             t: "wasm",
-            url: new URL("./zcv.wasm", import.meta.url).toString()
+            url: new URL("./zappar-cv.wasm", import.meta.url).toString()
         });
         yield waitForLoad(worker);
         function sendOutgoing() {

package/lib/worker-server.js

@@ -7,7 +7,7 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
         step((generator = generator.apply(thisArg, _arguments || [])).next());
     });
 };
-import * as ZNM from "./zcv";
+import * as ZNM from "./zappar-cv";
 import { getRuntimeObject } from "./gen/zappar-cwrap";
 import { zappar_server } from "./gen/zappar-server";
 import { MsgManager } from "./messages";
@@ -21,7 +21,7 @@ export function launchWorkerServer(wasmUrl) {
     return __awaiter(this, void 0, void 0, function* () {
         let mod = ZNM.default({
             locateFile: (path, prefix) => {
-                if (path.endsWith("zcv.wasm")) {
+                if (path.endsWith("zappar-cv.wasm")) {
                     return wasmUrl;
                 }
                 return prefix + path;

package/lib/worker.js

@@ -8,7 +8,7 @@ messageManager.onOutgoingMessage.bind(() => {
 });
 let launchHandler = (evt) => {
     if (evt && evt.data && evt.data.t === "wasm") {
-        let url = location.href.startsWith("blob") ? evt.data.url : new URL("./zcv.wasm", import.meta.url).toString();
+        let url = location.href.startsWith("blob") ? evt.data.url : new URL("./zappar-cv.wasm", import.meta.url).toString();
         launchWorkerServer(url);
         ctx.removeEventListener("message", launchHandler);
     }