@zappar/zappar-cv 3.10.0 → 3.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/lib/bridged-camera-source.d.ts +1 -0
- package/lib/bridged-camera-source.js +8 -0
- package/lib/camera-source.d.ts +3 -0
- package/lib/camera-source.js +8 -2
- package/lib/gen/zappar-native.d.ts +2 -1
- package/lib/gen/zappar-native.js +1 -0
- package/lib/gen/zappar.d.ts +1 -0
- package/lib/html-element-source.d.ts +1 -0
- package/lib/html-element-source.js +4 -3
- package/lib/image-process-gl.d.ts +4 -5
- package/lib/image-process-gl.js +14 -25
- package/lib/imagebitmap-camera-source.d.ts +3 -0
- package/lib/imagebitmap-camera-source.js +12 -5
- package/lib/mstp-camera-source.d.ts +3 -0
- package/lib/mstp-camera-source.js +8 -2
- package/lib/native.js +1 -1
- package/lib/profile.d.ts +2 -1
- package/lib/profile.js +33 -2
- package/lib/src/bridged-camera-source.d.ts +1 -0
- package/lib/src/camera-source.d.ts +3 -0
- package/lib/src/gen/zappar-native.d.ts +2 -1
- package/lib/src/gen/zappar.d.ts +1 -0
- package/lib/src/html-element-source.d.ts +1 -0
- package/lib/src/image-process-gl.d.ts +4 -5
- package/lib/src/imagebitmap-camera-source.d.ts +3 -0
- package/lib/src/mstp-camera-source.d.ts +3 -0
- package/lib/src/profile.d.ts +2 -1
- package/lib/src/version.d.ts +1 -1
- package/lib/src/webxr-camera-source.d.ts +5 -0
- package/lib/tests/common.d.ts +3 -1
- package/lib/version.d.ts +1 -1
- package/lib/version.js +1 -1
- package/lib/webxr-camera-source.d.ts +5 -0
- package/lib/webxr-camera-source.js +67 -35
- package/lib/worker-imagebitmap.js +2 -14
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -18,8 +18,8 @@ npm i @zappar/zappar-cv
|
|
|
18
18
|
|
|
19
19
|
You can use our CDN from within your HTML:
|
|
20
20
|
```
|
|
21
|
-
<script src="https://libs.zappar.com/zappar-cv/3.
|
|
21
|
+
<script src="https://libs.zappar.com/zappar-cv/3.11.0/zappar-cv.js"></script>
|
|
22
22
|
```
|
|
23
23
|
|
|
24
24
|
Or you can download and host our standalone JavaScript bundle:
|
|
25
|
-
[https://libs.zappar.com/zappar-cv/3.
|
|
25
|
+
[https://libs.zappar.com/zappar-cv/3.11.0/zappar-cv.zip](https://libs.zappar.com/zappar-cv/3.11.0/zappar-cv.zip)
|
|
@@ -39,6 +39,7 @@ export declare class BridgedCameraSource extends Source {
|
|
|
39
39
|
uploadGL(info: CameraFrameInfo): void;
|
|
40
40
|
recycle(info: CameraFrameInfo): void;
|
|
41
41
|
setProfile(p: camera_profile_t): void;
|
|
42
|
+
setPreferredResolution(w: number, h: number): void;
|
|
42
43
|
private _anchorPoseFloat32;
|
|
43
44
|
private _anchorPoseUint16;
|
|
44
45
|
setCustomAnchorPose(anchor: zappar_custom_anchor_t, pose_version: number, pose: Float32Array): void;
|
|
@@ -365,6 +365,11 @@ export class BridgedCameraSource extends Source {
|
|
|
365
365
|
this._decodeBuffers.push(recycleBuffer);
|
|
366
366
|
}
|
|
367
367
|
setProfile(p) {
|
|
368
|
+
// Treat FULL as the same as HIGH for the bridge so we don't
|
|
369
|
+
// change the interface. The main use case (high res D3) is
|
|
370
|
+
// already handled separately in the bridge anyway.
|
|
371
|
+
if (p == camera_profile_t.FULL)
|
|
372
|
+
p = camera_profile_t.HIGH;
|
|
368
373
|
if (this._profile === p)
|
|
369
374
|
return;
|
|
370
375
|
this._profile = p;
|
|
@@ -372,6 +377,9 @@ export class BridgedCameraSource extends Source {
|
|
|
372
377
|
this._messageHandler.postMessage(`setProfile${this._profile}`);
|
|
373
378
|
}
|
|
374
379
|
}
|
|
380
|
+
setPreferredResolution(w, h) {
|
|
381
|
+
// Not implemented over the bridge yet
|
|
382
|
+
}
|
|
375
383
|
setCustomAnchorPose(anchor, pose_version, pose) {
|
|
376
384
|
let message = `setCustomAnchorPose${anchor}:${pose_version}:`;
|
|
377
385
|
this._anchorPoseFloat32.set(pose);
|
package/lib/camera-source.d.ts
CHANGED
|
@@ -7,6 +7,8 @@ export declare class CameraSource extends HTMLElementSource {
|
|
|
7
7
|
static DEFAULT_DEVICE_ID: string;
|
|
8
8
|
private _currentStream;
|
|
9
9
|
private _activeDeviceId;
|
|
10
|
+
private _preferredWidth;
|
|
11
|
+
private _preferredHeight;
|
|
10
12
|
private _explicitUserCameraId;
|
|
11
13
|
private _explicitEnvironmentCameraId;
|
|
12
14
|
constructor(_impl: zappar_camera_source_t, pipeline: zappar_pipeline_t, _deviceId: string);
|
|
@@ -14,6 +16,7 @@ export declare class CameraSource extends HTMLElementSource {
|
|
|
14
16
|
private _stop;
|
|
15
17
|
pause(): void;
|
|
16
18
|
start(): void;
|
|
19
|
+
setPreferredResolution(w: number, h: number): void;
|
|
17
20
|
private _getConstraints;
|
|
18
21
|
getFrame(allowRead: boolean): void;
|
|
19
22
|
private _getUserMedia;
|
package/lib/camera-source.js
CHANGED
|
@@ -28,6 +28,8 @@ export class CameraSource extends HTMLElementSource {
|
|
|
28
28
|
this._deviceId = _deviceId;
|
|
29
29
|
this._currentStream = null;
|
|
30
30
|
this._activeDeviceId = null;
|
|
31
|
+
this._preferredWidth = profile.videoWidth;
|
|
32
|
+
this._preferredHeight = profile.videoHeight;
|
|
31
33
|
this._hasStartedOrientation = false;
|
|
32
34
|
this._lastTimestamp = -1;
|
|
33
35
|
this._deviceMotionListener = (ev) => {
|
|
@@ -90,6 +92,10 @@ export class CameraSource extends HTMLElementSource {
|
|
|
90
92
|
this._startDeviceMotion();
|
|
91
93
|
this._syncCamera();
|
|
92
94
|
}
|
|
95
|
+
setPreferredResolution(w, h) {
|
|
96
|
+
this._preferredWidth = w;
|
|
97
|
+
this._preferredHeight = h;
|
|
98
|
+
}
|
|
93
99
|
_getConstraints() {
|
|
94
100
|
return __awaiter(this, void 0, void 0, function* () {
|
|
95
101
|
let deviceId;
|
|
@@ -120,8 +126,8 @@ export class CameraSource extends HTMLElementSource {
|
|
|
120
126
|
audio: false,
|
|
121
127
|
video: {
|
|
122
128
|
facingMode: facingMode,
|
|
123
|
-
width:
|
|
124
|
-
height:
|
|
129
|
+
width: this._preferredWidth,
|
|
130
|
+
height: this._preferredHeight,
|
|
125
131
|
frameRate: profile.requestHighFrameRate ? 60 : undefined,
|
|
126
132
|
deviceId: deviceId
|
|
127
133
|
}
|
package/lib/gen/zappar-native.js
CHANGED
|
@@ -107,6 +107,7 @@ export var camera_profile_t;
|
|
|
107
107
|
(function (camera_profile_t) {
|
|
108
108
|
camera_profile_t[camera_profile_t["DEFAULT"] = 0] = "DEFAULT";
|
|
109
109
|
camera_profile_t[camera_profile_t["HIGH"] = 1] = "HIGH";
|
|
110
|
+
camera_profile_t[camera_profile_t["FULL"] = 2] = "FULL";
|
|
110
111
|
})(camera_profile_t || (camera_profile_t = {}));
|
|
111
112
|
;
|
|
112
113
|
export var world_scale_mode_t;
|
package/lib/gen/zappar.d.ts
CHANGED
|
@@ -125,6 +125,7 @@ export interface zappar {
|
|
|
125
125
|
camera_source_start(o: zappar_camera_source_t): void;
|
|
126
126
|
camera_source_pause(o: zappar_camera_source_t): void;
|
|
127
127
|
camera_source_profile_set(o: zappar_camera_source_t, profile: camera_profile_t): void;
|
|
128
|
+
camera_source_preferred_resolution_set(o: zappar_camera_source_t, width: number, height: number): void;
|
|
128
129
|
sequence_source_create(pipeline: zappar_pipeline_t): zappar_sequence_source_t;
|
|
129
130
|
sequence_source_destroy(o: zappar_sequence_source_t): void;
|
|
130
131
|
sequence_source_start(o: zappar_sequence_source_t): void;
|
|
@@ -10,6 +10,7 @@ export declare class HTMLElementSource extends Source {
|
|
|
10
10
|
protected _cameraToScreenRotation: number;
|
|
11
11
|
private _isUploadFrame;
|
|
12
12
|
private _currentVideoTexture;
|
|
13
|
+
private _currentDataSize;
|
|
13
14
|
private _imageProcessor;
|
|
14
15
|
private _cameraToDeviceTransform;
|
|
15
16
|
private _cameraToDeviceTransformUserFacing;
|
|
@@ -17,6 +17,7 @@ export class HTMLElementSource extends Source {
|
|
|
17
17
|
this._isUserFacing = false;
|
|
18
18
|
this._cameraToScreenRotation = 0;
|
|
19
19
|
this._isUploadFrame = true;
|
|
20
|
+
this._currentDataSize = [320, 240];
|
|
20
21
|
this._cameraToDeviceTransform = mat4.create();
|
|
21
22
|
this._cameraToDeviceTransformUserFacing = mat4.create();
|
|
22
23
|
this._cameraModel = new Float32Array([300, 300, 160, 120, 0, 0]);
|
|
@@ -148,7 +149,7 @@ export class HTMLElementSource extends Source {
|
|
|
148
149
|
return undefined;
|
|
149
150
|
if (pipeline.getCameraFrameDataRGBEnabled())
|
|
150
151
|
this._currentRgbFrame = this._generateRGBFrame();
|
|
151
|
-
this._imageProcessor.uploadFrame(this._currentVideoTexture, this._video, rotation, this._isUserFacing, this._profile);
|
|
152
|
+
this._currentDataSize = this._imageProcessor.uploadFrame(this._currentVideoTexture, this._video, rotation, this._isUserFacing, this._profile);
|
|
152
153
|
this._requestVideoFrameCallback();
|
|
153
154
|
this._isUploadFrame = !this._isUploadFrame;
|
|
154
155
|
return undefined;
|
|
@@ -156,7 +157,7 @@ export class HTMLElementSource extends Source {
|
|
|
156
157
|
if (currentlyProcessing || !this._currentVideoTexture)
|
|
157
158
|
return undefined;
|
|
158
159
|
this._isUploadFrame = !this._isUploadFrame;
|
|
159
|
-
const [dataWidth, dataHeight] =
|
|
160
|
+
const [dataWidth, dataHeight] = this._currentDataSize;
|
|
160
161
|
let greySize = dataWidth * dataHeight;
|
|
161
162
|
let pixels = pipeline.cameraPixelArrays.pop();
|
|
162
163
|
while (pixels) {
|
|
@@ -174,7 +175,7 @@ export class HTMLElementSource extends Source {
|
|
|
174
175
|
this._cameraModel[1] = focalLength;
|
|
175
176
|
this._cameraModel[2] = dataWidth * 0.5;
|
|
176
177
|
this._cameraModel[3] = dataHeight * 0.5;
|
|
177
|
-
const info = Object.assign(Object.assign({}, this._imageProcessor.readFrame(
|
|
178
|
+
const info = Object.assign(Object.assign({}, this._imageProcessor.readFrame(pixels)), { texture: tex, cameraModel: this._cameraModel, cameraSource: this, cameraToDevice: (this._isUserFacing ? this._cameraToDeviceTransformUserFacing : this._cameraToDeviceTransform) });
|
|
178
179
|
if (this._currentRgbFrame) {
|
|
179
180
|
info.rgbFrame = this._currentRgbFrame;
|
|
180
181
|
this._currentRgbFrame = undefined;
|
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
import { camera_profile_t } from "./gen/zappar-native";
|
|
2
2
|
export interface ProcessedFrame {
|
|
3
3
|
data?: ArrayBuffer;
|
|
4
|
-
texture: WebGLTexture | undefined;
|
|
5
4
|
dataWidth: number;
|
|
6
5
|
dataHeight: number;
|
|
7
6
|
uvTransform?: Float32Array;
|
|
@@ -17,6 +16,7 @@ export declare class ImageProcessGL {
|
|
|
17
16
|
private _computedTransformRotation;
|
|
18
17
|
private _computedFrontCameraRotation;
|
|
19
18
|
private _cameraUvTransform;
|
|
19
|
+
private _dataSize;
|
|
20
20
|
private _framebufferWidth;
|
|
21
21
|
private _framebufferHeight;
|
|
22
22
|
private _framebufferId;
|
|
@@ -29,10 +29,9 @@ export declare class ImageProcessGL {
|
|
|
29
29
|
constructor(_gl: WebGL2RenderingContext | WebGLRenderingContext);
|
|
30
30
|
resetGLContext(): void;
|
|
31
31
|
destroy(): void;
|
|
32
|
-
uploadFrame(texture: WebGLTexture, img: HTMLVideoElement | HTMLImageElement | ImageBitmap, rotation: number, fc: boolean, cp: camera_profile_t):
|
|
33
|
-
uploadFrameForSize(texture: WebGLTexture, img: HTMLVideoElement | HTMLImageElement | ImageBitmap, rotation: number, fc: boolean, dataWidth: number, dataHeight: number): void;
|
|
34
|
-
readFrame(
|
|
35
|
-
readFrameForSize(texture: WebGLTexture, pixels: ArrayBuffer, dataWidth: number, dataHeight: number): ProcessedFrame;
|
|
32
|
+
uploadFrame(texture: WebGLTexture, img: HTMLVideoElement | HTMLImageElement | ImageBitmap, rotation: number, fc: boolean, cp: camera_profile_t): [number, number];
|
|
33
|
+
uploadFrameForSize(texture: WebGLTexture, img: HTMLVideoElement | HTMLImageElement | ImageBitmap | null, rotation: number, fc: boolean, dataWidth: number, dataHeight: number): void;
|
|
34
|
+
readFrame(pixels: ArrayBuffer): ProcessedFrame;
|
|
36
35
|
private _updateTransforms;
|
|
37
36
|
private _getCameraUvTransform;
|
|
38
37
|
private _getFramebuffer;
|
package/lib/image-process-gl.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { mat4 } from "gl-matrix";
|
|
2
2
|
import { GLStateManager } from "./gl-state-manager";
|
|
3
|
-
import {
|
|
3
|
+
import { getDataSizeForSource } from "./profile";
|
|
4
4
|
import { compileShader, linkProgram } from "./shader";
|
|
5
5
|
export class ImageProcessGL {
|
|
6
6
|
constructor(_gl) {
|
|
@@ -13,6 +13,7 @@ export class ImageProcessGL {
|
|
|
13
13
|
this._computedTransformRotation = -1;
|
|
14
14
|
this._computedFrontCameraRotation = false;
|
|
15
15
|
this._cameraUvTransform = mat4.create();
|
|
16
|
+
this._dataSize = [0, 0];
|
|
16
17
|
this._framebufferWidth = 0;
|
|
17
18
|
this._framebufferHeight = 0;
|
|
18
19
|
this._framebufferId = null;
|
|
@@ -34,8 +35,9 @@ export class ImageProcessGL {
|
|
|
34
35
|
this.resetGLContext();
|
|
35
36
|
}
|
|
36
37
|
uploadFrame(texture, img, rotation, fc, cp) {
|
|
37
|
-
const
|
|
38
|
-
this.uploadFrameForSize(texture, img, rotation, fc,
|
|
38
|
+
const dataSize = getDataSizeForSource(cp, img, rotation);
|
|
39
|
+
this.uploadFrameForSize(texture, img, rotation, fc, ...dataSize);
|
|
40
|
+
return dataSize;
|
|
39
41
|
}
|
|
40
42
|
uploadFrameForSize(texture, img, rotation, fc, dataWidth, dataHeight) {
|
|
41
43
|
let gl = this._gl;
|
|
@@ -61,24 +63,15 @@ export class ImageProcessGL {
|
|
|
61
63
|
gl.disable(gl.STENCIL_TEST);
|
|
62
64
|
gl.pixelStorei(gl.UNPACK_FLIP_Y_WEBGL, false);
|
|
63
65
|
gl.bindTexture(gl.TEXTURE_2D, texture);
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
let videoHeight = 0;
|
|
71
|
-
if (typeof HTMLVideoElement !== "undefined" && img instanceof HTMLVideoElement) {
|
|
72
|
-
videoWidth = img.videoWidth;
|
|
73
|
-
videoHeight = img.videoHeight;
|
|
66
|
+
if (img) {
|
|
67
|
+
const level = 0;
|
|
68
|
+
const internalFormat = gl.RGBA;
|
|
69
|
+
const srcFormat = gl.RGBA;
|
|
70
|
+
const srcType = gl.UNSIGNED_BYTE;
|
|
71
|
+
gl.texImage2D(gl.TEXTURE_2D, level, internalFormat, srcFormat, srcType, img);
|
|
74
72
|
}
|
|
75
|
-
else {
|
|
76
|
-
videoWidth = img.width;
|
|
77
|
-
videoHeight = img.height;
|
|
78
|
-
}
|
|
79
|
-
if (videoHeight > videoWidth)
|
|
80
|
-
videoHeight = [videoWidth, videoWidth = videoHeight][0];
|
|
81
73
|
this._updateTransforms(rotation, fc);
|
|
74
|
+
this._dataSize = [dataWidth, dataHeight];
|
|
82
75
|
let framebuffer = this._getFramebuffer(gl, dataWidth / 4, dataHeight);
|
|
83
76
|
let vbo = this._getVertexBuffer(gl);
|
|
84
77
|
let ibo = this._getIndexBuffer(gl);
|
|
@@ -171,13 +164,10 @@ export class ImageProcessGL {
|
|
|
171
164
|
if (reenableStencilTest)
|
|
172
165
|
gl.enable(gl.STENCIL_TEST);
|
|
173
166
|
}
|
|
174
|
-
readFrame(
|
|
175
|
-
const [dataWidth, dataHeight] = profile.getDataSize(cp);
|
|
176
|
-
return this.readFrameForSize(texture, pixels, dataWidth, dataHeight);
|
|
177
|
-
}
|
|
178
|
-
readFrameForSize(texture, pixels, dataWidth, dataHeight) {
|
|
167
|
+
readFrame(pixels) {
|
|
179
168
|
let gl = this._gl;
|
|
180
169
|
let pixelsView = new Uint8Array(pixels);
|
|
170
|
+
const [dataWidth, dataHeight] = this._dataSize;
|
|
181
171
|
const previousBoundFramebuffer = gl.getParameter(gl.FRAMEBUFFER_BINDING);
|
|
182
172
|
let framebuffer = this._getFramebuffer(gl, dataWidth / 4, dataHeight);
|
|
183
173
|
gl.bindFramebuffer(gl.FRAMEBUFFER, framebuffer);
|
|
@@ -186,7 +176,6 @@ export class ImageProcessGL {
|
|
|
186
176
|
return {
|
|
187
177
|
uvTransform: this._cameraUvTransform,
|
|
188
178
|
data: pixels,
|
|
189
|
-
texture,
|
|
190
179
|
dataWidth,
|
|
191
180
|
dataHeight,
|
|
192
181
|
userFacing: this._computedFrontCameraRotation
|
|
@@ -8,6 +8,8 @@ export declare class ImageBitmapCameraSource extends Source {
|
|
|
8
8
|
static DEFAULT_DEVICE_ID: string;
|
|
9
9
|
private _currentStream;
|
|
10
10
|
private _activeDeviceId;
|
|
11
|
+
private _preferredWidth;
|
|
12
|
+
private _preferredHeight;
|
|
11
13
|
private _isPaused;
|
|
12
14
|
private _isUserFacing;
|
|
13
15
|
private _hadFrames;
|
|
@@ -22,6 +24,7 @@ export declare class ImageBitmapCameraSource extends Source {
|
|
|
22
24
|
constructor(_impl: zappar_camera_source_t, _pipeline: zappar_pipeline_t, _deviceId: string);
|
|
23
25
|
destroy(): void;
|
|
24
26
|
setProfile(p: camera_profile_t): void;
|
|
27
|
+
setPreferredResolution(w: number, h: number): void;
|
|
25
28
|
private _stop;
|
|
26
29
|
pause(): void;
|
|
27
30
|
start(): void;
|
|
@@ -8,7 +8,7 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
|
|
|
8
8
|
});
|
|
9
9
|
};
|
|
10
10
|
import { camera_profile_t } from "./gen/zappar";
|
|
11
|
-
import { profile } from "./profile";
|
|
11
|
+
import { profile, getDataSizeForSource } from "./profile";
|
|
12
12
|
import { Pipeline } from "./pipeline";
|
|
13
13
|
import { Source } from "./source";
|
|
14
14
|
import { cameraRotationForScreenOrientation } from "./cameramodel";
|
|
@@ -23,6 +23,8 @@ export class ImageBitmapCameraSource extends Source {
|
|
|
23
23
|
this._deviceId = _deviceId;
|
|
24
24
|
this._currentStream = null;
|
|
25
25
|
this._activeDeviceId = null;
|
|
26
|
+
this._preferredWidth = profile.videoWidth;
|
|
27
|
+
this._preferredHeight = profile.videoHeight;
|
|
26
28
|
this._isPaused = true;
|
|
27
29
|
this._isUserFacing = false;
|
|
28
30
|
this._hadFrames = false;
|
|
@@ -90,6 +92,10 @@ export class ImageBitmapCameraSource extends Source {
|
|
|
90
92
|
setProfile(p) {
|
|
91
93
|
this._profile = p;
|
|
92
94
|
}
|
|
95
|
+
setPreferredResolution(w, h) {
|
|
96
|
+
this._preferredWidth = w;
|
|
97
|
+
this._preferredHeight = h;
|
|
98
|
+
}
|
|
93
99
|
_stop() {
|
|
94
100
|
if (!this._currentStream)
|
|
95
101
|
return;
|
|
@@ -132,8 +138,8 @@ export class ImageBitmapCameraSource extends Source {
|
|
|
132
138
|
audio: false,
|
|
133
139
|
video: {
|
|
134
140
|
facingMode: facingMode,
|
|
135
|
-
width:
|
|
136
|
-
height:
|
|
141
|
+
width: this._preferredWidth,
|
|
142
|
+
height: this._preferredHeight,
|
|
137
143
|
frameRate: profile.requestHighFrameRate ? 60 : undefined,
|
|
138
144
|
deviceId: deviceId
|
|
139
145
|
}
|
|
@@ -190,7 +196,7 @@ export class ImageBitmapCameraSource extends Source {
|
|
|
190
196
|
let pipeline = Pipeline.get(this._pipeline);
|
|
191
197
|
if (!pipeline)
|
|
192
198
|
return;
|
|
193
|
-
const [dataWidth, dataHeight] =
|
|
199
|
+
const [dataWidth, dataHeight] = getDataSizeForSource(this._profile, imageBitmap, rotation);
|
|
194
200
|
let focalLength = 300.0 * dataWidth / 320.0;
|
|
195
201
|
this._cameraModel[0] = focalLength;
|
|
196
202
|
this._cameraModel[1] = focalLength;
|
|
@@ -281,10 +287,11 @@ export class ImageBitmapCameraSource extends Source {
|
|
|
281
287
|
let texture = pipeline.getVideoTexture();
|
|
282
288
|
if (!texture)
|
|
283
289
|
return;
|
|
290
|
+
const previousBoundTexture = gl.getParameter(gl.TEXTURE_BINDING_2D);
|
|
284
291
|
gl.bindTexture(gl.TEXTURE_2D, texture);
|
|
285
292
|
gl.pixelStorei(gl.UNPACK_FLIP_Y_WEBGL, false);
|
|
286
293
|
gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, info.frame);
|
|
287
|
-
gl.bindTexture(gl.TEXTURE_2D,
|
|
294
|
+
gl.bindTexture(gl.TEXTURE_2D, previousBoundTexture);
|
|
288
295
|
info.texture = texture;
|
|
289
296
|
}
|
|
290
297
|
}
|
|
@@ -8,6 +8,8 @@ export declare class MSTPCameraSource extends Source {
|
|
|
8
8
|
static DEFAULT_DEVICE_ID: string;
|
|
9
9
|
private _currentStream;
|
|
10
10
|
private _activeDeviceId;
|
|
11
|
+
private _preferredWidth;
|
|
12
|
+
private _preferredHeight;
|
|
11
13
|
private _isPaused;
|
|
12
14
|
private _isUserFacing;
|
|
13
15
|
private _cameraToScreenRotation;
|
|
@@ -20,6 +22,7 @@ export declare class MSTPCameraSource extends Source {
|
|
|
20
22
|
private _stop;
|
|
21
23
|
pause(): void;
|
|
22
24
|
start(): void;
|
|
25
|
+
setPreferredResolution(w: number, h: number): void;
|
|
23
26
|
private _getConstraints;
|
|
24
27
|
getFrame(allowRead: boolean): void;
|
|
25
28
|
private _getUserMedia;
|
|
@@ -21,6 +21,8 @@ export class MSTPCameraSource extends Source {
|
|
|
21
21
|
this._deviceId = _deviceId;
|
|
22
22
|
this._currentStream = null;
|
|
23
23
|
this._activeDeviceId = null;
|
|
24
|
+
this._preferredWidth = profile.videoWidth;
|
|
25
|
+
this._preferredHeight = profile.videoHeight;
|
|
24
26
|
this._isPaused = true;
|
|
25
27
|
this._isUserFacing = false;
|
|
26
28
|
this._cameraToScreenRotation = 0;
|
|
@@ -150,6 +152,10 @@ export class MSTPCameraSource extends Source {
|
|
|
150
152
|
this._startDeviceMotion();
|
|
151
153
|
this._syncCamera();
|
|
152
154
|
}
|
|
155
|
+
setPreferredResolution(w, h) {
|
|
156
|
+
this._preferredWidth = w;
|
|
157
|
+
this._preferredHeight = h;
|
|
158
|
+
}
|
|
153
159
|
_getConstraints() {
|
|
154
160
|
return __awaiter(this, void 0, void 0, function* () {
|
|
155
161
|
let deviceId;
|
|
@@ -166,8 +172,8 @@ export class MSTPCameraSource extends Source {
|
|
|
166
172
|
audio: false,
|
|
167
173
|
video: {
|
|
168
174
|
facingMode: facingMode,
|
|
169
|
-
width:
|
|
170
|
-
height:
|
|
175
|
+
width: this._preferredWidth,
|
|
176
|
+
height: this._preferredHeight,
|
|
171
177
|
frameRate: profile.requestHighFrameRate ? 60 : 30,
|
|
172
178
|
deviceId: deviceId
|
|
173
179
|
}
|
package/lib/native.js
CHANGED
|
@@ -271,7 +271,7 @@ export function initialize(opts) {
|
|
|
271
271
|
mat4.multiply(res, res, scale);
|
|
272
272
|
}
|
|
273
273
|
return res;
|
|
274
|
-
}, loaded: () => loaded, camera_default_device_id: userFacing => userFacing ? CameraSource.USER_DEFAULT_DEVICE_ID : CameraSource.DEFAULT_DEVICE_ID, camera_source_create: (p, deviceId) => createCameraSource(p, deviceId, preferWebXRCamera), camera_source_destroy: cam => { var _a; return (_a = getCameraSource(cam)) === null || _a === void 0 ? void 0 : _a.destroy(); }, camera_source_pause: cam => { var _a; return (_a = getCameraSource(cam)) === null || _a === void 0 ? void 0 : _a.pause(); }, camera_source_start: cam => { var _a; return (_a = getCameraSource(cam)) === null || _a === void 0 ? void 0 : _a.start(); }, camera_source_profile_set: (cam, p) => { var _a; return (_a = getCameraSource(cam)) === null || _a === void 0 ? void 0 : _a.setProfile(p); }, camera_count: () => 2, camera_id: indx => indx === 0 ? CameraSource.DEFAULT_DEVICE_ID : CameraSource.USER_DEFAULT_DEVICE_ID, camera_name: indx => indx === 0 ? "Rear-facing Camera" : "User-facing Camera", camera_user_facing: indx => indx !== 0, pose_from_raw: (raw_pose, mirror) => {
|
|
274
|
+
}, loaded: () => loaded, camera_default_device_id: userFacing => userFacing ? CameraSource.USER_DEFAULT_DEVICE_ID : CameraSource.DEFAULT_DEVICE_ID, camera_source_create: (p, deviceId) => createCameraSource(p, deviceId, preferWebXRCamera), camera_source_destroy: cam => { var _a; return (_a = getCameraSource(cam)) === null || _a === void 0 ? void 0 : _a.destroy(); }, camera_source_pause: cam => { var _a; return (_a = getCameraSource(cam)) === null || _a === void 0 ? void 0 : _a.pause(); }, camera_source_start: cam => { var _a; return (_a = getCameraSource(cam)) === null || _a === void 0 ? void 0 : _a.start(); }, camera_source_profile_set: (cam, p) => { var _a; return (_a = getCameraSource(cam)) === null || _a === void 0 ? void 0 : _a.setProfile(p); }, camera_source_preferred_resolution_set: (cam, w, h) => { var _a; return (_a = getCameraSource(cam)) === null || _a === void 0 ? void 0 : _a.setPreferredResolution(w, h); }, camera_count: () => 2, camera_id: indx => indx === 0 ? CameraSource.DEFAULT_DEVICE_ID : CameraSource.USER_DEFAULT_DEVICE_ID, camera_name: indx => indx === 0 ? "Rear-facing Camera" : "User-facing Camera", camera_user_facing: indx => indx !== 0, pose_from_raw: (raw_pose, mirror) => {
|
|
275
275
|
let res = applyScreenCounterRotation(undefined, raw_pose);
|
|
276
276
|
if (mirror) {
|
|
277
277
|
let scale = mat4.create();
|
package/lib/profile.d.ts
CHANGED
|
@@ -11,7 +11,6 @@ export declare let profile: {
|
|
|
11
11
|
requestHighFrameRate: boolean;
|
|
12
12
|
videoWidth: number;
|
|
13
13
|
videoHeight: number;
|
|
14
|
-
getDataSize: (p: camera_profile_t) => number[];
|
|
15
14
|
videoElementInDOM: boolean;
|
|
16
15
|
preferMediaStreamTrackProcessorCamera: boolean;
|
|
17
16
|
preferImageBitmapCamera: boolean;
|
|
@@ -22,3 +21,5 @@ export declare let profile: {
|
|
|
22
21
|
trustSensorIntervals: boolean;
|
|
23
22
|
offscreenCanvasSupported: boolean;
|
|
24
23
|
};
|
|
24
|
+
export declare function getDataSize(p: camera_profile_t, fullWidth: number, fullHeight: number): [number, number];
|
|
25
|
+
export declare function getDataSizeForSource(p: camera_profile_t, source: HTMLVideoElement | HTMLImageElement | ImageBitmap, rotation?: number): [number, number];
|
package/lib/profile.js
CHANGED
|
@@ -13,7 +13,6 @@ export let profile = {
|
|
|
13
13
|
requestHighFrameRate: false,
|
|
14
14
|
videoWidth: 1280,
|
|
15
15
|
videoHeight: 720,
|
|
16
|
-
getDataSize: (p) => p === camera_profile_t.HIGH ? [640, 360] : [320, 180],
|
|
17
16
|
videoElementInDOM: false,
|
|
18
17
|
preferMediaStreamTrackProcessorCamera: false,
|
|
19
18
|
preferImageBitmapCamera: false,
|
|
@@ -24,6 +23,39 @@ export let profile = {
|
|
|
24
23
|
trustSensorIntervals: false,
|
|
25
24
|
offscreenCanvasSupported: typeof globalThis.OffscreenCanvas !== "undefined",
|
|
26
25
|
};
|
|
26
|
+
export function getDataSize(p, fullWidth, fullHeight) {
|
|
27
|
+
if (p === camera_profile_t.FULL)
|
|
28
|
+
return [fullWidth, fullHeight];
|
|
29
|
+
let fullMax = fullWidth > fullHeight ? fullWidth : fullHeight;
|
|
30
|
+
let dataMax = p === camera_profile_t.HIGH ? 640 : 320;
|
|
31
|
+
let scale = dataMax / fullMax;
|
|
32
|
+
if (scale >= 1)
|
|
33
|
+
return [fullWidth, fullHeight];
|
|
34
|
+
// Ensure width is a multiple of 4 (required by ImageProcessGL greyscale conversion)
|
|
35
|
+
let width = Math.round(scale * fullWidth / 4) * 4;
|
|
36
|
+
let height = Math.round(width * fullHeight / fullWidth);
|
|
37
|
+
return [width, height];
|
|
38
|
+
}
|
|
39
|
+
export function getDataSizeForSource(p, source, rotation = 0) {
|
|
40
|
+
let fullWidth = source.width;
|
|
41
|
+
let fullHeight = source.height;
|
|
42
|
+
// NB: undefined checks needed as these types aren't available in workers
|
|
43
|
+
if (typeof HTMLVideoElement !== "undefined" && source instanceof HTMLVideoElement) {
|
|
44
|
+
fullWidth = source.videoWidth;
|
|
45
|
+
fullHeight = source.videoHeight;
|
|
46
|
+
}
|
|
47
|
+
else if (typeof HTMLImageElement !== "undefined" && source instanceof HTMLImageElement) {
|
|
48
|
+
fullWidth = source.naturalWidth;
|
|
49
|
+
fullHeight = source.naturalHeight;
|
|
50
|
+
}
|
|
51
|
+
if ((rotation == 90) || (rotation == 270)) {
|
|
52
|
+
// Will be rotated as part of greyscale conversion
|
|
53
|
+
return getDataSize(p, fullHeight, fullWidth);
|
|
54
|
+
}
|
|
55
|
+
else {
|
|
56
|
+
return getDataSize(p, fullWidth, fullHeight);
|
|
57
|
+
}
|
|
58
|
+
}
|
|
27
59
|
if (typeof window !== "undefined") {
|
|
28
60
|
window["zeeProfile"] = profile;
|
|
29
61
|
if (window.location.href.indexOf("_mstppipeline") >= 0) {
|
|
@@ -81,7 +113,6 @@ function iDevice(version) {
|
|
|
81
113
|
// Avoid bug where iOS letterboxes 16:9 into 4:3 for high fps
|
|
82
114
|
// Doesn't seem needed with newer versions of iOS, tested on 14.8.1 and 15.2
|
|
83
115
|
profile.videoHeight = 360;
|
|
84
|
-
profile.getDataSize = (p) => p === camera_profile_t.HIGH ? [640, 360] : [320, 180];
|
|
85
116
|
}
|
|
86
117
|
}
|
|
87
118
|
if (majorVersion === 16 && minorVersion >= 4) {
|
|
@@ -39,6 +39,7 @@ export declare class BridgedCameraSource extends Source {
|
|
|
39
39
|
uploadGL(info: CameraFrameInfo): void;
|
|
40
40
|
recycle(info: CameraFrameInfo): void;
|
|
41
41
|
setProfile(p: camera_profile_t): void;
|
|
42
|
+
setPreferredResolution(w: number, h: number): void;
|
|
42
43
|
private _anchorPoseFloat32;
|
|
43
44
|
private _anchorPoseUint16;
|
|
44
45
|
setCustomAnchorPose(anchor: zappar_custom_anchor_t, pose_version: number, pose: Float32Array): void;
|
|
@@ -7,6 +7,8 @@ export declare class CameraSource extends HTMLElementSource {
|
|
|
7
7
|
static DEFAULT_DEVICE_ID: string;
|
|
8
8
|
private _currentStream;
|
|
9
9
|
private _activeDeviceId;
|
|
10
|
+
private _preferredWidth;
|
|
11
|
+
private _preferredHeight;
|
|
10
12
|
private _explicitUserCameraId;
|
|
11
13
|
private _explicitEnvironmentCameraId;
|
|
12
14
|
constructor(_impl: zappar_camera_source_t, pipeline: zappar_pipeline_t, _deviceId: string);
|
|
@@ -14,6 +16,7 @@ export declare class CameraSource extends HTMLElementSource {
|
|
|
14
16
|
private _stop;
|
|
15
17
|
pause(): void;
|
|
16
18
|
start(): void;
|
|
19
|
+
setPreferredResolution(w: number, h: number): void;
|
|
17
20
|
private _getConstraints;
|
|
18
21
|
getFrame(allowRead: boolean): void;
|
|
19
22
|
private _getUserMedia;
|
package/lib/src/gen/zappar.d.ts
CHANGED
|
@@ -125,6 +125,7 @@ export interface zappar {
|
|
|
125
125
|
camera_source_start(o: zappar_camera_source_t): void;
|
|
126
126
|
camera_source_pause(o: zappar_camera_source_t): void;
|
|
127
127
|
camera_source_profile_set(o: zappar_camera_source_t, profile: camera_profile_t): void;
|
|
128
|
+
camera_source_preferred_resolution_set(o: zappar_camera_source_t, width: number, height: number): void;
|
|
128
129
|
sequence_source_create(pipeline: zappar_pipeline_t): zappar_sequence_source_t;
|
|
129
130
|
sequence_source_destroy(o: zappar_sequence_source_t): void;
|
|
130
131
|
sequence_source_start(o: zappar_sequence_source_t): void;
|
|
@@ -10,6 +10,7 @@ export declare class HTMLElementSource extends Source {
|
|
|
10
10
|
protected _cameraToScreenRotation: number;
|
|
11
11
|
private _isUploadFrame;
|
|
12
12
|
private _currentVideoTexture;
|
|
13
|
+
private _currentDataSize;
|
|
13
14
|
private _imageProcessor;
|
|
14
15
|
private _cameraToDeviceTransform;
|
|
15
16
|
private _cameraToDeviceTransformUserFacing;
|
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
import { camera_profile_t } from "./gen/zappar-native";
|
|
2
2
|
export interface ProcessedFrame {
|
|
3
3
|
data?: ArrayBuffer;
|
|
4
|
-
texture: WebGLTexture | undefined;
|
|
5
4
|
dataWidth: number;
|
|
6
5
|
dataHeight: number;
|
|
7
6
|
uvTransform?: Float32Array;
|
|
@@ -17,6 +16,7 @@ export declare class ImageProcessGL {
|
|
|
17
16
|
private _computedTransformRotation;
|
|
18
17
|
private _computedFrontCameraRotation;
|
|
19
18
|
private _cameraUvTransform;
|
|
19
|
+
private _dataSize;
|
|
20
20
|
private _framebufferWidth;
|
|
21
21
|
private _framebufferHeight;
|
|
22
22
|
private _framebufferId;
|
|
@@ -29,10 +29,9 @@ export declare class ImageProcessGL {
|
|
|
29
29
|
constructor(_gl: WebGL2RenderingContext | WebGLRenderingContext);
|
|
30
30
|
resetGLContext(): void;
|
|
31
31
|
destroy(): void;
|
|
32
|
-
uploadFrame(texture: WebGLTexture, img: HTMLVideoElement | HTMLImageElement | ImageBitmap, rotation: number, fc: boolean, cp: camera_profile_t):
|
|
33
|
-
uploadFrameForSize(texture: WebGLTexture, img: HTMLVideoElement | HTMLImageElement | ImageBitmap, rotation: number, fc: boolean, dataWidth: number, dataHeight: number): void;
|
|
34
|
-
readFrame(
|
|
35
|
-
readFrameForSize(texture: WebGLTexture, pixels: ArrayBuffer, dataWidth: number, dataHeight: number): ProcessedFrame;
|
|
32
|
+
uploadFrame(texture: WebGLTexture, img: HTMLVideoElement | HTMLImageElement | ImageBitmap, rotation: number, fc: boolean, cp: camera_profile_t): [number, number];
|
|
33
|
+
uploadFrameForSize(texture: WebGLTexture, img: HTMLVideoElement | HTMLImageElement | ImageBitmap | null, rotation: number, fc: boolean, dataWidth: number, dataHeight: number): void;
|
|
34
|
+
readFrame(pixels: ArrayBuffer): ProcessedFrame;
|
|
36
35
|
private _updateTransforms;
|
|
37
36
|
private _getCameraUvTransform;
|
|
38
37
|
private _getFramebuffer;
|
|
@@ -8,6 +8,8 @@ export declare class ImageBitmapCameraSource extends Source {
|
|
|
8
8
|
static DEFAULT_DEVICE_ID: string;
|
|
9
9
|
private _currentStream;
|
|
10
10
|
private _activeDeviceId;
|
|
11
|
+
private _preferredWidth;
|
|
12
|
+
private _preferredHeight;
|
|
11
13
|
private _isPaused;
|
|
12
14
|
private _isUserFacing;
|
|
13
15
|
private _hadFrames;
|
|
@@ -22,6 +24,7 @@ export declare class ImageBitmapCameraSource extends Source {
|
|
|
22
24
|
constructor(_impl: zappar_camera_source_t, _pipeline: zappar_pipeline_t, _deviceId: string);
|
|
23
25
|
destroy(): void;
|
|
24
26
|
setProfile(p: camera_profile_t): void;
|
|
27
|
+
setPreferredResolution(w: number, h: number): void;
|
|
25
28
|
private _stop;
|
|
26
29
|
pause(): void;
|
|
27
30
|
start(): void;
|
|
@@ -8,6 +8,8 @@ export declare class MSTPCameraSource extends Source {
|
|
|
8
8
|
static DEFAULT_DEVICE_ID: string;
|
|
9
9
|
private _currentStream;
|
|
10
10
|
private _activeDeviceId;
|
|
11
|
+
private _preferredWidth;
|
|
12
|
+
private _preferredHeight;
|
|
11
13
|
private _isPaused;
|
|
12
14
|
private _isUserFacing;
|
|
13
15
|
private _cameraToScreenRotation;
|
|
@@ -20,6 +22,7 @@ export declare class MSTPCameraSource extends Source {
|
|
|
20
22
|
private _stop;
|
|
21
23
|
pause(): void;
|
|
22
24
|
start(): void;
|
|
25
|
+
setPreferredResolution(w: number, h: number): void;
|
|
23
26
|
private _getConstraints;
|
|
24
27
|
getFrame(allowRead: boolean): void;
|
|
25
28
|
private _getUserMedia;
|
package/lib/src/profile.d.ts
CHANGED
|
@@ -11,7 +11,6 @@ export declare let profile: {
|
|
|
11
11
|
requestHighFrameRate: boolean;
|
|
12
12
|
videoWidth: number;
|
|
13
13
|
videoHeight: number;
|
|
14
|
-
getDataSize: (p: camera_profile_t) => number[];
|
|
15
14
|
videoElementInDOM: boolean;
|
|
16
15
|
preferMediaStreamTrackProcessorCamera: boolean;
|
|
17
16
|
preferImageBitmapCamera: boolean;
|
|
@@ -22,3 +21,5 @@ export declare let profile: {
|
|
|
22
21
|
trustSensorIntervals: boolean;
|
|
23
22
|
offscreenCanvasSupported: boolean;
|
|
24
23
|
};
|
|
24
|
+
export declare function getDataSize(p: camera_profile_t, fullWidth: number, fullHeight: number): [number, number];
|
|
25
|
+
export declare function getDataSizeForSource(p: camera_profile_t, source: HTMLVideoElement | HTMLImageElement | ImageBitmap, rotation?: number): [number, number];
|
package/lib/src/version.d.ts
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
export declare const VERSION = "3.
|
|
1
|
+
export declare const VERSION = "3.11.0";
|
|
@@ -47,7 +47,11 @@ export declare class WebXRCameraSource extends Source {
|
|
|
47
47
|
private _cameraToDeviceTransform;
|
|
48
48
|
private _cameraModel;
|
|
49
49
|
private _rawCameraModel;
|
|
50
|
+
private _rawCameraWidth;
|
|
51
|
+
private _rawCameraHeight;
|
|
50
52
|
private _profile;
|
|
53
|
+
private _preferredWidth;
|
|
54
|
+
private _preferredHeight;
|
|
51
55
|
private _permissionRequestShown;
|
|
52
56
|
private _planeHitTestSource;
|
|
53
57
|
private _customAnchorSetRequests;
|
|
@@ -57,6 +61,7 @@ export declare class WebXRCameraSource extends Source {
|
|
|
57
61
|
constructor(_impl: zappar_camera_source_t, _pipeline: zappar_pipeline_t);
|
|
58
62
|
destroy(): void;
|
|
59
63
|
setProfile(p: camera_profile_t): void;
|
|
64
|
+
setPreferredResolution(w: number, h: number): void;
|
|
60
65
|
private _stop;
|
|
61
66
|
pause(): void;
|
|
62
67
|
start(): void;
|
package/lib/tests/common.d.ts
CHANGED
|
@@ -1,9 +1,11 @@
|
|
|
1
|
-
import { Zappar, zappar_pipeline_t } from "../src";
|
|
1
|
+
import { Zappar, zappar_pipeline_t, zappar_camera_source_t } from "../src";
|
|
2
2
|
import { zappar_sequence_source_t } from "../src/gen/zappar";
|
|
3
3
|
import { Event1 } from "../src/event";
|
|
4
4
|
export declare let canvas: HTMLCanvasElement;
|
|
5
5
|
export declare let mirrorPoses: boolean;
|
|
6
6
|
export declare let pipeline: zappar_pipeline_t;
|
|
7
|
+
export declare let rearSource: zappar_camera_source_t;
|
|
8
|
+
export declare let frontSource: zappar_camera_source_t;
|
|
7
9
|
export declare let onSequenceLoaded: Event1<zappar_sequence_source_t>;
|
|
8
10
|
export declare function setup(webgl2?: boolean): Zappar;
|
|
9
11
|
export declare function currentPoseSetting(): PoseSetting;
|
package/lib/version.d.ts
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
export declare const VERSION = "3.
|
|
1
|
+
export declare const VERSION = "3.11.0";
|
package/lib/version.js
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
export const VERSION = "3.
|
|
1
|
+
export const VERSION = "3.11.0";
|
|
@@ -47,7 +47,11 @@ export declare class WebXRCameraSource extends Source {
|
|
|
47
47
|
private _cameraToDeviceTransform;
|
|
48
48
|
private _cameraModel;
|
|
49
49
|
private _rawCameraModel;
|
|
50
|
+
private _rawCameraWidth;
|
|
51
|
+
private _rawCameraHeight;
|
|
50
52
|
private _profile;
|
|
53
|
+
private _preferredWidth;
|
|
54
|
+
private _preferredHeight;
|
|
51
55
|
private _permissionRequestShown;
|
|
52
56
|
private _planeHitTestSource;
|
|
53
57
|
private _customAnchorSetRequests;
|
|
@@ -57,6 +61,7 @@ export declare class WebXRCameraSource extends Source {
|
|
|
57
61
|
constructor(_impl: zappar_camera_source_t, _pipeline: zappar_pipeline_t);
|
|
58
62
|
destroy(): void;
|
|
59
63
|
setProfile(p: camera_profile_t): void;
|
|
64
|
+
setPreferredResolution(w: number, h: number): void;
|
|
60
65
|
private _stop;
|
|
61
66
|
pause(): void;
|
|
62
67
|
start(): void;
|
|
@@ -8,7 +8,7 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
|
|
|
8
8
|
});
|
|
9
9
|
};
|
|
10
10
|
import { anchor_status_t, world_tracker_quality_t, camera_profile_t } from "./gen/zappar";
|
|
11
|
-
import { profile } from "./profile";
|
|
11
|
+
import { profile, getDataSizeForSource } from "./profile";
|
|
12
12
|
import { Pipeline } from "./pipeline";
|
|
13
13
|
import { Source } from "./source";
|
|
14
14
|
import { cameraRotationForScreenOrientation } from "./cameramodel";
|
|
@@ -50,13 +50,17 @@ export class WebXRCameraSource extends Source {
|
|
|
50
50
|
this._cameraToDeviceTransform = mat4.create();
|
|
51
51
|
this._cameraModel = new Float32Array([300, 300, 160, 120, 0, 0]);
|
|
52
52
|
this._rawCameraModel = new Float32Array([300, 300, 160, 120, 0, 0]);
|
|
53
|
+
this._rawCameraWidth = 320;
|
|
54
|
+
this._rawCameraHeight = 240;
|
|
53
55
|
this._profile = camera_profile_t.DEFAULT;
|
|
56
|
+
this._preferredWidth = profile.videoWidth;
|
|
57
|
+
this._preferredHeight = profile.videoHeight;
|
|
54
58
|
this._permissionRequestShown = false;
|
|
55
59
|
this._planeHitTestSource = null;
|
|
56
60
|
this._customAnchorSetRequests = new Map();
|
|
57
61
|
this._customAnchors = new Map();
|
|
58
62
|
this._onXRFrame = (time, frame) => {
|
|
59
|
-
var _a, _b, _c, _d, _e
|
|
63
|
+
var _a, _b, _c, _d, _e;
|
|
60
64
|
if (this._isPaused)
|
|
61
65
|
return;
|
|
62
66
|
(_a = this._xrSession) === null || _a === void 0 ? void 0 : _a.requestAnimationFrame(this._onXRFrame);
|
|
@@ -80,16 +84,19 @@ export class WebXRCameraSource extends Source {
|
|
|
80
84
|
}
|
|
81
85
|
if (!texture)
|
|
82
86
|
return;
|
|
83
|
-
|
|
84
|
-
this._updateCameraModel(camView, viewport);
|
|
87
|
+
this._updateCameraModel(camView, camera);
|
|
85
88
|
const localPose = frame.getViewerPose(this._xrLocalSpace);
|
|
89
|
+
let rot = cameraRotationForScreenOrientation(false);
|
|
90
|
+
let isPortrait = (rot == 90) || (rot == 270);
|
|
86
91
|
const gl = this._offscreenContext;
|
|
87
92
|
let blitWidth = camera.width;
|
|
88
93
|
let blitHeight = camera.height;
|
|
89
|
-
let
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
94
|
+
let prefWidth = isPortrait ? this._preferredHeight : this._preferredWidth;
|
|
95
|
+
let prefHeight = isPortrait ? this._preferredWidth : this._preferredHeight;
|
|
96
|
+
let scale = Math.min(prefWidth / blitWidth, prefHeight / blitHeight);
|
|
97
|
+
if (scale < 1) {
|
|
98
|
+
blitWidth = Math.round(scale * blitWidth);
|
|
99
|
+
blitHeight = Math.round(scale * blitHeight);
|
|
93
100
|
}
|
|
94
101
|
if (blitWidth != this._offscreenCanvas.width || blitHeight != this._offscreenCanvas.height) {
|
|
95
102
|
this._offscreenCanvas.width = blitWidth;
|
|
@@ -103,7 +110,6 @@ export class WebXRCameraSource extends Source {
|
|
|
103
110
|
if (this._latestFrame)
|
|
104
111
|
this._latestFrame.close();
|
|
105
112
|
this._latestFrame = this._offscreenCanvas.transferToImageBitmap();
|
|
106
|
-
let rot = cameraRotationForScreenOrientation(false);
|
|
107
113
|
const rotateMatrix = (m, rot) => {
|
|
108
114
|
let ret = m.slice();
|
|
109
115
|
if (rot === 90) {
|
|
@@ -147,7 +153,7 @@ export class WebXRCameraSource extends Source {
|
|
|
147
153
|
mat4.getRotation(_quat, req.pose);
|
|
148
154
|
mat4.getTranslation(_trans, req.pose);
|
|
149
155
|
const rigid = new XRRigidTransform({ x: _trans[0], y: _trans[1], z: _trans[2], w: 1 }, { x: _quat[0], y: _quat[1], z: _quat[2], w: _quat[3] });
|
|
150
|
-
(
|
|
156
|
+
(_c = (_b = frame.createAnchor) === null || _b === void 0 ? void 0 : _b.call(frame, rigid, this._xrLocalSpace)) === null || _c === void 0 ? void 0 : _c.then(anchor => {
|
|
151
157
|
const existing = this._customAnchors.get(customAnchor);
|
|
152
158
|
if (existing)
|
|
153
159
|
existing.anchor.delete();
|
|
@@ -235,7 +241,7 @@ export class WebXRCameraSource extends Source {
|
|
|
235
241
|
custom_anchors.push({
|
|
236
242
|
id: o,
|
|
237
243
|
status: frame.trackedAnchors.has(anchor.anchor) ? anchor_status_t.ANCHOR_STATUS_TRACKING : anchor_status_t.ANCHOR_STATUS_PAUSED,
|
|
238
|
-
pose: rotateMatrix((
|
|
244
|
+
pose: rotateMatrix((_d = pose === null || pose === void 0 ? void 0 : pose.transform) === null || _d === void 0 ? void 0 : _d.matrix, rot),
|
|
239
245
|
poseVersion: anchor.version
|
|
240
246
|
});
|
|
241
247
|
}
|
|
@@ -248,7 +254,7 @@ export class WebXRCameraSource extends Source {
|
|
|
248
254
|
else {
|
|
249
255
|
const world_pose = rotateMatrix(localPose.transform.inverse.matrix, rot);
|
|
250
256
|
let ground_pose = mat4.create();
|
|
251
|
-
mat4.fromTranslation(_ground, [0, (
|
|
257
|
+
mat4.fromTranslation(_ground, [0, (_e = this._groundH) !== null && _e !== void 0 ? _e : -1.5, 0]);
|
|
252
258
|
mat4.multiply(ground_pose, localPose.transform.inverse.matrix, _ground);
|
|
253
259
|
ground_pose = rotateMatrix(ground_pose, rot);
|
|
254
260
|
this._latestWtData = {
|
|
@@ -353,6 +359,10 @@ export class WebXRCameraSource extends Source {
|
|
|
353
359
|
setProfile(p) {
|
|
354
360
|
this._profile = p;
|
|
355
361
|
}
|
|
362
|
+
setPreferredResolution(w, h) {
|
|
363
|
+
this._preferredWidth = w;
|
|
364
|
+
this._preferredHeight = h;
|
|
365
|
+
}
|
|
356
366
|
_stop() {
|
|
357
367
|
if (!this._xrSession)
|
|
358
368
|
return;
|
|
@@ -401,25 +411,37 @@ export class WebXRCameraSource extends Source {
|
|
|
401
411
|
let pipeline = Pipeline.get(this._pipeline);
|
|
402
412
|
if (!pipeline)
|
|
403
413
|
return;
|
|
404
|
-
let
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
414
|
+
let [dataWidth, dataHeight] = getDataSizeForSource(this._profile, imageBitmap, rotation);
|
|
415
|
+
// Calculate the raw camera model after rotation to primary landscape for zcv
|
|
416
|
+
const isPortrait = (rotation == 90) || (rotation == 270);
|
|
417
|
+
const rawWidth = isPortrait ? this._rawCameraHeight : this._rawCameraWidth;
|
|
418
|
+
const rawHeight = isPortrait ? this._rawCameraWidth : this._rawCameraHeight;
|
|
419
|
+
this._cameraModel[0] = isPortrait ? this._rawCameraModel[1] : this._rawCameraModel[0];
|
|
420
|
+
this._cameraModel[1] = isPortrait ? this._rawCameraModel[0] : this._rawCameraModel[1];
|
|
421
|
+
if (rotation === 90) {
|
|
422
|
+
this._cameraModel[2] = rawWidth - this._rawCameraModel[3];
|
|
423
|
+
this._cameraModel[3] = this._rawCameraModel[2];
|
|
424
|
+
}
|
|
425
|
+
else if (rotation === 180) {
|
|
426
|
+
this._cameraModel[2] = rawWidth - this._rawCameraModel[2];
|
|
427
|
+
this._cameraModel[3] = rawHeight - this._rawCameraModel[3];
|
|
428
|
+
}
|
|
429
|
+
else if (rotation === 270) {
|
|
430
|
+
this._cameraModel[2] = this._rawCameraModel[3];
|
|
431
|
+
this._cameraModel[3] = rawHeight - this._rawCameraModel[2];
|
|
411
432
|
}
|
|
412
433
|
else {
|
|
413
|
-
|
|
434
|
+
this._cameraModel[2] = this._rawCameraModel[2];
|
|
435
|
+
this._cameraModel[3] = this._rawCameraModel[3];
|
|
414
436
|
}
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
this._cameraModel[
|
|
420
|
-
|
|
421
|
-
this._cameraModel[2]
|
|
422
|
-
this._cameraModel[3]
|
|
437
|
+
// Apply the scaling to "data" pixels rather than "raw" pixels
|
|
438
|
+
this._cameraModel[0] *= (dataWidth / rawWidth);
|
|
439
|
+
this._cameraModel[1] *= (dataHeight / rawHeight);
|
|
440
|
+
this._cameraModel[2] *= (dataWidth / rawWidth);
|
|
441
|
+
this._cameraModel[3] *= (dataHeight / rawHeight);
|
|
442
|
+
// Offset for pixel-centered coords assumption in zcv
|
|
443
|
+
this._cameraModel[2] -= 0.5;
|
|
444
|
+
this._cameraModel[3] -= 0.5;
|
|
423
445
|
let token = pipeline.registerToken({
|
|
424
446
|
dataWidth: imageBitmap.width,
|
|
425
447
|
dataHeight: imageBitmap.height,
|
|
@@ -516,10 +538,11 @@ export class WebXRCameraSource extends Source {
|
|
|
516
538
|
let texture = pipeline.getVideoTexture();
|
|
517
539
|
if (!texture)
|
|
518
540
|
return;
|
|
541
|
+
const previousBoundTexture = gl.getParameter(gl.TEXTURE_BINDING_2D);
|
|
519
542
|
gl.bindTexture(gl.TEXTURE_2D, texture);
|
|
520
543
|
gl.pixelStorei(gl.UNPACK_FLIP_Y_WEBGL, false);
|
|
521
544
|
gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, info.frame);
|
|
522
|
-
gl.bindTexture(gl.TEXTURE_2D,
|
|
545
|
+
gl.bindTexture(gl.TEXTURE_2D, previousBoundTexture);
|
|
523
546
|
info.texture = texture;
|
|
524
547
|
}
|
|
525
548
|
setCustomAnchorPose(o, version, pose) {
|
|
@@ -533,18 +556,27 @@ export class WebXRCameraSource extends Source {
|
|
|
533
556
|
(_a = this._customAnchors.get(o)) === null || _a === void 0 ? void 0 : _a.anchor.delete();
|
|
534
557
|
});
|
|
535
558
|
}
|
|
536
|
-
_updateCameraModel(camView,
|
|
559
|
+
_updateCameraModel(camView, camera) {
|
|
537
560
|
const p = camView.projectionMatrix;
|
|
538
|
-
|
|
539
|
-
|
|
561
|
+
// Principal points
|
|
562
|
+
// NB: The v0 expression is different from that given (as of 2026-01-16) in
|
|
563
|
+
// https://github.com/immersive-web/raw-camera-access/blob/main/explainer.md
|
|
564
|
+
// Simon re-derived the expressions considering v0 was measured from the top
|
|
565
|
+
// of the image. So the sign change vs the expression for u0 is expected.
|
|
566
|
+
// Chrome incorrectly always returns projection matrices centered on the
|
|
567
|
+
// viewport, with p[8] and p[9] zero, but this will make a difference if that
|
|
568
|
+
// gets fixed.
|
|
569
|
+
const u0 = (1 - p[8]) * camera.width / 2;
|
|
570
|
+
const v0 = (1 + p[9]) * camera.height / 2;
|
|
540
571
|
// Focal lengths in pixels (these are equal for square pixels)
|
|
541
|
-
const ax =
|
|
542
|
-
const ay =
|
|
572
|
+
const ax = camera.width / 2 * p[0];
|
|
573
|
+
const ay = camera.height / 2 * p[5];
|
|
543
574
|
this._rawCameraModel[0] = ax;
|
|
544
575
|
this._rawCameraModel[1] = ay;
|
|
545
|
-
// Principal point in pixels
|
|
546
576
|
this._rawCameraModel[2] = u0;
|
|
547
577
|
this._rawCameraModel[3] = v0;
|
|
578
|
+
this._rawCameraWidth = camera.width;
|
|
579
|
+
this._rawCameraHeight = camera.height;
|
|
548
580
|
}
|
|
549
581
|
_isPlaneVertical(pose) {
|
|
550
582
|
_quat[0] = pose.transform.orientation.x;
|
|
@@ -1,5 +1,4 @@
|
|
|
1
1
|
import { ImageProcessGL } from "./image-process-gl";
|
|
2
|
-
import { profile } from "./profile";
|
|
3
2
|
import { mat4 } from "gl-matrix";
|
|
4
3
|
let pixels;
|
|
5
4
|
let texture;
|
|
@@ -34,23 +33,12 @@ export function handleImageBitmap(m, r, server, mgr) {
|
|
|
34
33
|
}
|
|
35
34
|
if (!texture)
|
|
36
35
|
return;
|
|
37
|
-
let [dataWidth, dataHeight] =
|
|
38
|
-
let isPortrait = (m.r == 90) || (m.r == 270);
|
|
39
|
-
let ibWidth = isPortrait ? m.i.height : m.i.width;
|
|
40
|
-
let ibHeight = isPortrait ? m.i.width : m.i.height;
|
|
41
|
-
let cropHeight = Math.round(dataWidth * ibHeight / ibWidth);
|
|
42
|
-
if (cropHeight < dataHeight) {
|
|
43
|
-
dataHeight = cropHeight;
|
|
44
|
-
}
|
|
45
|
-
else {
|
|
46
|
-
dataWidth = Math.round(dataHeight * ibWidth / ibHeight);
|
|
47
|
-
}
|
|
36
|
+
let [dataWidth, dataHeight] = imageProcessor.uploadFrame(texture, m.i, m.r, m.userFacing, m.cp);
|
|
48
37
|
if (!pixels || pixels.byteLength < dataWidth * dataHeight) {
|
|
49
38
|
// console.log("Generating pixel buffer", profile.dataWidth * profile.dataHeight);
|
|
50
39
|
pixels = new ArrayBuffer(dataWidth * dataHeight);
|
|
51
40
|
}
|
|
52
|
-
|
|
53
|
-
let info = imageProcessor.readFrameForSize(texture, pixels, dataWidth, dataHeight);
|
|
41
|
+
let info = imageProcessor.readFrame(pixels);
|
|
54
42
|
let msg = {
|
|
55
43
|
t: "imageBitmapS2C",
|
|
56
44
|
dataWidth: info.dataWidth,
|