@zappar/zappar-cv 3.0.1-alpha.1 → 3.0.1-alpha.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/lib/additional.d.ts +8 -1
- package/lib/camera-frame-data.d.ts +5 -0
- package/lib/camera-frame-data.js +1 -0
- package/lib/data-download.d.ts +8 -0
- package/lib/data-download.js +8 -0
- package/lib/deserializer.d.ts +1 -0
- package/lib/deserializer.js +7 -0
- package/lib/direct.js +1 -1
- package/lib/drawgrid.d.ts +2 -0
- package/lib/drawgrid.js +136 -0
- package/lib/drawpoints.d.ts +11 -0
- package/lib/drawpoints.js +180 -0
- package/lib/gen/zappar-bridge.d.ts +141 -0
- package/lib/gen/zappar-bridge.js +8 -0
- package/lib/gen/zappar-client.d.ts +3 -2
- package/lib/gen/zappar-client.js +214 -18
- package/lib/gen/zappar-cwrap.js +179 -3
- package/lib/gen/zappar-native.d.ts +29 -1
- package/lib/gen/zappar-native.js +6 -0
- package/lib/gen/zappar-server.d.ts +3 -0
- package/lib/gen/zappar-server.js +108 -10
- package/lib/gen/zappar.d.ts +26 -0
- package/lib/gen/zappar.js +1 -0
- package/lib/html-element-source.d.ts +3 -1
- package/lib/html-element-source.js +13 -6
- package/lib/image-process-gl.d.ts +3 -2
- package/lib/image-process-gl.js +9 -7
- package/lib/imagebitmap-camera-source.d.ts +3 -1
- package/lib/imagebitmap-camera-source.js +10 -4
- package/lib/imagetracker.d.ts +3 -2
- package/lib/index.d.ts +2 -1
- package/lib/index.js +1 -1
- package/lib/mstp-camera-source.d.ts +2 -1
- package/lib/mstp-camera-source.js +5 -0
- package/lib/native.js +67 -3
- package/lib/pipeline.d.ts +14 -4
- package/lib/pipeline.js +84 -5
- package/lib/profile.d.ts +2 -2
- package/lib/profile.js +3 -3
- package/lib/serializer.d.ts +1 -0
- package/lib/serializer.js +1 -0
- package/lib/source.d.ts +1 -0
- package/lib/version.d.ts +1 -1
- package/lib/version.js +1 -1
- package/lib/worker-client.js +5 -1
- package/lib/worker-imagebitmap.js +5 -4
- package/lib/worker-server.d.ts +1 -1
- package/lib/worker-server.js +84 -12
- package/lib/worker.js +2 -1
- package/lib/workerinterface.d.ts +24 -1
- package/lib/zappar-cv.js +1 -174
- package/lib/zappar-cv.wasm +0 -0
- package/package.json +1 -1
- package/umd/751.zappar-cv.js +1 -0
- package/umd/{501114eb92cddf5f56fe.wasm → 7a7455e00ac55a4f34d3.wasm} +0 -0
- package/umd/867.zappar-cv.js +1 -1
- package/umd/zappar-cv.js +1 -1
- package/umd/zappar-cv.worker.js +1 -1
- package/umd/429.zappar-cv.js +0 -1
package/lib/gen/zappar-server.js
CHANGED
|
@@ -24,6 +24,8 @@ export class zappar_server {
|
|
|
24
24
|
this._barcode_finder_by_instance = new Map();
|
|
25
25
|
this._pipeline_id_by_instant_world_tracker_id = new Map();
|
|
26
26
|
this._instant_world_tracker_by_instance = new Map();
|
|
27
|
+
this._pipeline_id_by_zapcode_tracker_id = new Map();
|
|
28
|
+
this._zapcode_tracker_by_instance = new Map();
|
|
27
29
|
this._pipeline_id_by_world_tracker_id = new Map();
|
|
28
30
|
this._world_tracker_by_instance = new Map();
|
|
29
31
|
}
|
|
@@ -351,6 +353,40 @@ export class zappar_server {
|
|
|
351
353
|
break;
|
|
352
354
|
}
|
|
353
355
|
case 41: {
|
|
356
|
+
let clientId = msg.type();
|
|
357
|
+
let arg_pipeline_id = msg.type();
|
|
358
|
+
let arg_pipeline = this._pipeline_by_instance.get(arg_pipeline_id);
|
|
359
|
+
let handle = this._impl.zapcode_tracker_create(arg_pipeline);
|
|
360
|
+
this._zapcode_tracker_by_instance.set(clientId, handle);
|
|
361
|
+
this._pipeline_id_by_zapcode_tracker_id.set(clientId, arg_pipeline_id);
|
|
362
|
+
break;
|
|
363
|
+
}
|
|
364
|
+
case 44: {
|
|
365
|
+
let clientId = msg.type();
|
|
366
|
+
let obj = this._zapcode_tracker_by_instance.get(clientId);
|
|
367
|
+
if (obj === undefined)
|
|
368
|
+
return;
|
|
369
|
+
this._impl.zapcode_tracker_destroy(obj);
|
|
370
|
+
this._zapcode_tracker_by_instance.delete(clientId);
|
|
371
|
+
break;
|
|
372
|
+
}
|
|
373
|
+
case 43: {
|
|
374
|
+
let clientId = msg.type();
|
|
375
|
+
let obj = this._zapcode_tracker_by_instance.get(clientId);
|
|
376
|
+
if (obj === undefined)
|
|
377
|
+
return;
|
|
378
|
+
this._impl.zapcode_tracker_target_load_from_memory(obj, msg.dataWithLength());
|
|
379
|
+
break;
|
|
380
|
+
}
|
|
381
|
+
case 42: {
|
|
382
|
+
let clientId = msg.type();
|
|
383
|
+
let obj = this._zapcode_tracker_by_instance.get(clientId);
|
|
384
|
+
if (obj === undefined)
|
|
385
|
+
return;
|
|
386
|
+
this._impl.zapcode_tracker_enabled_set(obj, msg.bool());
|
|
387
|
+
break;
|
|
388
|
+
}
|
|
389
|
+
case 45: {
|
|
354
390
|
let clientId = msg.type();
|
|
355
391
|
let arg_pipeline_id = msg.type();
|
|
356
392
|
let arg_pipeline = this._pipeline_by_instance.get(arg_pipeline_id);
|
|
@@ -359,7 +395,7 @@ export class zappar_server {
|
|
|
359
395
|
this._pipeline_id_by_world_tracker_id.set(clientId, arg_pipeline_id);
|
|
360
396
|
break;
|
|
361
397
|
}
|
|
362
|
-
case
|
|
398
|
+
case 46: {
|
|
363
399
|
let clientId = msg.type();
|
|
364
400
|
let obj = this._world_tracker_by_instance.get(clientId);
|
|
365
401
|
if (obj === undefined)
|
|
@@ -368,7 +404,7 @@ export class zappar_server {
|
|
|
368
404
|
this._world_tracker_by_instance.delete(clientId);
|
|
369
405
|
break;
|
|
370
406
|
}
|
|
371
|
-
case
|
|
407
|
+
case 47: {
|
|
372
408
|
let clientId = msg.type();
|
|
373
409
|
let obj = this._world_tracker_by_instance.get(clientId);
|
|
374
410
|
if (obj === undefined)
|
|
@@ -376,7 +412,7 @@ export class zappar_server {
|
|
|
376
412
|
this._impl.world_tracker_enabled_set(obj, msg.bool());
|
|
377
413
|
break;
|
|
378
414
|
}
|
|
379
|
-
case
|
|
415
|
+
case 48: {
|
|
380
416
|
let clientId = msg.type();
|
|
381
417
|
let obj = this._world_tracker_by_instance.get(clientId);
|
|
382
418
|
if (obj === undefined)
|
|
@@ -384,6 +420,22 @@ export class zappar_server {
|
|
|
384
420
|
this._impl.world_tracker_reset(obj);
|
|
385
421
|
break;
|
|
386
422
|
}
|
|
423
|
+
case 49: {
|
|
424
|
+
let clientId = msg.type();
|
|
425
|
+
let obj = this._world_tracker_by_instance.get(clientId);
|
|
426
|
+
if (obj === undefined)
|
|
427
|
+
return;
|
|
428
|
+
this._impl.world_tracker_tracks_data_enabled_set(obj, msg.bool());
|
|
429
|
+
break;
|
|
430
|
+
}
|
|
431
|
+
case 50: {
|
|
432
|
+
let clientId = msg.type();
|
|
433
|
+
let obj = this._world_tracker_by_instance.get(clientId);
|
|
434
|
+
if (obj === undefined)
|
|
435
|
+
return;
|
|
436
|
+
this._impl.world_tracker_projections_data_enabled_set(obj, msg.bool());
|
|
437
|
+
break;
|
|
438
|
+
}
|
|
387
439
|
}
|
|
388
440
|
});
|
|
389
441
|
}
|
|
@@ -560,6 +612,36 @@ export class zappar_server {
|
|
|
560
612
|
msg.matrix4x4(this._impl.instant_world_tracker_anchor_pose_raw(v));
|
|
561
613
|
});
|
|
562
614
|
}
|
|
615
|
+
for (let [k, v] of this._zapcode_tracker_by_instance) {
|
|
616
|
+
let pipeline = this._pipeline_id_by_zapcode_tracker_id.get(k);
|
|
617
|
+
if (!pipeline)
|
|
618
|
+
continue;
|
|
619
|
+
let serializer = this.serializersByPipelineId.get(pipeline);
|
|
620
|
+
if (!serializer)
|
|
621
|
+
continue;
|
|
622
|
+
serializer.sendMessage(24, msg => {
|
|
623
|
+
msg.type(k);
|
|
624
|
+
msg.int(this._impl.zapcode_tracker_target_loaded_version(v));
|
|
625
|
+
});
|
|
626
|
+
serializer.sendMessage(21, msg => {
|
|
627
|
+
msg.type(k);
|
|
628
|
+
msg.int(this._impl.zapcode_tracker_anchor_count(v));
|
|
629
|
+
});
|
|
630
|
+
for (let i = 0; i < this._impl.zapcode_tracker_anchor_count(v); i++) {
|
|
631
|
+
serializer.sendMessage(22, msg => {
|
|
632
|
+
msg.type(k);
|
|
633
|
+
msg.int(i);
|
|
634
|
+
msg.string(this._impl.zapcode_tracker_anchor_id(v, i));
|
|
635
|
+
});
|
|
636
|
+
}
|
|
637
|
+
for (let i = 0; i < this._impl.zapcode_tracker_anchor_count(v); i++) {
|
|
638
|
+
serializer.sendMessage(23, msg => {
|
|
639
|
+
msg.type(k);
|
|
640
|
+
msg.int(i);
|
|
641
|
+
msg.matrix4x4(this._impl.zapcode_tracker_anchor_pose_raw(v, i));
|
|
642
|
+
});
|
|
643
|
+
}
|
|
644
|
+
}
|
|
563
645
|
for (let [k, v] of this._world_tracker_by_instance) {
|
|
564
646
|
let pipeline = this._pipeline_id_by_world_tracker_id.get(k);
|
|
565
647
|
if (!pipeline)
|
|
@@ -567,37 +649,53 @@ export class zappar_server {
|
|
|
567
649
|
let serializer = this.serializersByPipelineId.get(pipeline);
|
|
568
650
|
if (!serializer)
|
|
569
651
|
continue;
|
|
570
|
-
serializer.sendMessage(
|
|
652
|
+
serializer.sendMessage(31, msg => {
|
|
571
653
|
msg.type(k);
|
|
572
654
|
msg.int(this._impl.world_tracker_quality(v));
|
|
573
655
|
});
|
|
574
|
-
serializer.sendMessage(
|
|
656
|
+
serializer.sendMessage(25, msg => {
|
|
575
657
|
msg.type(k);
|
|
576
658
|
msg.int(this._impl.world_tracker_plane_count(v));
|
|
577
659
|
});
|
|
578
660
|
for (let i = 0; i < this._impl.world_tracker_plane_count(v); i++) {
|
|
579
|
-
serializer.sendMessage(
|
|
661
|
+
serializer.sendMessage(26, msg => {
|
|
580
662
|
msg.type(k);
|
|
581
663
|
msg.int(i);
|
|
582
664
|
msg.matrix4x4(this._impl.world_tracker_plane_pose_raw(v, i));
|
|
583
665
|
});
|
|
584
666
|
}
|
|
585
|
-
serializer.sendMessage(
|
|
667
|
+
serializer.sendMessage(27, msg => {
|
|
586
668
|
msg.type(k);
|
|
587
669
|
msg.bool(this._impl.world_tracker_world_anchor_valid(v));
|
|
588
670
|
});
|
|
589
|
-
serializer.sendMessage(
|
|
671
|
+
serializer.sendMessage(28, msg => {
|
|
590
672
|
msg.type(k);
|
|
591
673
|
msg.matrix4x4(this._impl.world_tracker_world_anchor_pose_raw(v));
|
|
592
674
|
});
|
|
593
|
-
serializer.sendMessage(
|
|
675
|
+
serializer.sendMessage(27, msg => {
|
|
594
676
|
msg.type(k);
|
|
595
677
|
msg.bool(this._impl.world_tracker_ground_anchor_valid(v));
|
|
596
678
|
});
|
|
597
|
-
serializer.sendMessage(
|
|
679
|
+
serializer.sendMessage(30, msg => {
|
|
598
680
|
msg.type(k);
|
|
599
681
|
msg.matrix4x4(this._impl.world_tracker_ground_anchor_pose_raw(v));
|
|
600
682
|
});
|
|
683
|
+
serializer.sendMessage(34, msg => {
|
|
684
|
+
msg.type(k);
|
|
685
|
+
msg.int(this._impl.world_tracker_tracks_data_size(v));
|
|
686
|
+
});
|
|
687
|
+
serializer.sendMessage(33, msg => {
|
|
688
|
+
msg.type(k);
|
|
689
|
+
msg.floatArray(this._impl.world_tracker_tracks_data(v));
|
|
690
|
+
});
|
|
691
|
+
serializer.sendMessage(37, msg => {
|
|
692
|
+
msg.type(k);
|
|
693
|
+
msg.int(this._impl.world_tracker_projections_data_size(v));
|
|
694
|
+
});
|
|
695
|
+
serializer.sendMessage(36, msg => {
|
|
696
|
+
msg.type(k);
|
|
697
|
+
msg.floatArray(this._impl.world_tracker_projections_data(v));
|
|
698
|
+
});
|
|
601
699
|
}
|
|
602
700
|
}
|
|
603
701
|
}
|
package/lib/gen/zappar.d.ts
CHANGED
|
@@ -10,6 +10,8 @@ export { log_level_t } from "./zappar-native";
|
|
|
10
10
|
import { image_target_type_t } from "./zappar-native";
|
|
11
11
|
export { image_target_type_t } from "./zappar-native";
|
|
12
12
|
export { world_tracker_quality_t } from "./zappar-native";
|
|
13
|
+
import { camera_profile_t } from "./zappar-native";
|
|
14
|
+
export { camera_profile_t } from "./zappar-native";
|
|
13
15
|
export declare type zappar_pipeline_t = number & {
|
|
14
16
|
_: 'zappar_pipeline_t';
|
|
15
17
|
};
|
|
@@ -37,6 +39,9 @@ export declare type zappar_barcode_finder_t = number & {
|
|
|
37
39
|
export declare type zappar_instant_world_tracker_t = number & {
|
|
38
40
|
_: 'zappar_instant_world_tracker_t';
|
|
39
41
|
};
|
|
42
|
+
export declare type zappar_zapcode_tracker_t = number & {
|
|
43
|
+
_: 'zappar_zapcode_tracker_t';
|
|
44
|
+
};
|
|
40
45
|
export declare type zappar_world_tracker_t = number & {
|
|
41
46
|
_: 'zappar_world_tracker_t';
|
|
42
47
|
};
|
|
@@ -49,6 +54,7 @@ export interface zappar {
|
|
|
49
54
|
camera_user_facing(indx: number): boolean;
|
|
50
55
|
projection_matrix_from_camera_model(model: Float32Array, renderWidth: number, renderHeight: number): Float32Array;
|
|
51
56
|
projection_matrix_from_camera_model_ext(model: Float32Array, renderWidth: number, renderHeight: number, zNear: number, zFar: number): Float32Array;
|
|
57
|
+
pose_from_raw(raw_pose: Float32Array, mirror: boolean): Float32Array;
|
|
52
58
|
log_level(): log_level_t;
|
|
53
59
|
log_level_set(level: log_level_t): void;
|
|
54
60
|
permission_request_ui(): void;
|
|
@@ -98,6 +104,7 @@ export interface zappar {
|
|
|
98
104
|
camera_source_destroy(o: zappar_camera_source_t): void;
|
|
99
105
|
camera_source_start(o: zappar_camera_source_t): void;
|
|
100
106
|
camera_source_pause(o: zappar_camera_source_t): void;
|
|
107
|
+
camera_source_profile_set(o: zappar_camera_source_t, profile: camera_profile_t): void;
|
|
101
108
|
sequence_source_create(pipeline: zappar_pipeline_t): zappar_sequence_source_t;
|
|
102
109
|
sequence_source_destroy(o: zappar_sequence_source_t): void;
|
|
103
110
|
sequence_source_start(o: zappar_sequence_source_t): void;
|
|
@@ -182,6 +189,17 @@ export interface zappar {
|
|
|
182
189
|
instant_world_tracker_anchor_pose(o: zappar_instant_world_tracker_t, camera_pose: Float32Array, mirror: boolean): Float32Array;
|
|
183
190
|
instant_world_tracker_anchor_pose_set_from_camera_offset_raw(o: zappar_instant_world_tracker_t, x: number, y: number, z: number, orientation: instant_world_tracker_transform_orientation_t): void;
|
|
184
191
|
instant_world_tracker_anchor_pose_set_from_camera_offset(o: zappar_instant_world_tracker_t, x: number, y: number, z: number, orientation: instant_world_tracker_transform_orientation_t): void;
|
|
192
|
+
zapcode_tracker_create(pipeline: zappar_pipeline_t): zappar_zapcode_tracker_t;
|
|
193
|
+
zapcode_tracker_destroy(o: zappar_zapcode_tracker_t): void;
|
|
194
|
+
zapcode_tracker_target_load_from_memory(o: zappar_zapcode_tracker_t, data: ArrayBuffer): void;
|
|
195
|
+
zapcode_tracker_target_loaded_version(o: zappar_zapcode_tracker_t): number;
|
|
196
|
+
zapcode_tracker_enabled(o: zappar_zapcode_tracker_t): boolean;
|
|
197
|
+
zapcode_tracker_enabled_set(o: zappar_zapcode_tracker_t, enabled: boolean): void;
|
|
198
|
+
zapcode_tracker_anchor_count(o: zappar_zapcode_tracker_t): number;
|
|
199
|
+
zapcode_tracker_anchor_id(o: zappar_zapcode_tracker_t, indx: number): string;
|
|
200
|
+
zapcode_tracker_anchor_pose_raw(o: zappar_zapcode_tracker_t, indx: number): Float32Array;
|
|
201
|
+
zapcode_tracker_anchor_pose_camera_relative(o: zappar_zapcode_tracker_t, indx: number, mirror: boolean): Float32Array;
|
|
202
|
+
zapcode_tracker_anchor_pose(o: zappar_zapcode_tracker_t, indx: number, camera_pose: Float32Array, mirror: boolean): Float32Array;
|
|
185
203
|
world_tracker_create(pipeline: zappar_pipeline_t): zappar_world_tracker_t;
|
|
186
204
|
world_tracker_destroy(o: zappar_world_tracker_t): void;
|
|
187
205
|
world_tracker_enabled(o: zappar_world_tracker_t): boolean;
|
|
@@ -200,4 +218,12 @@ export interface zappar {
|
|
|
200
218
|
world_tracker_ground_anchor_pose_camera_relative(o: zappar_world_tracker_t, mirror: boolean): Float32Array;
|
|
201
219
|
world_tracker_ground_anchor_pose(o: zappar_world_tracker_t, camera_pose: Float32Array, mirror: boolean): Float32Array;
|
|
202
220
|
world_tracker_reset(o: zappar_world_tracker_t): void;
|
|
221
|
+
world_tracker_tracks_data_enabled(o: zappar_world_tracker_t): boolean;
|
|
222
|
+
world_tracker_tracks_data_enabled_set(o: zappar_world_tracker_t, tracks_data_enabled: boolean): void;
|
|
223
|
+
world_tracker_tracks_data_size(o: zappar_world_tracker_t): number;
|
|
224
|
+
world_tracker_tracks_data(o: zappar_world_tracker_t): Float32Array;
|
|
225
|
+
world_tracker_projections_data_enabled(o: zappar_world_tracker_t): boolean;
|
|
226
|
+
world_tracker_projections_data_enabled_set(o: zappar_world_tracker_t, projections_data_enabled: boolean): void;
|
|
227
|
+
world_tracker_projections_data_size(o: zappar_world_tracker_t): number;
|
|
228
|
+
world_tracker_projections_data(o: zappar_world_tracker_t): Float32Array;
|
|
203
229
|
}
|
package/lib/gen/zappar.js
CHANGED
|
@@ -5,3 +5,4 @@ export { instant_world_tracker_transform_orientation_t } from "./zappar-native";
|
|
|
5
5
|
export { log_level_t } from "./zappar-native";
|
|
6
6
|
export { image_target_type_t } from "./zappar-native";
|
|
7
7
|
export { world_tracker_quality_t } from "./zappar-native";
|
|
8
|
+
export { camera_profile_t } from "./zappar-native";
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { zappar_pipeline_t } from "./gen/zappar";
|
|
1
|
+
import { camera_profile_t, zappar_pipeline_t } from "./gen/zappar";
|
|
2
2
|
import { Source } from "./source";
|
|
3
3
|
import { zappar_html_element_source_t } from "./additional";
|
|
4
4
|
export declare class HTMLElementSource extends Source {
|
|
@@ -14,6 +14,7 @@ export declare class HTMLElementSource extends Source {
|
|
|
14
14
|
private _cameraToDeviceTransform;
|
|
15
15
|
private _cameraToDeviceTransformUserFacing;
|
|
16
16
|
private _cameraModel;
|
|
17
|
+
private _profile;
|
|
17
18
|
static createVideoElementSource(p: zappar_pipeline_t, element: HTMLVideoElement | HTMLImageElement): zappar_html_element_source_t;
|
|
18
19
|
static getVideoElementSource(m: zappar_html_element_source_t): HTMLElementSource | undefined;
|
|
19
20
|
constructor(_video: HTMLVideoElement | HTMLImageElement, _pipeline: zappar_pipeline_t);
|
|
@@ -24,4 +25,5 @@ export declare class HTMLElementSource extends Source {
|
|
|
24
25
|
getFrame(currentlyProcessing: boolean): void;
|
|
25
26
|
private _processFrame;
|
|
26
27
|
uploadGL(): void;
|
|
28
|
+
setProfile(p: camera_profile_t): void;
|
|
27
29
|
}
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { camera_profile_t } from "./gen/zappar";
|
|
1
2
|
import { Pipeline } from "./pipeline";
|
|
2
3
|
import { Source } from "./source";
|
|
3
4
|
import { profile } from "./profile";
|
|
@@ -19,6 +20,7 @@ export class HTMLElementSource extends Source {
|
|
|
19
20
|
this._cameraToDeviceTransform = mat4.create();
|
|
20
21
|
this._cameraToDeviceTransformUserFacing = mat4.create();
|
|
21
22
|
this._cameraModel = new Float32Array([300, 300, 160, 120, 0, 0]);
|
|
23
|
+
this._profile = camera_profile_t.DEFAULT;
|
|
22
24
|
mat4.fromScaling(this._cameraToDeviceTransformUserFacing, [-1, 1, -1]);
|
|
23
25
|
let video = this._video;
|
|
24
26
|
if (this._video instanceof HTMLVideoElement) {
|
|
@@ -89,6 +91,7 @@ export class HTMLElementSource extends Source {
|
|
|
89
91
|
if (info) {
|
|
90
92
|
let token = pipeline.registerToken(info);
|
|
91
93
|
pipeline.sendDataToWorker(info.data || new ArrayBuffer(0), token, info.dataWidth, info.dataHeight, info.userFacing, info.cameraToDevice, info.cameraModel);
|
|
94
|
+
info.data = undefined;
|
|
92
95
|
}
|
|
93
96
|
}
|
|
94
97
|
catch (ex) {
|
|
@@ -108,14 +111,15 @@ export class HTMLElementSource extends Source {
|
|
|
108
111
|
}
|
|
109
112
|
if (!this._currentVideoTexture)
|
|
110
113
|
return undefined;
|
|
111
|
-
this._imageProcessor.uploadFrame(this._currentVideoTexture, this._video, rotation, this._isUserFacing);
|
|
114
|
+
this._imageProcessor.uploadFrame(this._currentVideoTexture, this._video, rotation, this._isUserFacing, this._profile);
|
|
112
115
|
this._isUploadFrame = !this._isUploadFrame;
|
|
113
116
|
return undefined;
|
|
114
117
|
}
|
|
115
118
|
if (currentlyProcessing || !this._currentVideoTexture)
|
|
116
119
|
return undefined;
|
|
117
120
|
this._isUploadFrame = !this._isUploadFrame;
|
|
118
|
-
|
|
121
|
+
const [dataWidth, dataHeight] = profile.getDataSize(this._profile);
|
|
122
|
+
let greySize = dataWidth * dataHeight;
|
|
119
123
|
let pixels = pipeline.cameraPixelArrays.pop();
|
|
120
124
|
while (pixels) {
|
|
121
125
|
if (pixels.byteLength === greySize)
|
|
@@ -127,14 +131,17 @@ export class HTMLElementSource extends Source {
|
|
|
127
131
|
}
|
|
128
132
|
let tex = this._currentVideoTexture;
|
|
129
133
|
this._currentVideoTexture = undefined;
|
|
130
|
-
let focalLength = 300.0 *
|
|
134
|
+
let focalLength = 300.0 * dataWidth / 320.0;
|
|
131
135
|
this._cameraModel[0] = focalLength;
|
|
132
136
|
this._cameraModel[1] = focalLength;
|
|
133
|
-
this._cameraModel[2] =
|
|
134
|
-
this._cameraModel[3] =
|
|
135
|
-
return Object.assign(Object.assign({}, this._imageProcessor.readFrame(tex, pixels)), { cameraModel: this._cameraModel, cameraSource: this, cameraToDevice: (this._isUserFacing ? this._cameraToDeviceTransformUserFacing : this._cameraToDeviceTransform) });
|
|
137
|
+
this._cameraModel[2] = dataWidth * 0.5;
|
|
138
|
+
this._cameraModel[3] = dataHeight * 0.5;
|
|
139
|
+
return Object.assign(Object.assign({}, this._imageProcessor.readFrame(tex, pixels, this._profile)), { cameraModel: this._cameraModel, cameraSource: this, cameraToDevice: (this._isUserFacing ? this._cameraToDeviceTransformUserFacing : this._cameraToDeviceTransform) });
|
|
136
140
|
}
|
|
137
141
|
uploadGL() {
|
|
138
142
|
// No-op as already uploaded
|
|
139
143
|
}
|
|
144
|
+
setProfile(p) {
|
|
145
|
+
this._profile = p;
|
|
146
|
+
}
|
|
140
147
|
}
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { camera_profile_t } from "./gen/zappar-native";
|
|
1
2
|
export interface ProcessedFrame {
|
|
2
3
|
data?: ArrayBuffer;
|
|
3
4
|
texture: WebGLTexture | undefined;
|
|
@@ -28,8 +29,8 @@ export declare class ImageProcessGL {
|
|
|
28
29
|
constructor(_gl: WebGL2RenderingContext | WebGLRenderingContext);
|
|
29
30
|
resetGLContext(): void;
|
|
30
31
|
destroy(): void;
|
|
31
|
-
uploadFrame(texture: WebGLTexture, img: HTMLVideoElement | HTMLImageElement | ImageBitmap, rotation: number, fc: boolean): void;
|
|
32
|
-
readFrame(texture: WebGLTexture, pixels: ArrayBuffer): ProcessedFrame;
|
|
32
|
+
uploadFrame(texture: WebGLTexture, img: HTMLVideoElement | HTMLImageElement | ImageBitmap, rotation: number, fc: boolean, cp: camera_profile_t): void;
|
|
33
|
+
readFrame(texture: WebGLTexture, pixels: ArrayBuffer, cp: camera_profile_t): ProcessedFrame;
|
|
33
34
|
private _updateTransforms;
|
|
34
35
|
private _getCameraUvTransform;
|
|
35
36
|
private _getFramebuffer;
|
package/lib/image-process-gl.js
CHANGED
|
@@ -33,7 +33,7 @@ export class ImageProcessGL {
|
|
|
33
33
|
destroy() {
|
|
34
34
|
this.resetGLContext();
|
|
35
35
|
}
|
|
36
|
-
uploadFrame(texture, img, rotation, fc) {
|
|
36
|
+
uploadFrame(texture, img, rotation, fc, cp) {
|
|
37
37
|
let gl = this._gl;
|
|
38
38
|
const glStateManager = GLStateManager.get(gl);
|
|
39
39
|
glStateManager.push();
|
|
@@ -75,7 +75,8 @@ export class ImageProcessGL {
|
|
|
75
75
|
if (videoHeight > videoWidth)
|
|
76
76
|
videoHeight = [videoWidth, videoWidth = videoHeight][0];
|
|
77
77
|
this._updateTransforms(rotation, fc);
|
|
78
|
-
|
|
78
|
+
const [dataWidth, dataHeight] = profile.getDataSize(cp);
|
|
79
|
+
let framebuffer = this._getFramebuffer(gl, dataWidth / 4, dataHeight);
|
|
79
80
|
let vbo = this._getVertexBuffer(gl);
|
|
80
81
|
let ibo = this._getIndexBuffer(gl);
|
|
81
82
|
let shader = this._getGreyscaleShader(gl);
|
|
@@ -123,7 +124,7 @@ export class ImageProcessGL {
|
|
|
123
124
|
// Tell WebGL to use our program when drawing
|
|
124
125
|
gl.useProgram(shader.program);
|
|
125
126
|
// Specify greyscale width for the correct offsets, and the uv transform
|
|
126
|
-
gl.uniform1f(shader.uTexWidthLoc,
|
|
127
|
+
gl.uniform1f(shader.uTexWidthLoc, dataWidth);
|
|
127
128
|
gl.uniformMatrix4fv(shader.uUvTransformLoc, false, this._cameraUvTransform);
|
|
128
129
|
gl.activeTexture(gl.TEXTURE0);
|
|
129
130
|
// Bind the texture to texture unit 0
|
|
@@ -167,11 +168,12 @@ export class ImageProcessGL {
|
|
|
167
168
|
if (reenableStencilTest)
|
|
168
169
|
gl.enable(gl.STENCIL_TEST);
|
|
169
170
|
}
|
|
170
|
-
readFrame(texture, pixels) {
|
|
171
|
+
readFrame(texture, pixels, cp) {
|
|
171
172
|
let gl = this._gl;
|
|
172
173
|
let pixelsView = new Uint8Array(pixels);
|
|
173
174
|
const previousBoundFramebuffer = gl.getParameter(gl.FRAMEBUFFER_BINDING);
|
|
174
|
-
|
|
175
|
+
const [dataWidth, dataHeight] = profile.getDataSize(cp);
|
|
176
|
+
let framebuffer = this._getFramebuffer(gl, dataWidth / 4, dataHeight);
|
|
175
177
|
gl.bindFramebuffer(gl.FRAMEBUFFER, framebuffer);
|
|
176
178
|
gl.readPixels(0, 0, this._framebufferWidth, this._framebufferHeight, gl.RGBA, gl.UNSIGNED_BYTE, pixelsView);
|
|
177
179
|
gl.bindFramebuffer(gl.FRAMEBUFFER, previousBoundFramebuffer);
|
|
@@ -179,8 +181,8 @@ export class ImageProcessGL {
|
|
|
179
181
|
uvTransform: this._cameraUvTransform,
|
|
180
182
|
data: pixels,
|
|
181
183
|
texture,
|
|
182
|
-
dataWidth
|
|
183
|
-
dataHeight
|
|
184
|
+
dataWidth,
|
|
185
|
+
dataHeight,
|
|
184
186
|
userFacing: this._computedFrontCameraRotation
|
|
185
187
|
};
|
|
186
188
|
}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { zappar_camera_source_t, zappar_pipeline_t } from "./gen/zappar";
|
|
1
|
+
import { camera_profile_t, zappar_camera_source_t, zappar_pipeline_t } from "./gen/zappar";
|
|
2
2
|
import { CameraFrameInfo, Source } from "./source";
|
|
3
3
|
export declare class ImageBitmapCameraSource extends Source {
|
|
4
4
|
private _impl;
|
|
@@ -18,8 +18,10 @@ export declare class ImageBitmapCameraSource extends Source {
|
|
|
18
18
|
private _cameraToDeviceTransform;
|
|
19
19
|
private _cameraToDeviceTransformUserFacing;
|
|
20
20
|
private _cameraModel;
|
|
21
|
+
private _profile;
|
|
21
22
|
constructor(_impl: zappar_camera_source_t, _pipeline: zappar_pipeline_t, _deviceId: string);
|
|
22
23
|
destroy(): void;
|
|
24
|
+
setProfile(p: camera_profile_t): void;
|
|
23
25
|
private _stop;
|
|
24
26
|
pause(): void;
|
|
25
27
|
start(): void;
|
|
@@ -7,6 +7,7 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
|
|
|
7
7
|
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
8
8
|
});
|
|
9
9
|
};
|
|
10
|
+
import { camera_profile_t } from "./gen/zappar";
|
|
10
11
|
import { profile } from "./profile";
|
|
11
12
|
import { Pipeline } from "./pipeline";
|
|
12
13
|
import { Source } from "./source";
|
|
@@ -32,6 +33,7 @@ export class ImageBitmapCameraSource extends Source {
|
|
|
32
33
|
this._cameraToDeviceTransform = mat4.create();
|
|
33
34
|
this._cameraToDeviceTransformUserFacing = mat4.create();
|
|
34
35
|
this._cameraModel = new Float32Array([300, 300, 160, 120, 0, 0]);
|
|
36
|
+
this._profile = camera_profile_t.DEFAULT;
|
|
35
37
|
this._hasStartedOrientation = false;
|
|
36
38
|
this._lastTimestamp = -1;
|
|
37
39
|
this._deviceMotionListener = (ev) => {
|
|
@@ -85,6 +87,9 @@ export class ImageBitmapCameraSource extends Source {
|
|
|
85
87
|
this.pause();
|
|
86
88
|
deleteCameraSource(this._impl);
|
|
87
89
|
}
|
|
90
|
+
setProfile(p) {
|
|
91
|
+
this._profile = p;
|
|
92
|
+
}
|
|
88
93
|
_stop() {
|
|
89
94
|
if (!this._currentStream)
|
|
90
95
|
return;
|
|
@@ -185,11 +190,12 @@ export class ImageBitmapCameraSource extends Source {
|
|
|
185
190
|
let pipeline = Pipeline.get(this._pipeline);
|
|
186
191
|
if (!pipeline)
|
|
187
192
|
return;
|
|
188
|
-
|
|
193
|
+
const [dataWidth, dataHeight] = profile.getDataSize(this._profile);
|
|
194
|
+
let focalLength = 300.0 * dataWidth / 320.0;
|
|
189
195
|
this._cameraModel[0] = focalLength;
|
|
190
196
|
this._cameraModel[1] = focalLength;
|
|
191
|
-
this._cameraModel[2] =
|
|
192
|
-
this._cameraModel[3] =
|
|
197
|
+
this._cameraModel[2] = dataWidth * 0.5;
|
|
198
|
+
this._cameraModel[3] = dataHeight * 0.5;
|
|
193
199
|
let token = pipeline.registerToken({
|
|
194
200
|
dataWidth: this._canvas.width,
|
|
195
201
|
dataHeight: this._canvas.height,
|
|
@@ -199,7 +205,7 @@ export class ImageBitmapCameraSource extends Source {
|
|
|
199
205
|
cameraModel: this._cameraModel.slice(),
|
|
200
206
|
cameraToDevice: this._isUserFacing ? this._cameraToDeviceTransformUserFacing : this._cameraToDeviceTransform
|
|
201
207
|
});
|
|
202
|
-
(_b = (_a = Pipeline.get(this._pipeline)) === null || _a === void 0 ? void 0 : _a.sendImageBitmapToWorker) === null || _b === void 0 ? void 0 : _b.call(_a, imageBitmap, rotation, this._isUserFacing, token, this._cameraModel, this._isUserFacing ? this._cameraToDeviceTransformUserFacing : this._cameraToDeviceTransform);
|
|
208
|
+
(_b = (_a = Pipeline.get(this._pipeline)) === null || _a === void 0 ? void 0 : _a.sendImageBitmapToWorker) === null || _b === void 0 ? void 0 : _b.call(_a, imageBitmap, rotation, this._isUserFacing, token, this._cameraModel, this._isUserFacing ? this._cameraToDeviceTransformUserFacing : this._cameraToDeviceTransform, this._profile);
|
|
203
209
|
return;
|
|
204
210
|
}
|
|
205
211
|
_getUserMedia() {
|
package/lib/imagetracker.d.ts
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { zappar_image_tracker_t, zappar_pipeline_t } from "./gen/zappar";
|
|
2
|
-
import {
|
|
2
|
+
import { zappar } from "./gen/zappar-bridge";
|
|
3
|
+
import { image_target_type_t } from "./gen/zappar-native";
|
|
3
4
|
interface PreviewInfo {
|
|
4
5
|
compressed: Uint8Array;
|
|
5
6
|
mimeType: string;
|
|
@@ -26,7 +27,7 @@ export declare class ImageTracker {
|
|
|
26
27
|
private _client;
|
|
27
28
|
private _impl;
|
|
28
29
|
private _targets;
|
|
29
|
-
static create(pipeline: zappar_pipeline_t, client:
|
|
30
|
+
static create(pipeline: zappar_pipeline_t, client: zappar): zappar_image_tracker_t;
|
|
30
31
|
static get(p: zappar_image_tracker_t): ImageTracker | undefined;
|
|
31
32
|
private constructor();
|
|
32
33
|
destroy(): void;
|
package/lib/index.d.ts
CHANGED
|
@@ -3,4 +3,5 @@ import { Additional } from "./additional";
|
|
|
3
3
|
import { Options } from "./options";
|
|
4
4
|
export declare type Zappar = zappar & Additional;
|
|
5
5
|
export declare function initialize(opts?: Options): Zappar;
|
|
6
|
-
export {
|
|
6
|
+
export { CameraFrameData } from './camera-frame-data';
|
|
7
|
+
export { zappar_image_tracker_t, zappar_instant_world_tracker_t, zappar_world_tracker_t, zappar_barcode_finder_t, zappar_face_tracker_t, zappar_face_landmark_t, zappar_zapcode_tracker_t, barcode_format_t, face_landmark_name_t, instant_world_tracker_transform_orientation_t, log_level_t, zappar_face_mesh_t, zappar_pipeline_t, zappar_camera_source_t, zappar_sequence_source_t, image_target_type_t, world_tracker_quality_t, camera_profile_t } from "./gen/zappar";
|
package/lib/index.js
CHANGED
|
@@ -4,4 +4,4 @@ export function initialize(opts) {
|
|
|
4
4
|
console.log(`Zappar CV v${VERSION}`);
|
|
5
5
|
return nativeInitialize(opts);
|
|
6
6
|
}
|
|
7
|
-
export { barcode_format_t, face_landmark_name_t, instant_world_tracker_transform_orientation_t, log_level_t, image_target_type_t, world_tracker_quality_t } from "./gen/zappar";
|
|
7
|
+
export { barcode_format_t, face_landmark_name_t, instant_world_tracker_transform_orientation_t, log_level_t, image_target_type_t, world_tracker_quality_t, camera_profile_t } from "./gen/zappar";
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { zappar_camera_source_t, zappar_pipeline_t } from "./gen/zappar";
|
|
1
|
+
import { camera_profile_t, zappar_camera_source_t, zappar_pipeline_t } from "./gen/zappar";
|
|
2
2
|
import { CameraFrameInfo, Source } from "./source";
|
|
3
3
|
export declare class MSTPCameraSource extends Source {
|
|
4
4
|
private _impl;
|
|
@@ -24,6 +24,7 @@ export declare class MSTPCameraSource extends Source {
|
|
|
24
24
|
private _getUserMedia;
|
|
25
25
|
streamEnded(): void;
|
|
26
26
|
private _syncCamera;
|
|
27
|
+
setProfile(profile: camera_profile_t): void;
|
|
27
28
|
private _hasStartedOrientation;
|
|
28
29
|
private _lastTimestamp;
|
|
29
30
|
private _deviceMotionListener;
|
|
@@ -249,6 +249,11 @@ export class MSTPCameraSource extends Source {
|
|
|
249
249
|
}
|
|
250
250
|
});
|
|
251
251
|
}
|
|
252
|
+
setProfile(profile) {
|
|
253
|
+
let p = Pipeline.get(this._pipeline);
|
|
254
|
+
if (p)
|
|
255
|
+
p.sendCameraProfileToWorker(this._impl, profile);
|
|
256
|
+
}
|
|
252
257
|
_startDeviceOrientation() {
|
|
253
258
|
if (this._hasStartedOrientation)
|
|
254
259
|
return;
|