@needle-tools/engine 4.8.7 → 4.8.8-next.857d744

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (65) hide show
  1. package/CHANGELOG.md +14 -0
  2. package/components.needle.json +1 -1
  3. package/dist/{needle-engine.bundle-B0qaChJt.js → needle-engine.bundle-BATGweff.js} +6505 -6452
  4. package/dist/{needle-engine.bundle-CZBThBDy.min.js → needle-engine.bundle-C0MpXHGe.min.js} +137 -137
  5. package/dist/{needle-engine.bundle-lBmpWgFp.umd.cjs → needle-engine.bundle-gnU2UVys.umd.cjs} +138 -138
  6. package/dist/needle-engine.js +2 -2
  7. package/dist/needle-engine.min.js +1 -1
  8. package/dist/needle-engine.umd.cjs +1 -1
  9. package/lib/engine/engine_addressables.d.ts +12 -12
  10. package/lib/engine/engine_addressables.js +30 -23
  11. package/lib/engine/engine_addressables.js.map +1 -1
  12. package/lib/engine/engine_animation.d.ts +1 -3
  13. package/lib/engine/engine_animation.js +15 -9
  14. package/lib/engine/engine_animation.js.map +1 -1
  15. package/lib/engine/engine_camera.d.ts +8 -1
  16. package/lib/engine/engine_camera.js +25 -0
  17. package/lib/engine/engine_camera.js.map +1 -1
  18. package/lib/engine/engine_context.d.ts +9 -0
  19. package/lib/engine/engine_context.js +15 -0
  20. package/lib/engine/engine_context.js.map +1 -1
  21. package/lib/engine/engine_loaders.js +15 -11
  22. package/lib/engine/engine_loaders.js.map +1 -1
  23. package/lib/engine/webcomponents/needle-engine.attributes.d.ts +1 -0
  24. package/lib/engine/webcomponents/needle-engine.js +11 -6
  25. package/lib/engine/webcomponents/needle-engine.js.map +1 -1
  26. package/lib/engine-components/Animation.js +2 -1
  27. package/lib/engine-components/Animation.js.map +1 -1
  28. package/lib/engine-components/AnimationUtilsAutoplay.js +1 -6
  29. package/lib/engine-components/AnimationUtilsAutoplay.js.map +1 -1
  30. package/lib/engine-components/DropListener.d.ts +17 -12
  31. package/lib/engine-components/DropListener.js +34 -31
  32. package/lib/engine-components/DropListener.js.map +1 -1
  33. package/lib/engine-components/LookAtConstraint.d.ts +5 -1
  34. package/lib/engine-components/LookAtConstraint.js +8 -0
  35. package/lib/engine-components/LookAtConstraint.js.map +1 -1
  36. package/lib/engine-components/OrbitControls.d.ts +5 -7
  37. package/lib/engine-components/OrbitControls.js +12 -11
  38. package/lib/engine-components/OrbitControls.js.map +1 -1
  39. package/lib/engine-components/export/usdz/Extension.d.ts +1 -1
  40. package/lib/engine-components/export/usdz/ThreeUSDZExporter.js +1 -1
  41. package/lib/engine-components/export/usdz/ThreeUSDZExporter.js.map +1 -1
  42. package/lib/engine-components/export/usdz/USDZExporter.d.ts +7 -0
  43. package/lib/engine-components/export/usdz/USDZExporter.js +8 -1
  44. package/lib/engine-components/export/usdz/USDZExporter.js.map +1 -1
  45. package/lib/engine-components/webxr/WebXRImageTracking.d.ts +4 -2
  46. package/lib/engine-components/webxr/WebXRImageTracking.js +117 -81
  47. package/lib/engine-components/webxr/WebXRImageTracking.js.map +1 -1
  48. package/package.json +3 -3
  49. package/plugins/vite/alias.js +45 -23
  50. package/src/engine/engine_addressables.ts +44 -33
  51. package/src/engine/engine_animation.ts +17 -9
  52. package/src/engine/engine_camera.ts +40 -1
  53. package/src/engine/engine_context.ts +21 -1
  54. package/src/engine/engine_loaders.ts +18 -13
  55. package/src/engine/webcomponents/needle-engine.attributes.ts +2 -0
  56. package/src/engine/webcomponents/needle-engine.ts +13 -6
  57. package/src/engine-components/Animation.ts +1 -1
  58. package/src/engine-components/AnimationUtilsAutoplay.ts +1 -6
  59. package/src/engine-components/DropListener.ts +40 -31
  60. package/src/engine-components/LookAtConstraint.ts +9 -1
  61. package/src/engine-components/OrbitControls.ts +19 -16
  62. package/src/engine-components/export/usdz/Extension.ts +1 -1
  63. package/src/engine-components/export/usdz/ThreeUSDZExporter.ts +1 -1
  64. package/src/engine-components/export/usdz/USDZExporter.ts +21 -12
  65. package/src/engine-components/webxr/WebXRImageTracking.ts +138 -90
@@ -649,12 +649,12 @@ export class OrbitControls extends Behaviour implements ICameraController {
649
649
  if (this._lookTargetLerpActive) {
650
650
  this._lookTargetLerp01 += this.context.time.deltaTime / this._lookTargetLerpDuration;
651
651
  if (this._lookTargetLerp01 >= 1) {
652
- this._controls.target.copy(this._lookTargetEndPosition);
652
+ this.lerpLookTarget(this._lookTargetEndPosition, this._lookTargetEndPosition, 1);
653
653
  this._lookTargetLerpActive = false;
654
654
  this.dispatchEvent(new CameraTargetReachedEvent(this, "lookat"));
655
655
  } else {
656
656
  const t = Mathf.easeInOutCubic(this._lookTargetLerp01);
657
- this._controls.target.lerpVectors(this._lookTargetStartPosition, this._lookTargetEndPosition, t);
657
+ this.lerpLookTarget(this._lookTargetStartPosition, this._lookTargetEndPosition, t);
658
658
  }
659
659
  }
660
660
 
@@ -729,7 +729,9 @@ export class OrbitControls extends Behaviour implements ICameraController {
729
729
 
730
730
  // this._controls.zoomToCursor = this.zoomToCursor;
731
731
  if (!this.context.isInXR) {
732
- if (!freeCam && this.lookAtConstraint?.locked) this.setLookTargetFromConstraint(0, this.lookAtConstraint01);
732
+ if (!freeCam && this.lookAtConstraint?.locked && !this._lookTargetLerpActive) {
733
+ this.setLookTargetFromConstraint(0, this.lookAtConstraint01);
734
+ }
733
735
  this._controls.update(this.context.time.deltaTime);
734
736
 
735
737
  if (debug) {
@@ -910,7 +912,7 @@ export class OrbitControls extends Behaviour implements ICameraController {
910
912
  }
911
913
 
912
914
  if (immediateOrDuration === true) {
913
- this._controls.target.copy(this._lookTargetEndPosition);
915
+ this.lerpLookTarget(this._lookTargetEndPosition, this._lookTargetEndPosition, 1);
914
916
  }
915
917
  else {
916
918
  this._lookTargetLerpActive = true;
@@ -941,20 +943,18 @@ export class OrbitControls extends Behaviour implements ICameraController {
941
943
  const target = sources[index];
942
944
  if (target) {
943
945
  target.getWorldPosition(this._lookTargetEndPosition);
944
- this.lerpLookTarget(this._lookTargetEndPosition, t);
946
+ this.lerpLookTarget(this._controls.target, this._lookTargetEndPosition, t);
945
947
  return true;
946
948
  }
947
949
  }
948
950
  return false;
949
951
  }
950
952
 
951
- /** @deprecated use `controls.target.lerp(position, delta)` */
952
- public lerpTarget(position: Vector3, delta: number) { return this.lerpLookTarget(position, delta); }
953
-
954
- private lerpLookTarget(position: Vector3, delta: number) {
953
+ private lerpLookTarget(start: Vector3, position: Vector3, t: number) {
955
954
  if (!this._controls) return;
956
- if (delta >= 1) this._controls.target.copy(position);
957
- else this._controls.target.lerp(position, delta);
955
+ if (t >= 1) this._controls.target.copy(position);
956
+ else this._controls.target.lerpVectors(start, position, t);
957
+ if (this.lookAtConstraint) this.lookAtConstraint.setConstraintPosition(this._controls.target);
958
958
  }
959
959
 
960
960
  private setTargetFromRaycast(ray?: Ray, immediateOrDuration: number | boolean = false): boolean {
@@ -1063,7 +1063,7 @@ export class OrbitControls extends Behaviour implements ICameraController {
1063
1063
  return;
1064
1064
  }
1065
1065
 
1066
- const verticalFov = options.fov || camera.fov;
1066
+ const verticalFov = fov;
1067
1067
  const horizontalFov = 2 * Math.atan(Math.tan(verticalFov * Math.PI / 360 / 2) * camera.aspect) / Math.PI * 360;
1068
1068
  const fitHeightDistance = size.y / (2 * Math.atan(Math.PI * verticalFov / 360));
1069
1069
  const fitWidthDistance = size.x / (2 * Math.atan(Math.PI * horizontalFov / 360));
@@ -1161,16 +1161,19 @@ declare type FitCameraOptions = {
1161
1161
  * The objects to fit the camera to. If not provided the scene children will be used
1162
1162
  */
1163
1163
  objects?: Object3D[] | Object3D;
1164
- /** Fit offset: A factor to multiply the distance to the objects by
1165
- * @default 1.1
1166
- */
1167
- fitOffset?: number,
1168
1164
  /** If true the camera will move immediately to the new position, otherwise it will lerp
1169
1165
  * @default false
1170
1166
  */
1171
1167
  immediate?: boolean,
1168
+
1172
1169
  /** If set to "y" the camera will be centered in the y axis */
1173
1170
  centerCamera?: "none" | "y",
1174
1171
  cameraNearFar?: "keep" | "auto",
1172
+
1175
1173
  fov?: number,
1174
+
1175
+ /** Fit offset: A factor to multiply the distance to the objects by
1176
+ * @default 1.1
1177
+ */
1178
+ fitOffset?: number,
1176
1179
  }
@@ -21,5 +21,5 @@ export interface IUSDExporterExtension {
21
21
  onAfterBuildDocument?(context: USDZExporterContext);
22
22
  onExportObject?(object: Object3D, model: USDObject, context: USDZExporterContext);
23
23
  onAfterSerialize?(context: USDZExporterContext);
24
- onAfterHierarchy?(context: USDZExporterContext, writer: USDWriter);
24
+ onAfterHierarchy?(context: USDZExporterContext, writer: USDWriter) : void | Promise<void>;
25
25
  }
@@ -1082,7 +1082,7 @@ async function parseDocument( context: USDZExporterContext, afterStageRoot: () =
1082
1082
  Progress.end("export-usdz-xforms");
1083
1083
 
1084
1084
  Progress.report("export-usdz", "invoke onAfterHierarchy");
1085
- invokeAll( context, 'onAfterHierarchy', writer );
1085
+ await invokeAll( context, 'onAfterHierarchy', writer );
1086
1086
 
1087
1087
  writer.closeBlock();
1088
1088
  writer.closeBlock();
@@ -1,5 +1,5 @@
1
1
  import { NEEDLE_progressive } from "@needle-tools/gltf-progressive";
2
- import { Euler, Matrix4, Mesh, Object3D, Quaternion, Vector3 } from "three";
2
+ import { Euler, EventDispatcher, Matrix4, Mesh, Object3D, Quaternion, Vector3 } from "three";
3
3
 
4
4
  import { isDevEnvironment, showBalloonMessage, showBalloonWarning } from "../../../engine/debug/index.js";
5
5
  import { hasProLicense } from "../../../engine/engine_license.js";
@@ -29,6 +29,7 @@ import { USDZUIExtension } from "./extensions/USDZUI.js";
29
29
  import { USDZExporter as ThreeUSDZExporter } from "./ThreeUSDZExporter.js";
30
30
  import { disableObjectsAtStart, registerAnimatorsImplictly, registerAudioSourcesImplictly } from "./utils/animationutils.js";
31
31
  import { ensureQuicklookLinkIsCreated } from "./utils/quicklook.js";
32
+ import { EventList } from "../../EventList.js";
32
33
 
33
34
  const debug = getParam("debugusdz");
34
35
  const debugUsdzPruning = getParam("debugusdzpruning");
@@ -71,6 +72,9 @@ export class CustomBranding {
71
72
  */
72
73
  export class USDZExporter extends Behaviour {
73
74
 
75
+ static readonly beforeExport = new EventList<{ exporter: USDZExporter }>();
76
+ static readonly afterExport = new EventList<{ exporter: USDZExporter }>();
77
+
74
78
  /**
75
79
  * Assign the object to export as USDZ file. If undefined or null, the whole scene will be exported.
76
80
  */
@@ -210,7 +214,7 @@ export class USDZExporter extends Behaviour {
210
214
  * Creates an USDZ file from the current scene or assigned objectToExport and opens it in QuickLook.
211
215
  * @returns a Promise<Blob> containing the USDZ file
212
216
  */
213
- async exportAndOpen() : Promise<Blob | null> {
217
+ async exportAndOpen(): Promise<Blob | null> {
214
218
 
215
219
  let name = this.exportFileName ?? this.objectToExport?.name ?? this.name;
216
220
  name += "-" + getFormattedDate(); // seems iOS caches the file in some cases, this ensures we always have a fresh file
@@ -221,7 +225,7 @@ export class USDZExporter extends Behaviour {
221
225
  }
222
226
 
223
227
  if (!this.link) this.link = ensureQuicklookLinkIsCreated(this.context, DeviceUtilities.supportsQuickLookAR());
224
-
228
+
225
229
  // ability to specify a custom USDZ file to be used instead of a dynamic one
226
230
  if (this.customUsdzFile) {
227
231
  if (debug) console.log("Exporting custom usdz", this.customUsdzFile)
@@ -234,14 +238,19 @@ export class USDZExporter extends Behaviour {
234
238
  return null;
235
239
  }
236
240
 
237
- const blob = await this.export(this.objectToExport);
241
+ USDZExporter.beforeExport.invoke({ exporter: this });
242
+ const blob = await this.export(this.objectToExport)
243
+ .finally(() => {
244
+ USDZExporter.afterExport.invoke({ exporter: this });
245
+ });
246
+
238
247
  if (!blob) {
239
248
  console.error("USDZ generation failed. Please report a bug", this);
240
249
  return null;
241
250
  }
242
251
 
243
252
  if (debug) console.log("USDZ generation done. Downloading as " + name);
244
-
253
+
245
254
  // TODO Potentially we have to detect QuickLook availability here,
246
255
  // and download the file instead. But browsers keep changing how they deal with non-user-initiated downloads...
247
256
  // https://webkit.org/blog/8421/viewing-augmented-reality-assets-in-safari-for-ios/#:~:text=inside%20the%20anchor.-,Feature%20Detection,-To%20detect%20support
@@ -364,7 +373,7 @@ export class USDZExporter extends Behaviour {
364
373
  if (this.interactive) {
365
374
  defaultExtensions.push(new BehaviorExtension());
366
375
  defaultExtensions.push(new AudioExtension());
367
-
376
+
368
377
  // If physics are enabled, and there are any Rigidbody components in the scene,
369
378
  // add the PhysicsExtension to the default extensions.
370
379
  if (globalThis["NEEDLE_USE_RAPIER"]) {
@@ -632,13 +641,13 @@ export class USDZExporter extends Behaviour {
632
641
  private _rootPositionBeforeExport: Vector3 = new Vector3();
633
642
  private _rootRotationBeforeExport: Quaternion = new Quaternion();
634
643
  private _rootScaleBeforeExport: Vector3 = new Vector3();
635
-
636
- getARScaleAndTarget(): { scale: number, _invertForward: boolean, target: Object3D, sessionRoot: Object3D | null} {
637
- if (!this.objectToExport) return { scale: 1, _invertForward: false, target: this.gameObject, sessionRoot: null};
644
+
645
+ getARScaleAndTarget(): { scale: number, _invertForward: boolean, target: Object3D, sessionRoot: Object3D | null } {
646
+ if (!this.objectToExport) return { scale: 1, _invertForward: false, target: this.gameObject, sessionRoot: null };
638
647
 
639
648
  const xr = GameObject.findObjectOfType(WebXR);
640
649
  let sessionRoot = GameObject.getComponentInParent(this.objectToExport, WebARSessionRoot);
641
- if(!sessionRoot) sessionRoot = GameObject.getComponentInChildren(this.objectToExport, WebARSessionRoot);
650
+ if (!sessionRoot) sessionRoot = GameObject.getComponentInChildren(this.objectToExport, WebARSessionRoot);
642
651
 
643
652
  let arScale = 1;
644
653
  let _invertForward = false;
@@ -653,7 +662,7 @@ export class USDZExporter extends Behaviour {
653
662
  // eslint-disable-next-line deprecation/deprecation
654
663
  _invertForward = sessionRoot.invertForward;
655
664
  }
656
-
665
+
657
666
  const scale = 1 / arScale;
658
667
  const result = { scale, _invertForward, target, sessionRoot: sessionRoot?.gameObject ?? null };
659
668
  return result;
@@ -699,7 +708,7 @@ export class USDZExporter extends Behaviour {
699
708
  private createQuicklookButton() {
700
709
  const buttoncontainer = WebXRButtonFactory.getOrCreate();
701
710
  const button = buttoncontainer.createQuicklookButton();
702
- if(!button.parentNode) this.context.menu.appendChild(button);
711
+ if (!button.parentNode) this.context.menu.appendChild(button);
703
712
  return button;
704
713
  }
705
714
  }
@@ -1,17 +1,18 @@
1
- import { Matrix4, Object3D, Quaternion, Vector3 } from "three";
1
+ import { ImageBitmapLoader, Matrix4, Object3D, Quaternion, Vector3 } from "three";
2
2
  import { Object3DEventMap } from "three";
3
3
 
4
4
  import { isDevEnvironment, showBalloonWarning } from "../../engine/debug/index.js";
5
5
  import { AssetReference } from "../../engine/engine_addressables.js";
6
6
  import { serializable } from "../../engine/engine_serialization.js";
7
7
  import { IGameObject } from "../../engine/engine_types.js";
8
- import { CircularBuffer, DeviceUtilities, getParam } from "../../engine/engine_utils.js";
8
+ import { CircularBuffer, delay, DeviceUtilities, getParam } from "../../engine/engine_utils.js";
9
9
  import { type NeedleXREventArgs, NeedleXRSession } from "../../engine/xr/api.js";
10
10
  import { IUSDExporterExtension } from "../../engine-components/export/usdz/Extension.js";
11
11
  import { imageToCanvas, USDObject, USDWriter, USDZExporterContext } from "../../engine-components/export/usdz/ThreeUSDZExporter.js";
12
12
  import { USDZExporter } from "../../engine-components/export/usdz/USDZExporter.js";
13
13
  import { Behaviour, GameObject } from "../Component.js";
14
14
  import { Renderer } from "../Renderer.js";
15
+ import { Context } from "../../engine/engine_context.js";
15
16
 
16
17
  // https://github.com/immersive-web/marker-tracking/blob/main/explainer.md
17
18
 
@@ -147,90 +148,122 @@ export class WebXRImageTrackingModel {
147
148
 
148
149
  class ImageTrackingExtension implements IUSDExporterExtension {
149
150
 
151
+
152
+
153
+ readonly isImageTrackingExtension = true;
150
154
  get extensionName() { return "image-tracking"; }
151
155
 
152
- private filename: string;
153
- private widthInMeters: number;
154
- private imageData: Uint8Array;
155
156
 
156
- constructor(filename: string, imageData: Uint8Array, widthInMeters: number) {
157
- this.filename = filename;
158
- this.imageData = imageData;
159
- this.widthInMeters = widthInMeters;
157
+ constructor(private readonly exporter: USDZExporter, private readonly component: WebXRImageTracking) {
158
+ if (debug) console.log(this);
159
+ this.exporter.anchoringType = "image";
160
+ }
161
+
162
+ // set during export
163
+ private shouldExport: boolean = true;
164
+
165
+ private filename: string = "marker.png";
166
+ private imageModel: WebXRImageTrackingModel | null = null;
167
+
168
+ onBeforeBuildDocument(_context: USDZExporterContext) {
169
+
170
+ // check if this extension is the first image tracking extension in the list
171
+ // since iOS can only track one image at a time we only allow one image tracking extension to be active
172
+ // we have to determine this at the earlierst export callback
173
+ // all subsequent export callbacks should then check is shouldExport is set to true
174
+ // this should only be the case for exactly one extension
175
+ const index = this.exporter.extensions
176
+ .filter(e => {
177
+ const ext = (e as ImageTrackingExtension);
178
+ return ext.isImageTrackingExtension && ext.component.activeAndEnabled && ext.component.trackedImages?.length > 0;
179
+ })
180
+ .indexOf(this);
181
+ this.shouldExport = index === 0;
182
+ if (!this.shouldExport) return;
183
+
184
+ // Warn if more than one tracked image is used for USDZ; that's not supported at the moment.
185
+ if (this.component.trackedImages?.length > 1) {
186
+ if (debug || isDevEnvironment()) {
187
+ showBalloonWarning("USDZ: Only one tracked image is supported.");
188
+ console.warn("USDZ: Only one tracked image is supported. Will choose the first one in the trackedImages list");
189
+ }
190
+ }
160
191
  }
161
192
 
162
193
  onAfterHierarchy(_context: USDZExporterContext, writer: USDWriter) {
194
+ if (!this.shouldExport) return;
195
+
163
196
  const iOSVersion = DeviceUtilities.getiOSVersion();
164
197
  const majorVersion = iOSVersion ? parseInt(iOSVersion.split(".")[0]) : 18;
165
198
  const workaroundForFB16119331 = majorVersion >= 18;
166
199
  const multiplier = workaroundForFB16119331 ? 1 : 100;
167
200
  writer.beginBlock(`def Preliminary_ReferenceImage "AnchoringReferenceImage"`);
168
201
  writer.appendLine(`uniform asset image = @image_tracking/` + this.filename + `@`);
169
- writer.appendLine(`uniform double physicalWidth = ` + (this.widthInMeters * multiplier).toFixed(8));
202
+ writer.appendLine(`uniform double physicalWidth = ` + (this.imageModel!.widthInMeters * multiplier).toFixed(8));
170
203
  writer.closeBlock();
204
+
171
205
  }
172
206
 
173
- onBeforeBuildDocument(_context: USDZExporterContext) {
174
- const imageTracking = GameObject.findObjectOfType(WebXRImageTracking);
175
- if (!imageTracking || !imageTracking.trackedImages) return;
207
+ async onAfterSerialize(context: USDZExporterContext) {
208
+ if (!this.shouldExport) return;
176
209
 
177
- // Warn if more than one tracked image is used for USDZ; that's not supported at the moment.
178
- if (imageTracking.trackedImages.length > 1)
179
- {
180
- if (isDevEnvironment()) showBalloonWarning("USDZ: Only one tracked image is supported.");
181
- console.warn("USDZ: Only one tracked image is supported.");
182
- }
183
- }
210
+ const imageModel = this.imageModel;
211
+ const img = _imageElements.get(imageModel!.image!)!;
184
212
 
185
- onAfterSerialize(context: USDZExporterContext) {
186
- context.files['image_tracking/' + this.filename] = this.imageData;
213
+ const canvas = await imageToCanvas(img);
214
+ const blob = await canvas.convertToBlob({ type: 'image/png' });
215
+ const arrayBuffer = await blob.arrayBuffer();
216
+ context.files['image_tracking/' + this.filename] = new Uint8Array(arrayBuffer);
187
217
  }
188
218
 
189
219
  onExportObject(object: Object3D<Object3DEventMap>, model: USDObject, _context: USDZExporterContext) {
190
- const imageTracking = GameObject.findObjectOfType(WebXRImageTracking);
191
- if (!imageTracking || !imageTracking.trackedImages) return;
220
+ if (!this.shouldExport) return;
192
221
 
222
+ const imageTracking = this.component;
223
+ if (!imageTracking || !imageTracking.trackedImages?.length || !imageTracking.activeAndEnabled) return;
193
224
 
194
- for (const trackedImage of imageTracking.trackedImages) {
195
- if (trackedImage.object?.asset === object) {
196
- const exporter = GameObject.findObjectOfType(USDZExporter);
197
- if (!exporter) continue;
225
+ // we only care about the first image
226
+ const trackedImage = imageTracking.trackedImages[0];
198
227
 
199
- const { scale, target } = exporter.getARScaleAndTarget();
228
+ if (trackedImage.object?.asset === object) {
229
+ this.imageModel = trackedImage;
200
230
 
201
- // We have to reset the image tracking object's position and rotation, because QuickLook applies them.
202
- // On Android WebXR they're replaced by the tracked data
203
- let parent = object;
204
- const relativeMatrix = new Matrix4();
205
- if (object !== target) {
206
- while (parent.parent && parent.parent !== target) {
207
- parent = parent.parent;
208
- relativeMatrix.premultiply(parent.matrix);
209
- }
231
+ const { scale, target } = this.exporter.getARScaleAndTarget();
232
+
233
+ // We have to reset the image tracking object's position and rotation, because QuickLook applies them.
234
+ // On Android WebXR they're replaced by the tracked data
235
+ let parent = object;
236
+
237
+ const relativeMatrix = new Matrix4();
238
+ if (object !== target) {
239
+ while (parent && parent.parent && parent.parent !== target) {
240
+ parent = parent.parent;
241
+ relativeMatrix.premultiply(parent.matrix);
210
242
  }
211
- const mat = relativeMatrix
212
- .clone()
213
- .invert()
214
- // apply session root scale again after undoing the world transformation
215
- model.setMatrix(mat.scale(new Vector3(scale, scale, scale)));
216
-
217
- // Unfortunately looks like Apple's docs are incomplete:
218
- // https://developer.apple.com/documentation/realitykit/preliminary_anchoringapi#Nest-and-Layer-Anchorable-Prims
219
- // In practice, it seems that nesting is not allowed – no image tracking will be applied to nested objects.
220
- // Thus, we can't have separate transforms for "regularly placing content" and "placing content with an image marker".
221
- // model.extraSchemas.push("Preliminary_AnchoringAPI");
222
- // model.addEventListener("serialize", (_writer: USDWriter, _context: USDZExporterContext) => {
223
- // writer.appendLine( `token preliminary:anchoring:type = "image"` );
224
- // writer.appendLine( `rel preliminary:imageAnchoring:referenceImage = </${context.document.name}/Scenes/Scene/AnchoringReferenceImage>` );
225
- // });
226
-
227
- // We can only apply this to the first tracked image, more are not supported by QuickLook.
228
- break;
229
243
  }
244
+ const mat = relativeMatrix
245
+ .clone()
246
+ .invert()
247
+ // apply session root scale again after undoing the world transformation
248
+ model.setMatrix(mat.scale(new Vector3(scale, scale, scale)));
249
+
250
+
251
+ // Unfortunately looks like Apple's docs are incomplete:
252
+ // https://developer.apple.com/documentation/realitykit/preliminary_anchoringapi#Nest-and-Layer-Anchorable-Prims
253
+ // In practice, it seems that nesting is not allowed – no image tracking will be applied to nested objects.
254
+ // Thus, we can't have separate transforms for "regularly placing content" and "placing content with an image marker".
255
+ // model.extraSchemas.push("Preliminary_AnchoringAPI");
256
+ // model.addEventListener("serialize", (_writer: USDWriter, _context: USDZExporterContext) => {
257
+ // writer.appendLine( `token preliminary:anchoring:type = "image"` );
258
+ // writer.appendLine( `rel preliminary:imageAnchoring:referenceImage = </${context.document.name}/Scenes/Scene/AnchoringReferenceImage>` );
259
+ // });
260
+
261
+ // We can only apply this to the first tracked image, more are not supported by QuickLook.
230
262
  }
231
263
  }
232
264
  }
233
265
 
266
+
234
267
  /**
235
268
  * @category XR
236
269
  * @group Components
@@ -238,14 +271,13 @@ class ImageTrackingExtension implements IUSDExporterExtension {
238
271
  export class WebXRImageTracking extends Behaviour {
239
272
 
240
273
  @serializable(WebXRImageTrackingModel)
241
- trackedImages?: WebXRImageTrackingModel[];
274
+ trackedImages: WebXRImageTrackingModel[] = [];
242
275
 
243
276
  /** Applies smoothing based on detected jitter to the tracked image. */
244
277
  @serializable()
245
278
  smooth: boolean = true;
246
279
 
247
280
  private readonly trackedImageIndexMap: Map<number, WebXRImageTrackingModel> = new Map();
248
- private static _imageElements: Map<string, ImageBitmap | null> = new Map();
249
281
 
250
282
  /** @returns true if image tracking is supported on this device. This may return false at runtime if the user's browser did not enable webxr incubations */
251
283
  get supported() { return this._supported; }
@@ -257,38 +289,24 @@ export class WebXRImageTracking extends Behaviour {
257
289
  if (!this.trackedImages) return;
258
290
  for (const trackedImage of this.trackedImages) {
259
291
  if (trackedImage.image) {
260
- if (WebXRImageTracking._imageElements.has(trackedImage.image)) {
261
- // already loaded
262
- }
263
- else {
264
- const url = trackedImage.image;
265
- WebXRImageTracking._imageElements.set(url, null);
266
- const imageElement = document.createElement("img") as HTMLImageElement;
267
- imageElement.src = url;
268
- imageElement.addEventListener("load", async () => {
269
- const img = await createImageBitmap(imageElement);
270
- WebXRImageTracking._imageElements.set(url, img);
271
-
272
- // read back Uint8Array to use in USDZ -
273
- // TODO better would be to do that once we actually need it
274
- const canvas = await imageToCanvas(img);
275
- if (canvas) {
276
- const blob = await canvas.convertToBlob({ type: 'image/png' });
277
- const arrayBuffer = await blob.arrayBuffer();
278
-
279
- const exporter = GameObject.findObjectOfType(USDZExporter);
280
- if (exporter && this.trackedImages) {
281
- exporter.extensions.push(
282
- new ImageTrackingExtension("marker.png", new Uint8Array(arrayBuffer), this.trackedImages[0].widthInMeters)
283
- );
284
- exporter.anchoringType = "image";
285
- }
286
- }
287
- });
288
- }
292
+ loadImage(trackedImage.image);
289
293
  }
290
294
  }
291
295
  }
296
+ onEnable() {
297
+ USDZExporter.beforeExport.addEventListener(this.onBeforeUSDZExport);
298
+ }
299
+ onDisable(): void {
300
+ USDZExporter.beforeExport.removeEventListener(this.onBeforeUSDZExport);
301
+ }
302
+
303
+ private onBeforeUSDZExport = (args: { exporter: USDZExporter }) => {
304
+ if (this.activeAndEnabled && this.trackedImages?.length) {
305
+ args.exporter.extensions.push(new ImageTrackingExtension(args.exporter, this));
306
+ }
307
+ }
308
+
309
+
292
310
 
293
311
  onBeforeXR(_mode: XRSessionMode, args: XRSessionInit & { trackedImages: Array<any> }): void {
294
312
  // console.log("onXRRequested", args, this.trackedImages)
@@ -300,7 +318,7 @@ export class WebXRImageTracking extends Behaviour {
300
318
  args.trackedImages = [];
301
319
  for (const trackedImage of this.trackedImages) {
302
320
  if (trackedImage.image?.length && trackedImage.widthInMeters > 0) {
303
- const bitmap = WebXRImageTracking._imageElements.get(trackedImage.image);
321
+ const bitmap = _imageElements.get(trackedImage.image);
304
322
  if (bitmap) {
305
323
  this.trackedImageIndexMap.set(args.trackedImages.length, trackedImage);
306
324
  args.trackedImages.push({
@@ -337,7 +355,7 @@ export class WebXRImageTracking extends Behaviour {
337
355
 
338
356
  onLeaveXR(_args: NeedleXREventArgs): void {
339
357
 
340
- if(!this.supported && DeviceUtilities.isAndroidDevice()) {
358
+ if (!this.supported && DeviceUtilities.isAndroidDevice()) {
341
359
  showBalloonWarning(this.webXRIncubationsWarning);
342
360
  }
343
361
 
@@ -364,7 +382,7 @@ export class WebXRImageTracking extends Behaviour {
364
382
  private readonly imageToObjectMap = new Map<WebXRImageTrackingModel, { object: Object3D | null, frames: number, lastTrackingTime: number }>();
365
383
  private readonly currentImages: WebXRTrackedImage[] = [];
366
384
 
367
-
385
+
368
386
  private readonly webXRIncubationsWarning = "Image tracking is currently not supported on this device. On Chrome for Android, you can enable the <a target=\"_blank\" href=\"#\" onclick=\"() => console.log('I')\">chrome://flags/#webxr-incubations</a> flag.";
369
387
 
370
388
  onUpdateXR(args: NeedleXREventArgs): void {
@@ -517,4 +535,34 @@ export class WebXRImageTracking extends Behaviour {
517
535
  }
518
536
  }
519
537
  }
538
+ }
539
+
540
+
541
+
542
+
543
+ const _imageElements: Map<string, ImageBitmap | null> = new Map();
544
+ const _imageLoadingPromises: Map<string, Promise<boolean>> = new Map();
545
+
546
+ async function loadImage(url: string) {
547
+ if (_imageElements.has(url)) {
548
+ if (_imageLoadingPromises.has(url)) return _imageLoadingPromises.get(url);
549
+ return Promise.resolve(true);
550
+ }
551
+ const promise = new Promise<boolean>(res => {
552
+ _imageElements.set(url, null);
553
+ const imageElement = document.createElement("img") as HTMLImageElement;
554
+ imageElement.src = url;
555
+ imageElement.addEventListener("load", async () => {
556
+ const img = await createImageBitmap(imageElement);
557
+ _imageElements.set(url, img);
558
+ res(true);
559
+ });
560
+ });
561
+
562
+ _imageLoadingPromises.set(url, promise);
563
+ promise.finally(() => {
564
+ _imageLoadingPromises.delete(url);
565
+ });
566
+
567
+ return promise;
520
568
  }