@needle-tools/engine 4.8.7-next.e134730 → 4.8.8-next.12b5946

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (118) hide show
  1. package/CHANGELOG.md +13 -0
  2. package/README.md +55 -42
  3. package/components.needle.json +1 -1
  4. package/dist/{needle-engine.bundle-DrGsgE4t.min.js → needle-engine.bundle-CS7vqRb3.min.js} +144 -144
  5. package/dist/{needle-engine.bundle-X9nxhICu.umd.cjs → needle-engine.bundle-kYzccQZF.umd.cjs} +147 -147
  6. package/dist/{needle-engine.bundle-CvRpjtJj.js → needle-engine.bundle-lC7eSFno.js} +7090 -7010
  7. package/dist/needle-engine.d.ts +7 -0
  8. package/dist/needle-engine.js +2 -2
  9. package/dist/needle-engine.min.js +1 -1
  10. package/dist/needle-engine.umd.cjs +1 -1
  11. package/lib/engine/engine_addressables.d.ts +12 -12
  12. package/lib/engine/engine_addressables.js +32 -23
  13. package/lib/engine/engine_addressables.js.map +1 -1
  14. package/lib/engine/engine_animation.d.ts +1 -3
  15. package/lib/engine/engine_animation.js +18 -13
  16. package/lib/engine/engine_animation.js.map +1 -1
  17. package/lib/engine/engine_assetdatabase.js +6 -6
  18. package/lib/engine/engine_assetdatabase.js.map +1 -1
  19. package/lib/engine/engine_camera.d.ts +23 -3
  20. package/lib/engine/engine_camera.js +34 -2
  21. package/lib/engine/engine_camera.js.map +1 -1
  22. package/lib/engine/engine_context.d.ts +15 -0
  23. package/lib/engine/engine_context.js +33 -0
  24. package/lib/engine/engine_context.js.map +1 -1
  25. package/lib/engine/engine_create_objects.d.ts +3 -3
  26. package/lib/engine/engine_create_objects.js +5 -4
  27. package/lib/engine/engine_create_objects.js.map +1 -1
  28. package/lib/engine/engine_gameobject.js +2 -2
  29. package/lib/engine/engine_gameobject.js.map +1 -1
  30. package/lib/engine/engine_gltf_builtin_components.js +11 -11
  31. package/lib/engine/engine_gltf_builtin_components.js.map +1 -1
  32. package/lib/engine/engine_loaders.callbacks.d.ts +1 -0
  33. package/lib/engine/engine_loaders.callbacks.js +1 -0
  34. package/lib/engine/engine_loaders.callbacks.js.map +1 -1
  35. package/lib/engine/engine_loaders.js +15 -11
  36. package/lib/engine/engine_loaders.js.map +1 -1
  37. package/lib/engine/extensions/NEEDLE_lighting_settings.js +5 -2
  38. package/lib/engine/extensions/NEEDLE_lighting_settings.js.map +1 -1
  39. package/lib/engine/extensions/NEEDLE_lightmaps.js +1 -1
  40. package/lib/engine/extensions/NEEDLE_lightmaps.js.map +1 -1
  41. package/lib/engine/js-extensions/Object3D.d.ts +1 -1
  42. package/lib/engine/js-extensions/Vector.d.ts +5 -0
  43. package/lib/engine/js-extensions/Vector.js.map +1 -1
  44. package/lib/engine/webcomponents/needle-engine.attributes.d.ts +1 -0
  45. package/lib/engine/webcomponents/needle-engine.d.ts +2 -2
  46. package/lib/engine/webcomponents/needle-engine.js +19 -21
  47. package/lib/engine/webcomponents/needle-engine.js.map +1 -1
  48. package/lib/engine-components/Animation.js +2 -1
  49. package/lib/engine-components/Animation.js.map +1 -1
  50. package/lib/engine-components/AnimationUtilsAutoplay.js +1 -6
  51. package/lib/engine-components/AnimationUtilsAutoplay.js.map +1 -1
  52. package/lib/engine-components/Camera.d.ts +2 -0
  53. package/lib/engine-components/Camera.js +5 -1
  54. package/lib/engine-components/Camera.js.map +1 -1
  55. package/lib/engine-components/ContactShadows.d.ts +12 -2
  56. package/lib/engine-components/ContactShadows.js +24 -4
  57. package/lib/engine-components/ContactShadows.js.map +1 -1
  58. package/lib/engine-components/DropListener.d.ts +21 -15
  59. package/lib/engine-components/DropListener.js +38 -34
  60. package/lib/engine-components/DropListener.js.map +1 -1
  61. package/lib/engine-components/LookAtConstraint.d.ts +5 -1
  62. package/lib/engine-components/LookAtConstraint.js +8 -0
  63. package/lib/engine-components/LookAtConstraint.js.map +1 -1
  64. package/lib/engine-components/NestedGltf.d.ts +9 -4
  65. package/lib/engine-components/NestedGltf.js +32 -26
  66. package/lib/engine-components/NestedGltf.js.map +1 -1
  67. package/lib/engine-components/OrbitControls.d.ts +30 -9
  68. package/lib/engine-components/OrbitControls.js +56 -19
  69. package/lib/engine-components/OrbitControls.js.map +1 -1
  70. package/lib/engine-components/Renderer.js +2 -1
  71. package/lib/engine-components/Renderer.js.map +1 -1
  72. package/lib/engine-components/Skybox.js +8 -9
  73. package/lib/engine-components/Skybox.js.map +1 -1
  74. package/lib/engine-components/api.d.ts +1 -0
  75. package/lib/engine-components/api.js.map +1 -1
  76. package/lib/engine-components/export/usdz/Extension.d.ts +1 -1
  77. package/lib/engine-components/export/usdz/ThreeUSDZExporter.js +1 -1
  78. package/lib/engine-components/export/usdz/ThreeUSDZExporter.js.map +1 -1
  79. package/lib/engine-components/export/usdz/USDZExporter.d.ts +7 -0
  80. package/lib/engine-components/export/usdz/USDZExporter.js +8 -1
  81. package/lib/engine-components/export/usdz/USDZExporter.js.map +1 -1
  82. package/lib/engine-components/webxr/WebXRImageTracking.d.ts +4 -2
  83. package/lib/engine-components/webxr/WebXRImageTracking.js +117 -81
  84. package/lib/engine-components/webxr/WebXRImageTracking.js.map +1 -1
  85. package/package.json +2 -2
  86. package/plugins/common/files.js +6 -3
  87. package/plugins/vite/alias.js +26 -17
  88. package/plugins/vite/editor-connection.js +4 -4
  89. package/src/engine/engine_addressables.ts +46 -33
  90. package/src/engine/engine_animation.ts +20 -13
  91. package/src/engine/engine_assetdatabase.ts +7 -7
  92. package/src/engine/engine_camera.ts +54 -3
  93. package/src/engine/engine_context.ts +39 -1
  94. package/src/engine/engine_create_objects.ts +8 -7
  95. package/src/engine/engine_gameobject.ts +2 -2
  96. package/src/engine/engine_gltf_builtin_components.ts +12 -11
  97. package/src/engine/engine_loaders.callbacks.ts +1 -0
  98. package/src/engine/engine_loaders.ts +18 -13
  99. package/src/engine/extensions/NEEDLE_lighting_settings.ts +5 -2
  100. package/src/engine/extensions/NEEDLE_lightmaps.ts +1 -1
  101. package/src/engine/js-extensions/Vector.ts +6 -0
  102. package/src/engine/webcomponents/needle-engine.attributes.ts +2 -0
  103. package/src/engine/webcomponents/needle-engine.ts +21 -21
  104. package/src/engine-components/Animation.ts +1 -1
  105. package/src/engine-components/AnimationUtilsAutoplay.ts +1 -6
  106. package/src/engine-components/Camera.ts +7 -1
  107. package/src/engine-components/ContactShadows.ts +27 -6
  108. package/src/engine-components/DropListener.ts +44 -34
  109. package/src/engine-components/LookAtConstraint.ts +9 -1
  110. package/src/engine-components/NestedGltf.ts +33 -24
  111. package/src/engine-components/OrbitControls.ts +81 -32
  112. package/src/engine-components/Renderer.ts +2 -1
  113. package/src/engine-components/Skybox.ts +9 -10
  114. package/src/engine-components/api.ts +2 -1
  115. package/src/engine-components/export/usdz/Extension.ts +1 -1
  116. package/src/engine-components/export/usdz/ThreeUSDZExporter.ts +1 -1
  117. package/src/engine-components/export/usdz/USDZExporter.ts +21 -12
  118. package/src/engine-components/webxr/WebXRImageTracking.ts +138 -90
@@ -48,21 +48,20 @@ function createRemoteSkyboxComponent(context: IContext, url: string, skybox: boo
48
48
  const promises = new Array<Promise<any>>();
49
49
  ContextRegistry.registerCallback(ContextEvent.ContextCreationStart, (args) => {
50
50
  const context = args.context;
51
- const skyboxImage = context.domElement.getAttribute("background-image");
51
+ const backgroundImage = context.domElement.getAttribute("background-image");
52
52
  const environmentImage = context.domElement.getAttribute("environment-image");
53
- const envAndSkyboxAreSame = skyboxImage === environmentImage;
54
-
55
- if (skyboxImage && !envAndSkyboxAreSame) {
56
- if (debug) console.log("Creating remote skybox to load " + skyboxImage);
53
+
54
+ if (backgroundImage) {
55
+ if (debug) console.log("Creating RemoteSkybox to load background " + backgroundImage);
57
56
  // if the user is loading a GLB without a camera then the CameraUtils (which creates the default camera)
58
57
  // checks if we have this attribute set and then sets the skybox clearflags accordingly
59
58
  // if the user has a GLB with a camera but set to solid color then the skybox image is not visible -> we will just warn then and not override the camera settings
60
- const promise = createRemoteSkyboxComponent(context, skyboxImage, true, false, "background-image");
59
+ const promise = createRemoteSkyboxComponent(context, backgroundImage, true, false, "background-image");
61
60
  if (promise) promises.push(promise);
62
61
  }
63
62
  if (environmentImage) {
64
- if (debug) console.log("Creating remote environment to load " + environmentImage);
65
- const promise = createRemoteSkyboxComponent(context, environmentImage, envAndSkyboxAreSame, true, "environment-image");
63
+ if (debug) console.log("Creating RemoteSkybox to load environment " + environmentImage);
64
+ const promise = createRemoteSkyboxComponent(context, environmentImage, false, true, "environment-image");
66
65
  if (promise) promises.push(promise);
67
66
  }
68
67
  });
@@ -200,7 +199,7 @@ export class RemoteSkybox extends Behaviour {
200
199
  console.warn("Potentially invalid skybox URL: \"" + name + "\" on " + (this.name || this.gameObject?.name || "context"));
201
200
  }
202
201
 
203
- if (debug) console.log("Set remote skybox url: " + url);
202
+ if (debug) console.log("Set RemoteSkybox url: " + url);
204
203
 
205
204
  if (this._prevUrl === url && this._prevLoadedEnvironment) {
206
205
  this.apply();
@@ -258,7 +257,7 @@ export class RemoteSkybox extends Behaviour {
258
257
  this._prevBackground = this.context.scene.background;
259
258
  if (this.context.scene.environment !== envMap)
260
259
  this._prevEnvironment = this.context.scene.environment;
261
- if (debug) console.log("Set remote skybox", this.url, !Camera.backgroundShouldBeTransparent(this.context));
260
+ if (debug) console.log("Set RemoteSkybox (" + ((this.environment && this.background) ? "environment and background" : this.environment ? "environment" : this.background ? "background" : "none") + ")", this.url, !Camera.backgroundShouldBeTransparent(this.context));
262
261
  if (this.environment)
263
262
  this.context.scene.environment = envMap;
264
263
  if (this.background && !Camera.backgroundShouldBeTransparent(this.context))
@@ -51,8 +51,9 @@ import "./CameraUtils.js"
51
51
  import "./AnimationUtils.js"
52
52
  import "./AnimationUtilsAutoplay.js"
53
53
 
54
- export { DragMode } from "./DragControls.js"
54
+ export { DragMode } from "./DragControls.js";
55
55
  export type { DropListenerNetworkEventArguments, DropListenerOnDropArguments } from "./DropListener.js";
56
+ export { type FitCameraOptions } from "./OrbitControls.js";
56
57
  export * from "./particlesystem/api.js"
57
58
 
58
59
  // for correct type resolution in JSDoc
@@ -21,5 +21,5 @@ export interface IUSDExporterExtension {
21
21
  onAfterBuildDocument?(context: USDZExporterContext);
22
22
  onExportObject?(object: Object3D, model: USDObject, context: USDZExporterContext);
23
23
  onAfterSerialize?(context: USDZExporterContext);
24
- onAfterHierarchy?(context: USDZExporterContext, writer: USDWriter);
24
+ onAfterHierarchy?(context: USDZExporterContext, writer: USDWriter) : void | Promise<void>;
25
25
  }
@@ -1082,7 +1082,7 @@ async function parseDocument( context: USDZExporterContext, afterStageRoot: () =
1082
1082
  Progress.end("export-usdz-xforms");
1083
1083
 
1084
1084
  Progress.report("export-usdz", "invoke onAfterHierarchy");
1085
- invokeAll( context, 'onAfterHierarchy', writer );
1085
+ await invokeAll( context, 'onAfterHierarchy', writer );
1086
1086
 
1087
1087
  writer.closeBlock();
1088
1088
  writer.closeBlock();
@@ -1,5 +1,5 @@
1
1
  import { NEEDLE_progressive } from "@needle-tools/gltf-progressive";
2
- import { Euler, Matrix4, Mesh, Object3D, Quaternion, Vector3 } from "three";
2
+ import { Euler, EventDispatcher, Matrix4, Mesh, Object3D, Quaternion, Vector3 } from "three";
3
3
 
4
4
  import { isDevEnvironment, showBalloonMessage, showBalloonWarning } from "../../../engine/debug/index.js";
5
5
  import { hasProLicense } from "../../../engine/engine_license.js";
@@ -12,6 +12,7 @@ import { InstancingHandler } from "../../../engine-components/RendererInstancing
12
12
  import { Collider } from "../../Collider.js";
13
13
  import { Behaviour, GameObject } from "../../Component.js";
14
14
  import { ContactShadows } from "../../ContactShadows.js";
15
+ import { EventList } from "../../EventList.js";
15
16
  import { GroundProjectedEnv } from "../../GroundProjection.js";
16
17
  import { Renderer } from "../../Renderer.js"
17
18
  import { Rigidbody } from "../../RigidBody.js";
@@ -71,6 +72,9 @@ export class CustomBranding {
71
72
  */
72
73
  export class USDZExporter extends Behaviour {
73
74
 
75
+ static readonly beforeExport = new EventList<{ exporter: USDZExporter }>();
76
+ static readonly afterExport = new EventList<{ exporter: USDZExporter }>();
77
+
74
78
  /**
75
79
  * Assign the object to export as USDZ file. If undefined or null, the whole scene will be exported.
76
80
  */
@@ -210,7 +214,7 @@ export class USDZExporter extends Behaviour {
210
214
  * Creates an USDZ file from the current scene or assigned objectToExport and opens it in QuickLook.
211
215
  * @returns a Promise<Blob> containing the USDZ file
212
216
  */
213
- async exportAndOpen() : Promise<Blob | null> {
217
+ async exportAndOpen(): Promise<Blob | null> {
214
218
 
215
219
  let name = this.exportFileName ?? this.objectToExport?.name ?? this.name;
216
220
  name += "-" + getFormattedDate(); // seems iOS caches the file in some cases, this ensures we always have a fresh file
@@ -221,7 +225,7 @@ export class USDZExporter extends Behaviour {
221
225
  }
222
226
 
223
227
  if (!this.link) this.link = ensureQuicklookLinkIsCreated(this.context, DeviceUtilities.supportsQuickLookAR());
224
-
228
+
225
229
  // ability to specify a custom USDZ file to be used instead of a dynamic one
226
230
  if (this.customUsdzFile) {
227
231
  if (debug) console.log("Exporting custom usdz", this.customUsdzFile)
@@ -234,14 +238,19 @@ export class USDZExporter extends Behaviour {
234
238
  return null;
235
239
  }
236
240
 
237
- const blob = await this.export(this.objectToExport);
241
+ USDZExporter.beforeExport.invoke({ exporter: this });
242
+ const blob = await this.export(this.objectToExport)
243
+ .finally(() => {
244
+ USDZExporter.afterExport.invoke({ exporter: this });
245
+ });
246
+
238
247
  if (!blob) {
239
248
  console.error("USDZ generation failed. Please report a bug", this);
240
249
  return null;
241
250
  }
242
251
 
243
252
  if (debug) console.log("USDZ generation done. Downloading as " + name);
244
-
253
+
245
254
  // TODO Potentially we have to detect QuickLook availability here,
246
255
  // and download the file instead. But browsers keep changing how they deal with non-user-initiated downloads...
247
256
  // https://webkit.org/blog/8421/viewing-augmented-reality-assets-in-safari-for-ios/#:~:text=inside%20the%20anchor.-,Feature%20Detection,-To%20detect%20support
@@ -364,7 +373,7 @@ export class USDZExporter extends Behaviour {
364
373
  if (this.interactive) {
365
374
  defaultExtensions.push(new BehaviorExtension());
366
375
  defaultExtensions.push(new AudioExtension());
367
-
376
+
368
377
  // If physics are enabled, and there are any Rigidbody components in the scene,
369
378
  // add the PhysicsExtension to the default extensions.
370
379
  if (globalThis["NEEDLE_USE_RAPIER"]) {
@@ -632,13 +641,13 @@ export class USDZExporter extends Behaviour {
632
641
  private _rootPositionBeforeExport: Vector3 = new Vector3();
633
642
  private _rootRotationBeforeExport: Quaternion = new Quaternion();
634
643
  private _rootScaleBeforeExport: Vector3 = new Vector3();
635
-
636
- getARScaleAndTarget(): { scale: number, _invertForward: boolean, target: Object3D, sessionRoot: Object3D | null} {
637
- if (!this.objectToExport) return { scale: 1, _invertForward: false, target: this.gameObject, sessionRoot: null};
644
+
645
+ getARScaleAndTarget(): { scale: number, _invertForward: boolean, target: Object3D, sessionRoot: Object3D | null } {
646
+ if (!this.objectToExport) return { scale: 1, _invertForward: false, target: this.gameObject, sessionRoot: null };
638
647
 
639
648
  const xr = GameObject.findObjectOfType(WebXR);
640
649
  let sessionRoot = GameObject.getComponentInParent(this.objectToExport, WebARSessionRoot);
641
- if(!sessionRoot) sessionRoot = GameObject.getComponentInChildren(this.objectToExport, WebARSessionRoot);
650
+ if (!sessionRoot) sessionRoot = GameObject.getComponentInChildren(this.objectToExport, WebARSessionRoot);
642
651
 
643
652
  let arScale = 1;
644
653
  let _invertForward = false;
@@ -653,7 +662,7 @@ export class USDZExporter extends Behaviour {
653
662
  // eslint-disable-next-line deprecation/deprecation
654
663
  _invertForward = sessionRoot.invertForward;
655
664
  }
656
-
665
+
657
666
  const scale = 1 / arScale;
658
667
  const result = { scale, _invertForward, target, sessionRoot: sessionRoot?.gameObject ?? null };
659
668
  return result;
@@ -699,7 +708,7 @@ export class USDZExporter extends Behaviour {
699
708
  private createQuicklookButton() {
700
709
  const buttoncontainer = WebXRButtonFactory.getOrCreate();
701
710
  const button = buttoncontainer.createQuicklookButton();
702
- if(!button.parentNode) this.context.menu.appendChild(button);
711
+ if (!button.parentNode) this.context.menu.appendChild(button);
703
712
  return button;
704
713
  }
705
714
  }
@@ -1,11 +1,12 @@
1
- import { Matrix4, Object3D, Quaternion, Vector3 } from "three";
1
+ import { ImageBitmapLoader, Matrix4, Object3D, Quaternion, Vector3 } from "three";
2
2
  import { Object3DEventMap } from "three";
3
3
 
4
4
  import { isDevEnvironment, showBalloonWarning } from "../../engine/debug/index.js";
5
5
  import { AssetReference } from "../../engine/engine_addressables.js";
6
+ import { Context } from "../../engine/engine_context.js";
6
7
  import { serializable } from "../../engine/engine_serialization.js";
7
8
  import { IGameObject } from "../../engine/engine_types.js";
8
- import { CircularBuffer, DeviceUtilities, getParam } from "../../engine/engine_utils.js";
9
+ import { CircularBuffer, delay, DeviceUtilities, getParam } from "../../engine/engine_utils.js";
9
10
  import { type NeedleXREventArgs, NeedleXRSession } from "../../engine/xr/api.js";
10
11
  import { IUSDExporterExtension } from "../../engine-components/export/usdz/Extension.js";
11
12
  import { imageToCanvas, USDObject, USDWriter, USDZExporterContext } from "../../engine-components/export/usdz/ThreeUSDZExporter.js";
@@ -147,90 +148,122 @@ export class WebXRImageTrackingModel {
147
148
 
148
149
  class ImageTrackingExtension implements IUSDExporterExtension {
149
150
 
151
+
152
+
153
+ readonly isImageTrackingExtension = true;
150
154
  get extensionName() { return "image-tracking"; }
151
155
 
152
- private filename: string;
153
- private widthInMeters: number;
154
- private imageData: Uint8Array;
155
156
 
156
- constructor(filename: string, imageData: Uint8Array, widthInMeters: number) {
157
- this.filename = filename;
158
- this.imageData = imageData;
159
- this.widthInMeters = widthInMeters;
157
+ constructor(private readonly exporter: USDZExporter, private readonly component: WebXRImageTracking) {
158
+ if (debug) console.log(this);
159
+ this.exporter.anchoringType = "image";
160
+ }
161
+
162
+ // set during export
163
+ private shouldExport: boolean = true;
164
+
165
+ private filename: string = "marker.png";
166
+ private imageModel: WebXRImageTrackingModel | null = null;
167
+
168
+ onBeforeBuildDocument(_context: USDZExporterContext) {
169
+
170
+ // check if this extension is the first image tracking extension in the list
171
+ // since iOS can only track one image at a time we only allow one image tracking extension to be active
172
+ // we have to determine this at the earlierst export callback
173
+ // all subsequent export callbacks should then check is shouldExport is set to true
174
+ // this should only be the case for exactly one extension
175
+ const index = this.exporter.extensions
176
+ .filter(e => {
177
+ const ext = (e as ImageTrackingExtension);
178
+ return ext.isImageTrackingExtension && ext.component.activeAndEnabled && ext.component.trackedImages?.length > 0;
179
+ })
180
+ .indexOf(this);
181
+ this.shouldExport = index === 0;
182
+ if (!this.shouldExport) return;
183
+
184
+ // Warn if more than one tracked image is used for USDZ; that's not supported at the moment.
185
+ if (this.component.trackedImages?.length > 1) {
186
+ if (debug || isDevEnvironment()) {
187
+ showBalloonWarning("USDZ: Only one tracked image is supported.");
188
+ console.warn("USDZ: Only one tracked image is supported. Will choose the first one in the trackedImages list");
189
+ }
190
+ }
160
191
  }
161
192
 
162
193
  onAfterHierarchy(_context: USDZExporterContext, writer: USDWriter) {
194
+ if (!this.shouldExport) return;
195
+
163
196
  const iOSVersion = DeviceUtilities.getiOSVersion();
164
197
  const majorVersion = iOSVersion ? parseInt(iOSVersion.split(".")[0]) : 18;
165
198
  const workaroundForFB16119331 = majorVersion >= 18;
166
199
  const multiplier = workaroundForFB16119331 ? 1 : 100;
167
200
  writer.beginBlock(`def Preliminary_ReferenceImage "AnchoringReferenceImage"`);
168
201
  writer.appendLine(`uniform asset image = @image_tracking/` + this.filename + `@`);
169
- writer.appendLine(`uniform double physicalWidth = ` + (this.widthInMeters * multiplier).toFixed(8));
202
+ writer.appendLine(`uniform double physicalWidth = ` + (this.imageModel!.widthInMeters * multiplier).toFixed(8));
170
203
  writer.closeBlock();
204
+
171
205
  }
172
206
 
173
- onBeforeBuildDocument(_context: USDZExporterContext) {
174
- const imageTracking = GameObject.findObjectOfType(WebXRImageTracking);
175
- if (!imageTracking || !imageTracking.trackedImages) return;
207
+ async onAfterSerialize(context: USDZExporterContext) {
208
+ if (!this.shouldExport) return;
176
209
 
177
- // Warn if more than one tracked image is used for USDZ; that's not supported at the moment.
178
- if (imageTracking.trackedImages.length > 1)
179
- {
180
- if (isDevEnvironment()) showBalloonWarning("USDZ: Only one tracked image is supported.");
181
- console.warn("USDZ: Only one tracked image is supported.");
182
- }
183
- }
210
+ const imageModel = this.imageModel;
211
+ const img = _imageElements.get(imageModel!.image!)!;
184
212
 
185
- onAfterSerialize(context: USDZExporterContext) {
186
- context.files['image_tracking/' + this.filename] = this.imageData;
213
+ const canvas = await imageToCanvas(img);
214
+ const blob = await canvas.convertToBlob({ type: 'image/png' });
215
+ const arrayBuffer = await blob.arrayBuffer();
216
+ context.files['image_tracking/' + this.filename] = new Uint8Array(arrayBuffer);
187
217
  }
188
218
 
189
219
  onExportObject(object: Object3D<Object3DEventMap>, model: USDObject, _context: USDZExporterContext) {
190
- const imageTracking = GameObject.findObjectOfType(WebXRImageTracking);
191
- if (!imageTracking || !imageTracking.trackedImages) return;
220
+ if (!this.shouldExport) return;
192
221
 
222
+ const imageTracking = this.component;
223
+ if (!imageTracking || !imageTracking.trackedImages?.length || !imageTracking.activeAndEnabled) return;
193
224
 
194
- for (const trackedImage of imageTracking.trackedImages) {
195
- if (trackedImage.object?.asset === object) {
196
- const exporter = GameObject.findObjectOfType(USDZExporter);
197
- if (!exporter) continue;
225
+ // we only care about the first image
226
+ const trackedImage = imageTracking.trackedImages[0];
198
227
 
199
- const { scale, target } = exporter.getARScaleAndTarget();
228
+ if (trackedImage.object?.asset === object) {
229
+ this.imageModel = trackedImage;
200
230
 
201
- // We have to reset the image tracking object's position and rotation, because QuickLook applies them.
202
- // On Android WebXR they're replaced by the tracked data
203
- let parent = object;
204
- const relativeMatrix = new Matrix4();
205
- if (object !== target) {
206
- while (parent.parent && parent.parent !== target) {
207
- parent = parent.parent;
208
- relativeMatrix.premultiply(parent.matrix);
209
- }
231
+ const { scale, target } = this.exporter.getARScaleAndTarget();
232
+
233
+ // We have to reset the image tracking object's position and rotation, because QuickLook applies them.
234
+ // On Android WebXR they're replaced by the tracked data
235
+ let parent = object;
236
+
237
+ const relativeMatrix = new Matrix4();
238
+ if (object !== target) {
239
+ while (parent && parent.parent && parent.parent !== target) {
240
+ parent = parent.parent;
241
+ relativeMatrix.premultiply(parent.matrix);
210
242
  }
211
- const mat = relativeMatrix
212
- .clone()
213
- .invert()
214
- // apply session root scale again after undoing the world transformation
215
- model.setMatrix(mat.scale(new Vector3(scale, scale, scale)));
216
-
217
- // Unfortunately looks like Apple's docs are incomplete:
218
- // https://developer.apple.com/documentation/realitykit/preliminary_anchoringapi#Nest-and-Layer-Anchorable-Prims
219
- // In practice, it seems that nesting is not allowed – no image tracking will be applied to nested objects.
220
- // Thus, we can't have separate transforms for "regularly placing content" and "placing content with an image marker".
221
- // model.extraSchemas.push("Preliminary_AnchoringAPI");
222
- // model.addEventListener("serialize", (_writer: USDWriter, _context: USDZExporterContext) => {
223
- // writer.appendLine( `token preliminary:anchoring:type = "image"` );
224
- // writer.appendLine( `rel preliminary:imageAnchoring:referenceImage = </${context.document.name}/Scenes/Scene/AnchoringReferenceImage>` );
225
- // });
226
-
227
- // We can only apply this to the first tracked image, more are not supported by QuickLook.
228
- break;
229
243
  }
244
+ const mat = relativeMatrix
245
+ .clone()
246
+ .invert()
247
+ // apply session root scale again after undoing the world transformation
248
+ model.setMatrix(mat.scale(new Vector3(scale, scale, scale)));
249
+
250
+
251
+ // Unfortunately looks like Apple's docs are incomplete:
252
+ // https://developer.apple.com/documentation/realitykit/preliminary_anchoringapi#Nest-and-Layer-Anchorable-Prims
253
+ // In practice, it seems that nesting is not allowed – no image tracking will be applied to nested objects.
254
+ // Thus, we can't have separate transforms for "regularly placing content" and "placing content with an image marker".
255
+ // model.extraSchemas.push("Preliminary_AnchoringAPI");
256
+ // model.addEventListener("serialize", (_writer: USDWriter, _context: USDZExporterContext) => {
257
+ // writer.appendLine( `token preliminary:anchoring:type = "image"` );
258
+ // writer.appendLine( `rel preliminary:imageAnchoring:referenceImage = </${context.document.name}/Scenes/Scene/AnchoringReferenceImage>` );
259
+ // });
260
+
261
+ // We can only apply this to the first tracked image, more are not supported by QuickLook.
230
262
  }
231
263
  }
232
264
  }
233
265
 
266
+
234
267
  /**
235
268
  * @category XR
236
269
  * @group Components
@@ -238,14 +271,13 @@ class ImageTrackingExtension implements IUSDExporterExtension {
238
271
  export class WebXRImageTracking extends Behaviour {
239
272
 
240
273
  @serializable(WebXRImageTrackingModel)
241
- trackedImages?: WebXRImageTrackingModel[];
274
+ trackedImages: WebXRImageTrackingModel[] = [];
242
275
 
243
276
  /** Applies smoothing based on detected jitter to the tracked image. */
244
277
  @serializable()
245
278
  smooth: boolean = true;
246
279
 
247
280
  private readonly trackedImageIndexMap: Map<number, WebXRImageTrackingModel> = new Map();
248
- private static _imageElements: Map<string, ImageBitmap | null> = new Map();
249
281
 
250
282
  /** @returns true if image tracking is supported on this device. This may return false at runtime if the user's browser did not enable webxr incubations */
251
283
  get supported() { return this._supported; }
@@ -257,38 +289,24 @@ export class WebXRImageTracking extends Behaviour {
257
289
  if (!this.trackedImages) return;
258
290
  for (const trackedImage of this.trackedImages) {
259
291
  if (trackedImage.image) {
260
- if (WebXRImageTracking._imageElements.has(trackedImage.image)) {
261
- // already loaded
262
- }
263
- else {
264
- const url = trackedImage.image;
265
- WebXRImageTracking._imageElements.set(url, null);
266
- const imageElement = document.createElement("img") as HTMLImageElement;
267
- imageElement.src = url;
268
- imageElement.addEventListener("load", async () => {
269
- const img = await createImageBitmap(imageElement);
270
- WebXRImageTracking._imageElements.set(url, img);
271
-
272
- // read back Uint8Array to use in USDZ -
273
- // TODO better would be to do that once we actually need it
274
- const canvas = await imageToCanvas(img);
275
- if (canvas) {
276
- const blob = await canvas.convertToBlob({ type: 'image/png' });
277
- const arrayBuffer = await blob.arrayBuffer();
278
-
279
- const exporter = GameObject.findObjectOfType(USDZExporter);
280
- if (exporter && this.trackedImages) {
281
- exporter.extensions.push(
282
- new ImageTrackingExtension("marker.png", new Uint8Array(arrayBuffer), this.trackedImages[0].widthInMeters)
283
- );
284
- exporter.anchoringType = "image";
285
- }
286
- }
287
- });
288
- }
292
+ loadImage(trackedImage.image);
289
293
  }
290
294
  }
291
295
  }
296
+ onEnable() {
297
+ USDZExporter.beforeExport.addEventListener(this.onBeforeUSDZExport);
298
+ }
299
+ onDisable(): void {
300
+ USDZExporter.beforeExport.removeEventListener(this.onBeforeUSDZExport);
301
+ }
302
+
303
+ private onBeforeUSDZExport = (args: { exporter: USDZExporter }) => {
304
+ if (this.activeAndEnabled && this.trackedImages?.length) {
305
+ args.exporter.extensions.push(new ImageTrackingExtension(args.exporter, this));
306
+ }
307
+ }
308
+
309
+
292
310
 
293
311
  onBeforeXR(_mode: XRSessionMode, args: XRSessionInit & { trackedImages: Array<any> }): void {
294
312
  // console.log("onXRRequested", args, this.trackedImages)
@@ -300,7 +318,7 @@ export class WebXRImageTracking extends Behaviour {
300
318
  args.trackedImages = [];
301
319
  for (const trackedImage of this.trackedImages) {
302
320
  if (trackedImage.image?.length && trackedImage.widthInMeters > 0) {
303
- const bitmap = WebXRImageTracking._imageElements.get(trackedImage.image);
321
+ const bitmap = _imageElements.get(trackedImage.image);
304
322
  if (bitmap) {
305
323
  this.trackedImageIndexMap.set(args.trackedImages.length, trackedImage);
306
324
  args.trackedImages.push({
@@ -337,7 +355,7 @@ export class WebXRImageTracking extends Behaviour {
337
355
 
338
356
  onLeaveXR(_args: NeedleXREventArgs): void {
339
357
 
340
- if(!this.supported && DeviceUtilities.isAndroidDevice()) {
358
+ if (!this.supported && DeviceUtilities.isAndroidDevice()) {
341
359
  showBalloonWarning(this.webXRIncubationsWarning);
342
360
  }
343
361
 
@@ -364,7 +382,7 @@ export class WebXRImageTracking extends Behaviour {
364
382
  private readonly imageToObjectMap = new Map<WebXRImageTrackingModel, { object: Object3D | null, frames: number, lastTrackingTime: number }>();
365
383
  private readonly currentImages: WebXRTrackedImage[] = [];
366
384
 
367
-
385
+
368
386
  private readonly webXRIncubationsWarning = "Image tracking is currently not supported on this device. On Chrome for Android, you can enable the <a target=\"_blank\" href=\"#\" onclick=\"() => console.log('I')\">chrome://flags/#webxr-incubations</a> flag.";
369
387
 
370
388
  onUpdateXR(args: NeedleXREventArgs): void {
@@ -517,4 +535,34 @@ export class WebXRImageTracking extends Behaviour {
517
535
  }
518
536
  }
519
537
  }
538
+ }
539
+
540
+
541
+
542
+
543
+ const _imageElements: Map<string, ImageBitmap | null> = new Map();
544
+ const _imageLoadingPromises: Map<string, Promise<boolean>> = new Map();
545
+
546
+ async function loadImage(url: string) {
547
+ if (_imageElements.has(url)) {
548
+ if (_imageLoadingPromises.has(url)) return _imageLoadingPromises.get(url);
549
+ return Promise.resolve(true);
550
+ }
551
+ const promise = new Promise<boolean>(res => {
552
+ _imageElements.set(url, null);
553
+ const imageElement = document.createElement("img") as HTMLImageElement;
554
+ imageElement.src = url;
555
+ imageElement.addEventListener("load", async () => {
556
+ const img = await createImageBitmap(imageElement);
557
+ _imageElements.set(url, img);
558
+ res(true);
559
+ });
560
+ });
561
+
562
+ _imageLoadingPromises.set(url, promise);
563
+ promise.finally(() => {
564
+ _imageLoadingPromises.delete(url);
565
+ });
566
+
567
+ return promise;
520
568
  }