@loaders.gl/tile-converter 3.1.8 → 3.2.0-alpha.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (238) hide show
  1. package/dist/3d-tiles-attributes-worker.d.ts +28 -0
  2. package/dist/3d-tiles-attributes-worker.d.ts.map +1 -0
  3. package/dist/3d-tiles-attributes-worker.js +4 -0
  4. package/dist/3d-tiles-attributes-worker.js.map +7 -0
  5. package/dist/3d-tiles-converter/3d-tiles-converter.d.ts +82 -0
  6. package/dist/3d-tiles-converter/3d-tiles-converter.d.ts.map +1 -0
  7. package/dist/3d-tiles-converter/3d-tiles-converter.js +268 -0
  8. package/dist/3d-tiles-converter/helpers/b3dm-converter.d.ts +84 -0
  9. package/dist/3d-tiles-converter/helpers/b3dm-converter.d.ts.map +1 -0
  10. package/dist/3d-tiles-converter/helpers/b3dm-converter.js +278 -0
  11. package/dist/3d-tiles-converter/helpers/i3s-obb-to-3d-tiles-obb.d.ts +13 -0
  12. package/dist/3d-tiles-converter/helpers/i3s-obb-to-3d-tiles-obb.d.ts.map +1 -0
  13. package/dist/3d-tiles-converter/helpers/i3s-obb-to-3d-tiles-obb.js +23 -0
  14. package/dist/3d-tiles-converter/helpers/texture-atlas.d.ts +9 -0
  15. package/dist/3d-tiles-converter/helpers/texture-atlas.d.ts.map +1 -0
  16. package/dist/3d-tiles-converter/helpers/texture-atlas.js +52 -0
  17. package/dist/3d-tiles-converter/json-templates/tileset.d.ts +15 -0
  18. package/dist/3d-tiles-converter/json-templates/tileset.d.ts.map +1 -0
  19. package/dist/3d-tiles-converter/json-templates/tileset.js +43 -0
  20. package/dist/bundle.d.ts +2 -0
  21. package/dist/bundle.d.ts.map +1 -0
  22. package/dist/bundle.js +5 -0
  23. package/dist/converter-cli.d.ts +2 -0
  24. package/dist/converter-cli.d.ts.map +1 -0
  25. package/dist/converter-cli.js +232 -0
  26. package/dist/converter.min.js +68 -68
  27. package/dist/deps-installer/deps-installer.d.ts +14 -0
  28. package/dist/deps-installer/deps-installer.d.ts.map +1 -0
  29. package/dist/deps-installer/deps-installer.js +31 -0
  30. package/dist/dist.min.js +1151 -1264
  31. package/dist/es5/3d-tiles-attributes-worker.js +29 -0
  32. package/dist/es5/3d-tiles-attributes-worker.js.map +1 -0
  33. package/dist/es5/3d-tiles-converter/3d-tiles-converter.js +104 -44
  34. package/dist/es5/3d-tiles-converter/3d-tiles-converter.js.map +1 -1
  35. package/dist/es5/3d-tiles-converter/helpers/b3dm-converter.js +34 -43
  36. package/dist/es5/3d-tiles-converter/helpers/b3dm-converter.js.map +1 -1
  37. package/dist/es5/converter-cli.js +306 -0
  38. package/dist/es5/converter-cli.js.map +1 -0
  39. package/dist/es5/deps-installer/deps-installer.js.map +1 -1
  40. package/dist/es5/i3s-attributes-worker.js +29 -0
  41. package/dist/es5/i3s-attributes-worker.js.map +1 -0
  42. package/dist/es5/i3s-converter/helpers/coordinate-converter.js +19 -11
  43. package/dist/es5/i3s-converter/helpers/coordinate-converter.js.map +1 -1
  44. package/dist/es5/i3s-converter/helpers/geometry-attributes.js +2 -2
  45. package/dist/es5/i3s-converter/helpers/geometry-attributes.js.map +1 -1
  46. package/dist/es5/i3s-converter/helpers/geometry-converter.js +271 -182
  47. package/dist/es5/i3s-converter/helpers/geometry-converter.js.map +1 -1
  48. package/dist/es5/i3s-converter/helpers/gltf-attributes.js +71 -0
  49. package/dist/es5/i3s-converter/helpers/gltf-attributes.js.map +1 -0
  50. package/dist/es5/i3s-converter/helpers/node-pages.js +47 -99
  51. package/dist/es5/i3s-converter/helpers/node-pages.js.map +1 -1
  52. package/dist/es5/i3s-converter/i3s-converter.js +311 -245
  53. package/dist/es5/i3s-converter/i3s-converter.js.map +1 -1
  54. package/dist/es5/index.js +0 -16
  55. package/dist/es5/index.js.map +1 -1
  56. package/dist/es5/lib/utils/compress-util.js +14 -17
  57. package/dist/es5/lib/utils/compress-util.js.map +1 -1
  58. package/dist/es5/lib/utils/file-utils.js +39 -14
  59. package/dist/es5/lib/utils/file-utils.js.map +1 -1
  60. package/dist/es5/lib/utils/lod-conversion-utils.js.map +1 -1
  61. package/dist/es5/lib/utils/queue.js +61 -0
  62. package/dist/es5/lib/utils/queue.js.map +1 -0
  63. package/dist/es5/lib/utils/statistic-utills.js.map +1 -1
  64. package/dist/es5/lib/utils/write-queue.js +225 -0
  65. package/dist/es5/lib/utils/write-queue.js.map +1 -0
  66. package/dist/es5/pgm-loader.js +1 -1
  67. package/dist/es5/pgm-loader.js.map +1 -1
  68. package/dist/es5/workers/3d-tiles-attributes-worker.js +37 -0
  69. package/dist/es5/workers/3d-tiles-attributes-worker.js.map +1 -0
  70. package/dist/es5/workers/i3s-attributes-worker.js +40 -0
  71. package/dist/es5/workers/i3s-attributes-worker.js.map +1 -0
  72. package/dist/esm/3d-tiles-attributes-worker.js +16 -0
  73. package/dist/esm/3d-tiles-attributes-worker.js.map +1 -0
  74. package/dist/esm/3d-tiles-converter/3d-tiles-converter.js +32 -5
  75. package/dist/esm/3d-tiles-converter/3d-tiles-converter.js.map +1 -1
  76. package/dist/esm/3d-tiles-converter/helpers/b3dm-converter.js +23 -23
  77. package/dist/esm/3d-tiles-converter/helpers/b3dm-converter.js.map +1 -1
  78. package/dist/esm/converter-cli.js +230 -0
  79. package/dist/esm/converter-cli.js.map +1 -0
  80. package/dist/esm/deps-installer/deps-installer.js.map +1 -1
  81. package/dist/esm/i3s-attributes-worker.js +16 -0
  82. package/dist/esm/i3s-attributes-worker.js.map +1 -0
  83. package/dist/esm/i3s-converter/helpers/coordinate-converter.js +19 -11
  84. package/dist/esm/i3s-converter/helpers/coordinate-converter.js.map +1 -1
  85. package/dist/esm/i3s-converter/helpers/geometry-attributes.js +2 -2
  86. package/dist/esm/i3s-converter/helpers/geometry-attributes.js.map +1 -1
  87. package/dist/esm/i3s-converter/helpers/geometry-converter.js +121 -62
  88. package/dist/esm/i3s-converter/helpers/geometry-converter.js.map +1 -1
  89. package/dist/esm/i3s-converter/helpers/gltf-attributes.js +54 -0
  90. package/dist/esm/i3s-converter/helpers/gltf-attributes.js.map +1 -0
  91. package/dist/esm/i3s-converter/helpers/node-pages.js +12 -4
  92. package/dist/esm/i3s-converter/helpers/node-pages.js.map +1 -1
  93. package/dist/esm/i3s-converter/i3s-converter.js +155 -50
  94. package/dist/esm/i3s-converter/i3s-converter.js.map +1 -1
  95. package/dist/esm/index.js +0 -2
  96. package/dist/esm/index.js.map +1 -1
  97. package/dist/esm/lib/utils/compress-util.js +6 -8
  98. package/dist/esm/lib/utils/compress-util.js.map +1 -1
  99. package/dist/esm/lib/utils/file-utils.js +11 -1
  100. package/dist/esm/lib/utils/file-utils.js.map +1 -1
  101. package/dist/esm/lib/utils/lod-conversion-utils.js.map +1 -1
  102. package/dist/esm/lib/utils/queue.js +19 -0
  103. package/dist/esm/lib/utils/queue.js.map +1 -0
  104. package/dist/esm/lib/utils/statistic-utills.js.map +1 -1
  105. package/dist/esm/lib/utils/write-queue.js +88 -0
  106. package/dist/esm/lib/utils/write-queue.js.map +1 -0
  107. package/dist/esm/pgm-loader.js +1 -1
  108. package/dist/esm/pgm-loader.js.map +1 -1
  109. package/dist/esm/workers/3d-tiles-attributes-worker.js +5 -0
  110. package/dist/esm/workers/3d-tiles-attributes-worker.js.map +1 -0
  111. package/dist/esm/workers/i3s-attributes-worker.js +4 -0
  112. package/dist/esm/workers/i3s-attributes-worker.js.map +1 -0
  113. package/dist/i3s-attributes-worker.d.ts +33 -0
  114. package/dist/i3s-attributes-worker.d.ts.map +1 -0
  115. package/dist/i3s-attributes-worker.js +10 -0
  116. package/dist/i3s-attributes-worker.js.map +7 -0
  117. package/dist/i3s-converter/helpers/coordinate-converter.d.ts +41 -0
  118. package/dist/i3s-converter/helpers/coordinate-converter.d.ts.map +1 -0
  119. package/dist/i3s-converter/helpers/coordinate-converter.js +122 -0
  120. package/dist/i3s-converter/helpers/create-scene-server-path.d.ts +9 -0
  121. package/dist/i3s-converter/helpers/create-scene-server-path.d.ts.map +1 -0
  122. package/dist/i3s-converter/helpers/create-scene-server-path.js +28 -0
  123. package/dist/i3s-converter/helpers/geometry-attributes.d.ts +8 -0
  124. package/dist/i3s-converter/helpers/geometry-attributes.d.ts.map +1 -0
  125. package/dist/i3s-converter/helpers/geometry-attributes.js +177 -0
  126. package/dist/i3s-converter/helpers/geometry-converter.d.ts +29 -0
  127. package/dist/i3s-converter/helpers/geometry-converter.d.ts.map +1 -0
  128. package/dist/i3s-converter/helpers/geometry-converter.js +901 -0
  129. package/dist/i3s-converter/helpers/gltf-attributes.d.ts +9 -0
  130. package/dist/i3s-converter/helpers/gltf-attributes.d.ts.map +1 -0
  131. package/dist/i3s-converter/helpers/gltf-attributes.js +56 -0
  132. package/dist/i3s-converter/helpers/node-debug.d.ts +8 -0
  133. package/dist/i3s-converter/helpers/node-debug.d.ts.map +1 -0
  134. package/dist/i3s-converter/helpers/node-debug.js +114 -0
  135. package/dist/i3s-converter/helpers/node-pages.d.ts +117 -0
  136. package/dist/i3s-converter/helpers/node-pages.d.ts.map +1 -0
  137. package/dist/i3s-converter/helpers/node-pages.js +208 -0
  138. package/dist/i3s-converter/i3s-converter.d.ts +323 -0
  139. package/dist/i3s-converter/i3s-converter.d.ts.map +1 -0
  140. package/dist/i3s-converter/i3s-converter.js +1074 -0
  141. package/dist/i3s-converter/json-templates/layers.d.ts +95 -0
  142. package/dist/i3s-converter/json-templates/layers.d.ts.map +1 -0
  143. package/dist/i3s-converter/json-templates/layers.js +199 -0
  144. package/dist/i3s-converter/json-templates/metadata.d.ts +22 -0
  145. package/dist/i3s-converter/json-templates/metadata.d.ts.map +1 -0
  146. package/dist/i3s-converter/json-templates/metadata.js +25 -0
  147. package/dist/i3s-converter/json-templates/node.d.ts +61 -0
  148. package/dist/i3s-converter/json-templates/node.d.ts.map +1 -0
  149. package/dist/i3s-converter/json-templates/node.js +89 -0
  150. package/dist/i3s-converter/json-templates/scene-server.d.ts +28 -0
  151. package/dist/i3s-converter/json-templates/scene-server.d.ts.map +1 -0
  152. package/dist/i3s-converter/json-templates/scene-server.js +31 -0
  153. package/dist/i3s-converter/json-templates/shared-resources.d.ts +14 -0
  154. package/dist/i3s-converter/json-templates/shared-resources.d.ts.map +1 -0
  155. package/dist/i3s-converter/json-templates/shared-resources.js +129 -0
  156. package/dist/i3s-converter/json-templates/store.d.ts +95 -0
  157. package/dist/i3s-converter/json-templates/store.d.ts.map +1 -0
  158. package/dist/i3s-converter/json-templates/store.js +103 -0
  159. package/dist/i3s-converter/types.d.ts +114 -0
  160. package/dist/i3s-converter/types.d.ts.map +1 -0
  161. package/dist/i3s-converter/types.js +2 -0
  162. package/dist/i3s-server/app.d.ts +3 -0
  163. package/dist/i3s-server/app.d.ts.map +1 -0
  164. package/dist/i3s-server/app.js +14 -0
  165. package/dist/i3s-server/controllers/index-controller.d.ts +2 -0
  166. package/dist/i3s-server/controllers/index-controller.d.ts.map +1 -0
  167. package/dist/i3s-server/controllers/index-controller.js +23 -0
  168. package/dist/i3s-server/routes/index.d.ts +3 -0
  169. package/dist/i3s-server/routes/index.d.ts.map +1 -0
  170. package/dist/i3s-server/routes/index.js +16 -0
  171. package/dist/index.d.ts +3 -0
  172. package/dist/index.d.ts.map +1 -0
  173. package/dist/index.js +10 -0
  174. package/dist/lib/utils/compress-util.d.ts +45 -0
  175. package/dist/lib/utils/compress-util.d.ts.map +1 -0
  176. package/dist/lib/utils/compress-util.js +257 -0
  177. package/dist/{esm/lib → lib}/utils/file-utils.d.ts +6 -14
  178. package/dist/lib/utils/file-utils.d.ts.map +1 -0
  179. package/dist/lib/utils/file-utils.js +81 -0
  180. package/dist/lib/utils/lod-conversion-utils.d.ts +41 -0
  181. package/dist/lib/utils/lod-conversion-utils.d.ts.map +1 -0
  182. package/dist/lib/utils/lod-conversion-utils.js +76 -0
  183. package/dist/lib/utils/queue.d.ts +7 -0
  184. package/dist/lib/utils/queue.d.ts.map +1 -0
  185. package/dist/lib/utils/queue.js +18 -0
  186. package/dist/lib/utils/statistic-utills.d.ts +3 -0
  187. package/dist/lib/utils/statistic-utills.d.ts.map +1 -0
  188. package/dist/lib/utils/statistic-utills.js +64 -0
  189. package/dist/lib/utils/write-queue.d.ts +22 -0
  190. package/dist/lib/utils/write-queue.d.ts.map +1 -0
  191. package/dist/lib/utils/write-queue.js +62 -0
  192. package/dist/pgm-loader.d.ts +6 -0
  193. package/dist/pgm-loader.d.ts.map +1 -0
  194. package/dist/pgm-loader.js +23 -0
  195. package/dist/workers/3d-tiles-attributes-worker.d.ts +2 -0
  196. package/dist/workers/3d-tiles-attributes-worker.d.ts.map +1 -0
  197. package/dist/workers/3d-tiles-attributes-worker.js +9 -0
  198. package/dist/workers/i3s-attributes-worker.d.ts +2 -0
  199. package/dist/workers/i3s-attributes-worker.d.ts.map +1 -0
  200. package/dist/workers/i3s-attributes-worker.js +5 -0
  201. package/package.json +19 -18
  202. package/src/3d-tiles-attributes-worker.ts +43 -0
  203. package/src/3d-tiles-converter/3d-tiles-converter.ts +49 -6
  204. package/src/3d-tiles-converter/helpers/b3dm-converter.ts +21 -18
  205. package/src/converter-cli.ts +310 -0
  206. package/src/deps-installer/{deps-installer.js → deps-installer.ts} +11 -1
  207. package/src/i3s-attributes-worker.ts +46 -0
  208. package/src/i3s-converter/helpers/coordinate-converter.ts +29 -24
  209. package/src/i3s-converter/helpers/geometry-attributes.ts +4 -3
  210. package/src/i3s-converter/helpers/{geometry-converter.js → geometry-converter.ts} +425 -179
  211. package/src/i3s-converter/helpers/gltf-attributes.ts +68 -0
  212. package/src/i3s-converter/helpers/node-pages.ts +25 -17
  213. package/src/i3s-converter/i3s-converter.ts +154 -87
  214. package/src/i3s-converter/types.ts +90 -8
  215. package/src/index.ts +0 -4
  216. package/src/lib/utils/{compress-util.js → compress-util.ts} +105 -18
  217. package/src/lib/utils/file-utils.ts +84 -0
  218. package/src/lib/utils/{lod-conversion-utils.js → lod-conversion-utils.ts} +27 -5
  219. package/src/lib/utils/queue.ts +17 -0
  220. package/src/lib/utils/{statistic-utills.js → statistic-utills.ts} +0 -0
  221. package/src/lib/utils/write-queue.ts +75 -0
  222. package/src/workers/3d-tiles-attributes-worker.ts +6 -0
  223. package/src/workers/i3s-attributes-worker.ts +6 -0
  224. package/dist/es5/deps-installer/deps-installer.d.ts +0 -10
  225. package/dist/es5/i3s-converter/helpers/geometry-converter.d.ts +0 -44
  226. package/dist/es5/lib/utils/compress-util.d.ts +0 -53
  227. package/dist/es5/lib/utils/file-utils.d.ts +0 -43
  228. package/dist/es5/lib/utils/lod-conversion-utils.d.ts +0 -32
  229. package/dist/esm/deps-installer/deps-installer.d.ts +0 -10
  230. package/dist/esm/i3s-converter/helpers/geometry-converter.d.ts +0 -44
  231. package/dist/esm/lib/utils/compress-util.d.ts +0 -53
  232. package/dist/esm/lib/utils/lod-conversion-utils.d.ts +0 -32
  233. package/src/deps-installer/deps-installer.d.ts +0 -10
  234. package/src/i3s-converter/helpers/geometry-converter.d.ts +0 -44
  235. package/src/lib/utils/compress-util.d.ts +0 -53
  236. package/src/lib/utils/file-utils.d.ts +0 -43
  237. package/src/lib/utils/file-utils.js +0 -38
  238. package/src/lib/utils/lod-conversion-utils.d.ts +0 -32
@@ -0,0 +1,68 @@
1
+ import type {B3DMContent} from '@loaders.gl/3d-tiles';
2
+ import type {Accessor} from 'modules/gltf/src/lib/types/gltf-postprocessed-schema';
3
+ import type {B3DMAttributesData} from '../../i3s-attributes-worker';
4
+
5
+ type AttributesObject = {
6
+ [k: string]: Accessor;
7
+ };
8
+
9
+ /**
10
+ * Keep only values for B3DM attributes to pass data to worker thread.
11
+ * @param attributes
12
+ */
13
+ function getB3DMAttributesWithoutBufferView(attributes: AttributesObject): AttributesObject {
14
+ const attributesWithoutBufferView = {};
15
+
16
+ for (const attributeName in attributes) {
17
+ attributesWithoutBufferView[attributeName] = {
18
+ value: attributes[attributeName].value
19
+ };
20
+ }
21
+
22
+ return attributesWithoutBufferView;
23
+ }
24
+
25
+ /**
26
+ * Prepare attributes for conversion to avoid binary data breaking in worker thread.
27
+ * @param tileContent
28
+ * @returns
29
+ */
30
+ export function prepareDataForAttributesConversion(tileContent: B3DMContent): B3DMAttributesData {
31
+ const gltfMaterials = tileContent.gltf?.materials?.map((material) => ({id: material.id}));
32
+ let nodes =
33
+ tileContent.gltf?.scene?.nodes ||
34
+ tileContent.gltf?.scenes?.[0]?.nodes ||
35
+ tileContent.gltf?.nodes ||
36
+ [];
37
+
38
+ const prepearedNodes = nodes.map((node) => {
39
+ if (!node.mesh) {
40
+ return node;
41
+ }
42
+
43
+ return {
44
+ ...node,
45
+ mesh: {
46
+ ...node.mesh,
47
+ primitives: node.mesh?.primitives.map((primitive) => ({
48
+ ...primitive,
49
+ indices: {value: primitive?.indices?.value},
50
+ attributes: getB3DMAttributesWithoutBufferView(primitive.attributes),
51
+ material: {
52
+ id: primitive?.material?.id
53
+ }
54
+ }))
55
+ }
56
+ };
57
+ });
58
+
59
+ const cartographicOrigin = tileContent.cartographicOrigin;
60
+ const cartesianModelMatrix = tileContent.cartesianModelMatrix;
61
+
62
+ return {
63
+ gltfMaterials,
64
+ nodes: prepearedNodes,
65
+ cartographicOrigin,
66
+ cartesianModelMatrix
67
+ };
68
+ }
@@ -1,7 +1,10 @@
1
+ import type {WriteQueueItem} from '../../lib/utils/write-queue';
2
+
1
3
  import {join} from 'path';
2
4
  import transform from 'json-map-transform';
3
5
  import {METADATA as metadataTemplate} from '../json-templates/metadata';
4
6
  import {NodeInPage} from '@loaders.gl/i3s';
7
+ import WriteQueue from '../../lib/utils/write-queue';
5
8
 
6
9
  // @ts-nocheck
7
10
  /**
@@ -188,35 +191,40 @@ export default class NodePages {
188
191
  /**
189
192
  * Save all the node pages
190
193
  * Run this method when all nodes is pushed in nodePages
191
- * @param {string} layers0Path - path of layer
192
- * @param {Object} fileMap - fileMap which keep info for slpk archive
193
- * @param {boolean} slpk
194
- * @return {promise}
194
+ * @param layers0Path - path of layer
195
+ * @param writeQueue - write queue that controlls files write concurrency
196
+ * @param slpk
195
197
  */
196
- async save(layers0Path: string, fileMap: Object, slpk: boolean = false): Promise<void> {
198
+ async save(
199
+ layers0Path: string,
200
+ writeQueue: WriteQueue<WriteQueueItem>,
201
+ slpk: boolean = false
202
+ ): Promise<void> {
197
203
  if (slpk) {
198
204
  for (const [index, nodePage] of this.nodePages.entries()) {
199
205
  const nodePageStr = JSON.stringify(nodePage);
200
206
  const slpkPath = join(layers0Path, 'nodepages');
201
- fileMap[`nodePages/${index.toString()}.json.gz`] = await this.writeFile(
202
- slpkPath,
203
- nodePageStr,
204
- `${index.toString()}.json`
205
- );
207
+ writeQueue.enqueue({
208
+ archiveKey: `nodePages/${index.toString()}.json.gz`,
209
+ writePromise: this.writeFile(slpkPath, nodePageStr, `${index.toString()}.json`)
210
+ });
206
211
  }
207
212
  const metadata = transform({nodeCount: this.nodesCounter}, metadataTemplate());
208
213
  const compress = false;
209
- fileMap['metadata.json'] = await this.writeFile(
210
- layers0Path,
211
- JSON.stringify(metadata),
212
- 'metadata.json',
213
- compress
214
- );
214
+ writeQueue.enqueue({
215
+ archiveKey: 'metadata.json',
216
+ writePromise: this.writeFile(
217
+ layers0Path,
218
+ JSON.stringify(metadata),
219
+ 'metadata.json',
220
+ compress
221
+ )
222
+ });
215
223
  } else {
216
224
  for (const [index, nodePage] of this.nodePages.entries()) {
217
225
  const nodePageStr = JSON.stringify(nodePage);
218
226
  const nodePagePath = join(layers0Path, 'nodepages', index.toString());
219
- await this.writeFile(nodePagePath, nodePageStr);
227
+ writeQueue.enqueue({writePromise: this.writeFile(nodePagePath, nodePageStr)});
220
228
  }
221
229
  }
222
230
  }
@@ -1,6 +1,6 @@
1
1
  import type {Tile3D, Tileset3DProps} from '@loaders.gl/tiles';
2
2
  import type {BatchTableJson, B3DMContent} from '@loaders.gl/3d-tiles';
3
-
3
+ import type {WriteQueueItem} from '../lib/utils/write-queue';
4
4
  import type {
5
5
  AttributeStorageInfo,
6
6
  SceneLayer3D,
@@ -10,14 +10,13 @@ import type {
10
10
  MaxScreenThresholdSQ,
11
11
  NodeInPage,
12
12
  LodSelection,
13
- SharedResources,
14
13
  Attribute,
15
14
  ESRIField,
16
15
  Field,
17
16
  PopupInfo,
18
17
  FieldInfo
19
18
  } from '@loaders.gl/i3s';
20
- import {load, encode} from '@loaders.gl/core';
19
+ import {load, encode, fetchFile, getLoaderOptions} from '@loaders.gl/core';
21
20
  import {Tileset3D} from '@loaders.gl/tiles';
22
21
  import {CesiumIonLoader, Tiles3DLoader} from '@loaders.gl/3d-tiles';
23
22
  import {Geoid} from '@math.gl/geoid';
@@ -38,7 +37,7 @@ import {calculateFilesSize, timeConverter} from '../lib/utils/statistic-utills';
38
37
  import convertB3dmToI3sGeometry from './helpers/geometry-converter';
39
38
  import {
40
39
  createBoundingVolumes,
41
- convertCommonToI3SExtentCoordinate
40
+ convertBoundingVolumeToI3SFullExtent
42
41
  } from './helpers/coordinate-converter';
43
42
  import {createSceneServerPath} from './helpers/create-scene-server-path';
44
43
  import {convertGeometricErrorToScreenThreshold} from '../lib/utils/lod-conversion-utils';
@@ -49,12 +48,16 @@ import {NODE as nodeTemplate} from './json-templates/node';
49
48
  import {SHARED_RESOURCES as sharedResourcesTemplate} from './json-templates/shared-resources';
50
49
  import {validateNodeBoundingVolumes} from './helpers/node-debug';
51
50
  import TileHeader from '@loaders.gl/tiles/src/tileset/tile-3d';
52
- import {KTX2BasisUniversalTextureWriter} from '@loaders.gl/textures';
51
+ import {KTX2BasisWriterWorker} from '@loaders.gl/textures';
53
52
  import {LoaderWithParser} from '@loaders.gl/loader-utils';
54
53
  import {I3SMaterialDefinition, TextureSetDefinitionFormats} from '@loaders.gl/i3s/src/types';
55
54
  import {ImageWriter} from '@loaders.gl/images';
56
55
  import {GLTFImagePostprocessed} from '@loaders.gl/gltf';
57
- import {I3SConvertedResources} from './types';
56
+ import {I3SConvertedResources, SharedResourcesArrays} from './types';
57
+ import {getWorkerURL, WorkerFarm} from '@loaders.gl/worker-utils';
58
+ import {DracoWriterWorker} from '@loaders.gl/draco';
59
+ import WriteQueue from '../lib/utils/write-queue';
60
+ import {I3SAttributesWorker} from '../i3s-attributes-worker';
58
61
 
59
62
  const ION_DEFAULT_TOKEN =
60
63
  process.env.IonToken || // eslint-disable-line
@@ -75,7 +78,6 @@ const CESIUM_DATASET_PREFIX = 'https://';
75
78
  */
76
79
  export default class I3SConverter {
77
80
  nodePages: NodePages;
78
- fileMap: {[key: string]: string};
79
81
  options: any;
80
82
  layers0Path: string;
81
83
  materialMap: Map<any, any>;
@@ -96,10 +98,12 @@ export default class I3SConverter {
96
98
  Loader: LoaderWithParser = Tiles3DLoader;
97
99
  generateTextures: boolean;
98
100
  generateBoundingVolumes: boolean;
101
+ layersHasTexture: boolean;
102
+ workerSource: {[key: string]: string} = {};
103
+ writeQueue: WriteQueue<WriteQueueItem> = new WriteQueue();
99
104
 
100
105
  constructor() {
101
106
  this.nodePages = new NodePages(writeFile, HARDCODED_NODES_PER_PAGE);
102
- this.fileMap = {};
103
107
  this.options = {};
104
108
  this.layers0Path = '';
105
109
  this.materialMap = new Map();
@@ -114,6 +118,7 @@ export default class I3SConverter {
114
118
  this.validate = false;
115
119
  this.generateTextures = false;
116
120
  this.generateBoundingVolumes = false;
121
+ this.layersHasTexture = false;
117
122
  }
118
123
 
119
124
  /**
@@ -143,8 +148,6 @@ export default class I3SConverter {
143
148
  validate?: boolean;
144
149
  generateTextures?: boolean;
145
150
  generateBoundingVolumes?: boolean;
146
- /** @deprecated */
147
- inputType?: string;
148
151
  }): Promise<any> {
149
152
  this.conversionStartTime = process.hrtime();
150
153
  const {
@@ -167,6 +170,9 @@ export default class I3SConverter {
167
170
  this.generateTextures = Boolean(generateTextures);
168
171
  this.generateBoundingVolumes = Boolean(generateBoundingVolumes);
169
172
 
173
+ this.writeQueue = new WriteQueue();
174
+ this.writeQueue.startListening();
175
+
170
176
  console.log('Loading egm file...'); // eslint-disable-line
171
177
  this.geoidHeightModel = await load(egmFilePath, PGMLoader);
172
178
  console.log('Loading egm file completed!'); // eslint-disable-line
@@ -175,19 +181,29 @@ export default class I3SConverter {
175
181
  this.nodePages.useWriteFunction(writeFileForSlpk);
176
182
  }
177
183
 
178
- const preloadOptions = await this._fetchPreloadOptions();
179
- const tilesetOptions: Tileset3DProps = {loadOptions: {basis: {format: 'rgba32'}}};
180
- if (preloadOptions.headers) {
181
- tilesetOptions.loadOptions!.fetch = {headers: preloadOptions.headers};
184
+ await this.loadWorkers();
185
+
186
+ try {
187
+ const preloadOptions = await this._fetchPreloadOptions();
188
+ const tilesetOptions: Tileset3DProps = {loadOptions: {basis: {format: 'rgba32'}}};
189
+ if (preloadOptions.headers) {
190
+ tilesetOptions.loadOptions!.fetch = {headers: preloadOptions.headers};
191
+ }
192
+ Object.assign(tilesetOptions, preloadOptions);
193
+ const sourceTilesetJson = await load(inputUrl, this.Loader, tilesetOptions.loadOptions);
194
+ // console.log(tilesetJson); // eslint-disable-line
195
+ this.sourceTileset = new Tileset3D(sourceTilesetJson, tilesetOptions);
196
+
197
+ await this._createAndSaveTileset(outputPath, tilesetName);
198
+ await this._finishConversion({slpk: Boolean(slpk), outputPath, tilesetName});
199
+ return sourceTilesetJson;
200
+ } catch (error) {
201
+ throw error;
202
+ } finally {
203
+ // Clean up worker pools
204
+ const workerFarm = WorkerFarm.getWorkerFarm({});
205
+ workerFarm.destroy();
182
206
  }
183
- Object.assign(tilesetOptions, preloadOptions);
184
- const sourceTilesetJson = await load(inputUrl, this.Loader, tilesetOptions.loadOptions);
185
- // console.log(tilesetJson); // eslint-disable-line
186
- this.sourceTileset = new Tileset3D(sourceTilesetJson, tilesetOptions);
187
-
188
- await this._createAndSaveTileset(outputPath, tilesetName);
189
- await this._finishConversion({slpk: Boolean(slpk), outputPath, tilesetName});
190
- return sourceTilesetJson;
191
207
  }
192
208
 
193
209
  /**
@@ -226,10 +242,19 @@ export default class I3SConverter {
226
242
  await this._convertNodesTree(root0, sourceRootTile, parentId, boundingVolumes);
227
243
 
228
244
  this.layers0!.materialDefinitions = this.materialDefinitions;
245
+
246
+ if (this.layersHasTexture === false) {
247
+ this.layers0!.store.defaultGeometrySchema.ordering =
248
+ this.layers0!.store.defaultGeometrySchema.ordering.filter(
249
+ (attribute) => attribute !== 'uv0'
250
+ );
251
+ }
252
+
229
253
  await this._writeLayers0();
230
254
  createSceneServerPath(tilesetName, this.layers0!, tilesetPath);
231
255
  await this._writeNodeIndexDocument(root0, 'root', join(this.layers0Path, 'nodes', 'root'));
232
- await this.nodePages.save(this.layers0Path, this.fileMap, isCreateSlpk);
256
+ await this.nodePages.save(this.layers0Path, this.writeQueue, isCreateSlpk);
257
+ await this.writeQueue.finalize();
233
258
  await this._createSlpk(tilesetPath);
234
259
  }
235
260
 
@@ -238,7 +263,10 @@ export default class I3SConverter {
238
263
  * @param tilesetName - Name of layer
239
264
  */
240
265
  private _formLayers0(tilesetName: string): void {
241
- const extent = convertCommonToI3SExtentCoordinate(this.sourceTileset);
266
+ const fullExtent = convertBoundingVolumeToI3SFullExtent(
267
+ this.sourceTileset?.boundingVolume || this.sourceTileset?.root?.boundingVolume
268
+ );
269
+ const extent = [fullExtent.xmin, fullExtent.ymin, fullExtent.xmax, fullExtent.ymax];
242
270
  const layers0data = {
243
271
  version: `{${uuidv4().toUpperCase()}}`,
244
272
  id: 0,
@@ -308,13 +336,16 @@ export default class I3SConverter {
308
336
  const childPath = join(this.layers0Path, 'nodes', child.path!);
309
337
 
310
338
  if (this.options.slpk) {
311
- this.fileMap['nodes/1/3dNodeIndexDocument.json.gz'] = await writeFileForSlpk(
312
- childPath,
313
- JSON.stringify(child),
314
- '3dNodeIndexDocument.json'
315
- );
339
+ this.writeQueue.enqueue({
340
+ archiveKey: 'nodes/1/3dNodeIndexDocument.json.gz',
341
+ writePromise: writeFileForSlpk(
342
+ childPath,
343
+ JSON.stringify(child),
344
+ '3dNodeIndexDocument.json'
345
+ )
346
+ });
316
347
  } else {
317
- await writeFile(childPath, JSON.stringify(child));
348
+ this.writeQueue.enqueue({writePromise: writeFile(childPath, JSON.stringify(child))});
318
349
  }
319
350
  } else {
320
351
  await this._addChildrenWithNeighborsAndWriteFile({
@@ -332,13 +363,18 @@ export default class I3SConverter {
332
363
  */
333
364
  private async _writeLayers0(): Promise<void> {
334
365
  if (this.options.slpk) {
335
- this.fileMap['3dSceneLayer.json.gz'] = await writeFileForSlpk(
336
- this.layers0Path,
337
- JSON.stringify(this.layers0),
338
- '3dSceneLayer.json'
339
- );
366
+ this.writeQueue.enqueue({
367
+ archiveKey: '3dSceneLayer.json.gz',
368
+ writePromise: writeFileForSlpk(
369
+ this.layers0Path,
370
+ JSON.stringify(this.layers0),
371
+ '3dSceneLayer.json'
372
+ )
373
+ });
340
374
  } else {
341
- await writeFile(this.layers0Path, JSON.stringify(this.layers0));
375
+ this.writeQueue.enqueue({
376
+ writePromise: writeFile(this.layers0Path, JSON.stringify(this.layers0))
377
+ });
342
378
  }
343
379
  }
344
380
 
@@ -351,13 +387,12 @@ export default class I3SConverter {
351
387
  rootPath: string
352
388
  ): Promise<void> {
353
389
  if (this.options.slpk) {
354
- this.fileMap[`nodes/${nodePath}/3dNodeIndexDocument.json.gz`] = await writeFileForSlpk(
355
- rootPath,
356
- JSON.stringify(root0),
357
- '3dNodeIndexDocument.json'
358
- );
390
+ this.writeQueue.enqueue({
391
+ archiveKey: `nodes/${nodePath}/3dNodeIndexDocument.json.gz`,
392
+ writePromise: writeFileForSlpk(rootPath, JSON.stringify(root0), '3dNodeIndexDocument.json')
393
+ });
359
394
  } else {
360
- await writeFile(rootPath, JSON.stringify(root0));
395
+ this.writeQueue.enqueue({writePromise: writeFile(rootPath, JSON.stringify(root0))});
361
396
  }
362
397
  }
363
398
 
@@ -374,7 +409,6 @@ export default class I3SConverter {
374
409
  slpkFileName,
375
410
  0,
376
411
  '.',
377
- // @ts-expect-error
378
412
  this.options.sevenZipExe
379
413
  );
380
414
 
@@ -556,6 +590,8 @@ export default class I3SConverter {
556
590
  };
557
591
 
558
592
  for (const resources of resourcesData || [emptyResources]) {
593
+ this.layersHasTexture = this.layersHasTexture || Boolean(resources.texture);
594
+
559
595
  if (this.generateBoundingVolumes && resources.boundingVolumes) {
560
596
  boundingVolumes = resources.boundingVolumes;
561
597
  }
@@ -644,7 +680,8 @@ export default class I3SConverter {
644
680
  this.layers0?.attributeStorageInfo,
645
681
  this.options.draco,
646
682
  this.generateBoundingVolumes,
647
- this.geoidHeightModel!
683
+ this.geoidHeightModel!,
684
+ this.workerSource
648
685
  );
649
686
  return resourcesData;
650
687
  }
@@ -794,7 +831,7 @@ export default class I3SConverter {
794
831
  const slpkChildPath = join('nodes', nodePath);
795
832
 
796
833
  await this._writeGeometries(geometryBuffer!, compressedGeometry!, childPath, slpkChildPath);
797
- await this._writeShared(sharedResources!, childPath, slpkChildPath, nodePath);
834
+ await this._writeShared(sharedResources, childPath, slpkChildPath, nodePath);
798
835
  await this._writeTexture(texture, childPath, slpkChildPath);
799
836
  await this._writeAttributes(attributes, childPath, slpkChildPath);
800
837
  }
@@ -808,33 +845,35 @@ export default class I3SConverter {
808
845
  */
809
846
  private async _writeGeometries(
810
847
  geometryBuffer: ArrayBuffer,
811
- compressedGeometry: ArrayBuffer,
848
+ compressedGeometry: Promise<ArrayBuffer>,
812
849
  childPath: string,
813
850
  slpkChildPath: string
814
851
  ): Promise<void> {
815
852
  if (this.options.slpk) {
816
853
  const slpkGeometryPath = join(childPath, 'geometries');
817
- this.fileMap[`${slpkChildPath}/geometries/0.bin.gz`] = await writeFileForSlpk(
818
- slpkGeometryPath,
819
- geometryBuffer,
820
- '0.bin'
821
- );
854
+ this.writeQueue.enqueue({
855
+ archiveKey: `${slpkChildPath}/geometries/0.bin.gz`,
856
+ writePromise: writeFileForSlpk(slpkGeometryPath, geometryBuffer, '0.bin')
857
+ });
822
858
  } else {
823
859
  const geometryPath = join(childPath, 'geometries/0/');
824
- await writeFile(geometryPath, geometryBuffer, 'index.bin');
860
+ this.writeQueue.enqueue({
861
+ writePromise: writeFile(geometryPath, geometryBuffer, 'index.bin')
862
+ });
825
863
  }
826
864
 
827
865
  if (this.options.draco) {
828
866
  if (this.options.slpk) {
829
867
  const slpkCompressedGeometryPath = join(childPath, 'geometries');
830
- this.fileMap[`${slpkChildPath}/geometries/1.bin.gz`] = await writeFileForSlpk(
831
- slpkCompressedGeometryPath,
832
- compressedGeometry,
833
- '1.bin'
834
- );
868
+ this.writeQueue.enqueue({
869
+ archiveKey: `${slpkChildPath}/geometries/1.bin.gz`,
870
+ writePromise: writeFileForSlpk(slpkCompressedGeometryPath, compressedGeometry, '1.bin')
871
+ });
835
872
  } else {
836
873
  const compressedGeometryPath = join(childPath, 'geometries/1/');
837
- await writeFile(compressedGeometryPath, compressedGeometry, 'index.bin');
874
+ this.writeQueue.enqueue({
875
+ writePromise: writeFile(compressedGeometryPath, compressedGeometry, 'index.bin')
876
+ });
838
877
  }
839
878
  }
840
879
  }
@@ -847,24 +886,26 @@ export default class I3SConverter {
847
886
  * @param nodePath - a node path
848
887
  */
849
888
  private async _writeShared(
850
- sharedResources: SharedResources,
889
+ sharedResources: SharedResourcesArrays | null,
851
890
  childPath: string,
852
891
  slpkChildPath: string,
853
892
  nodePath: string
854
893
  ): Promise<void> {
894
+ if (!sharedResources) {
895
+ return;
896
+ }
855
897
  sharedResources.nodePath = nodePath;
856
898
  const sharedData = transform(sharedResources, sharedResourcesTemplate());
857
899
  const sharedDataStr = JSON.stringify(sharedData);
858
900
  if (this.options.slpk) {
859
901
  const slpkSharedPath = join(childPath, 'shared');
860
- this.fileMap[`${slpkChildPath}/shared/sharedResource.json.gz`] = await writeFileForSlpk(
861
- slpkSharedPath,
862
- sharedDataStr,
863
- 'sharedResource.json'
864
- );
902
+ this.writeQueue.enqueue({
903
+ archiveKey: `${slpkChildPath}/shared/sharedResource.json.gz`,
904
+ writePromise: writeFileForSlpk(slpkSharedPath, sharedDataStr, 'sharedResource.json')
905
+ });
865
906
  } else {
866
907
  const sharedPath = join(childPath, 'shared/');
867
- await writeFile(sharedPath, sharedDataStr);
908
+ this.writeQueue.enqueue({writePromise: writeFile(sharedPath, sharedDataStr)});
868
909
  }
869
910
  }
870
911
 
@@ -892,9 +933,13 @@ export default class I3SConverter {
892
933
 
893
934
  if (this.generateTextures) {
894
935
  formats.push({name: '1', format: 'ktx2'});
895
- const ktx2TextureData = new Uint8Array(
896
- await encode(texture.image, KTX2BasisUniversalTextureWriter)
897
- );
936
+ const ktx2TextureData = encode(texture.image, KTX2BasisWriterWorker, {
937
+ ...KTX2BasisWriterWorker.options,
938
+ source: this.workerSource.ktx2,
939
+ reuseWorkers: true,
940
+ _nodeWorkers: true
941
+ });
942
+
898
943
  await this.writeTextureFile(ktx2TextureData, '1', 'ktx2', childPath, slpkChildPath);
899
944
  }
900
945
 
@@ -907,9 +952,7 @@ export default class I3SConverter {
907
952
 
908
953
  if (this.generateTextures) {
909
954
  formats.push({name: '0', format: 'jpg'});
910
- const decodedFromKTX2TextureData = new Uint8Array(
911
- await encode(texture.image!.data[0], ImageWriter)
912
- );
955
+ const decodedFromKTX2TextureData = encode(texture.image!.data[0], ImageWriter);
913
956
  await this.writeTextureFile(
914
957
  decodedFromKTX2TextureData,
915
958
  '0',
@@ -936,7 +979,7 @@ export default class I3SConverter {
936
979
  * @param slpkChildPath
937
980
  */
938
981
  private async writeTextureFile(
939
- textureData: ArrayBuffer,
982
+ textureData: Promise<ArrayBuffer>,
940
983
  name: string,
941
984
  format: 'jpg' | 'png' | 'ktx2',
942
985
  childPath: string,
@@ -946,15 +989,15 @@ export default class I3SConverter {
946
989
  const slpkTexturePath = join(childPath, 'textures');
947
990
  const compress = false;
948
991
 
949
- this.fileMap[`${slpkChildPath}/textures/${name}.${format}`] = await writeFileForSlpk(
950
- slpkTexturePath,
951
- textureData,
952
- `${name}.${format}`,
953
- compress
954
- );
992
+ this.writeQueue.enqueue({
993
+ archiveKey: `${slpkChildPath}/textures/${name}.${format}`,
994
+ writePromise: writeFileForSlpk(slpkTexturePath, textureData, `${name}.${format}`, compress)
995
+ });
955
996
  } else {
956
997
  const texturePath = join(childPath, `textures/${name}/`);
957
- await writeFile(texturePath, textureData, `index.${format}`);
998
+ this.writeQueue.enqueue({
999
+ writePromise: writeFile(texturePath, textureData, `index.${format}`)
1000
+ });
958
1001
  }
959
1002
  }
960
1003
 
@@ -965,25 +1008,26 @@ export default class I3SConverter {
965
1008
  * @param slpkChildPath - the resource path inside *slpk file
966
1009
  */
967
1010
  private async _writeAttributes(
968
- attributes: ArrayBuffer[],
1011
+ attributes: ArrayBuffer[] | null = [],
969
1012
  childPath: string,
970
1013
  slpkChildPath: string
971
1014
  ): Promise<void> {
972
- if (attributes.length && this.layers0?.attributeStorageInfo?.length) {
1015
+ if (attributes?.length && this.layers0?.attributeStorageInfo?.length) {
973
1016
  for (let index = 0; index < attributes.length; index++) {
974
1017
  const folderName = this.layers0.attributeStorageInfo[index].key;
975
1018
  const fileBuffer = new Uint8Array(attributes[index]);
976
1019
 
977
1020
  if (this.options.slpk) {
978
1021
  const slpkAttributesPath = join(childPath, 'attributes', folderName);
979
- this.fileMap[`${slpkChildPath}/attributes/${folderName}.bin.gz`] = await writeFileForSlpk(
980
- slpkAttributesPath,
981
- fileBuffer,
982
- '0.bin'
983
- );
1022
+ this.writeQueue.enqueue({
1023
+ archiveKey: `${slpkChildPath}/attributes/${folderName}.bin.gz`,
1024
+ writePromise: writeFileForSlpk(slpkAttributesPath, fileBuffer, '0.bin')
1025
+ });
984
1026
  } else {
985
1027
  const attributesPath = join(childPath, `attributes/${folderName}/0`);
986
- await writeFile(attributesPath, fileBuffer, 'index.bin');
1028
+ this.writeQueue.enqueue({
1029
+ writePromise: writeFile(attributesPath, fileBuffer, 'index.bin')
1030
+ });
987
1031
  }
988
1032
  }
989
1033
  }
@@ -1297,4 +1341,27 @@ export default class I3SConverter {
1297
1341
  private isContentSupported(sourceRootTile: Tile3D): boolean {
1298
1342
  return ['b3dm', 'glTF'].includes(sourceRootTile?.content?.type);
1299
1343
  }
1344
+
1345
+ private async loadWorkers(): Promise<void> {
1346
+ console.log(`Loading workers source...`); // eslint-disable-line no-undef, no-console
1347
+ if (this.options.draco) {
1348
+ const url = getWorkerURL(DracoWriterWorker, {...getLoaderOptions()});
1349
+ const sourceResponse = await fetchFile(url);
1350
+ const source = await sourceResponse.text();
1351
+ this.workerSource.draco = source;
1352
+ }
1353
+
1354
+ if (this.generateTextures) {
1355
+ const url = getWorkerURL(KTX2BasisWriterWorker, {...getLoaderOptions()});
1356
+ const sourceResponse = await fetchFile(url);
1357
+ const source = await sourceResponse.text();
1358
+ this.workerSource.ktx2 = source;
1359
+ }
1360
+
1361
+ const i3sAttributesWorkerUrl = getWorkerURL(I3SAttributesWorker, {...getLoaderOptions()});
1362
+ const sourceResponse = await fetchFile(i3sAttributesWorkerUrl);
1363
+ const source = await sourceResponse.text();
1364
+ this.workerSource.I3SAttributes = source;
1365
+ console.log(`Loading workers source completed!`); // eslint-disable-line no-undef, no-console
1366
+ }
1300
1367
  }