@holoscript/engine 6.0.3 → 6.0.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/AutoMesher-CK47F6AV.js +17 -0
- package/dist/GPUBuffers-2LHBCD7X.js +9 -0
- package/dist/WebGPUContext-TNEUYU2Y.js +11 -0
- package/dist/animation/index.cjs +38 -38
- package/dist/animation/index.d.cts +1 -1
- package/dist/animation/index.d.ts +1 -1
- package/dist/animation/index.js +1 -1
- package/dist/audio/index.cjs +16 -6
- package/dist/audio/index.d.cts +1 -1
- package/dist/audio/index.d.ts +1 -1
- package/dist/audio/index.js +1 -1
- package/dist/camera/index.cjs +23 -23
- package/dist/camera/index.d.cts +1 -1
- package/dist/camera/index.d.ts +1 -1
- package/dist/camera/index.js +1 -1
- package/dist/character/index.cjs +6 -4
- package/dist/character/index.js +1 -1
- package/dist/choreography/index.cjs +1194 -0
- package/dist/choreography/index.d.cts +687 -0
- package/dist/choreography/index.d.ts +687 -0
- package/dist/choreography/index.js +1156 -0
- package/dist/chunk-2CSNRI2N.js +217 -0
- package/dist/chunk-33T2WINR.js +266 -0
- package/dist/chunk-35R73OFM.js +1257 -0
- package/dist/chunk-4MMDSUNP.js +1256 -0
- package/dist/chunk-5V6HOU72.js +319 -0
- package/dist/chunk-6QOP6PYF.js +1038 -0
- package/dist/chunk-7KMJVHIL.js +8944 -0
- package/dist/chunk-7VPUC62U.js +1106 -0
- package/dist/chunk-A2Y6RCAT.js +1878 -0
- package/dist/chunk-AHM42MK6.js +8944 -0
- package/dist/chunk-BL7IDTHE.js +218 -0
- package/dist/chunk-CITOMSWL.js +10462 -0
- package/dist/chunk-CXDPKW2K.js +8944 -0
- package/dist/chunk-CXZPLD4S.js +223 -0
- package/dist/chunk-CZYJE7IH.js +5169 -0
- package/dist/chunk-D2OP7YC7.js +6325 -0
- package/dist/chunk-EDRVQHUU.js +1544 -0
- package/dist/chunk-EJSLOOW2.js +3589 -0
- package/dist/chunk-F53SFGW5.js +1878 -0
- package/dist/chunk-HCFPELPY.js +919 -0
- package/dist/chunk-HNEE36PY.js +93 -0
- package/dist/chunk-HYXNV36F.js +1256 -0
- package/dist/chunk-IB7KHVFY.js +821 -0
- package/dist/chunk-IBBO7YYG.js +690 -0
- package/dist/chunk-ILIBGINU.js +5470 -0
- package/dist/chunk-IS4MHLKN.js +5479 -0
- package/dist/chunk-JT2PFKWD.js +5479 -0
- package/dist/chunk-K4CUB4NY.js +1038 -0
- package/dist/chunk-KATDQXRJ.js +10462 -0
- package/dist/chunk-KBQE6ZFJ.js +8944 -0
- package/dist/chunk-KBVD5K7E.js +560 -0
- package/dist/chunk-KCDPVQRY.js +4088 -0
- package/dist/chunk-KN4QJPKN.js +8944 -0
- package/dist/chunk-KWJ3ROSI.js +8944 -0
- package/dist/chunk-L45VF6DD.js +919 -0
- package/dist/chunk-LY4T37YK.js +307 -0
- package/dist/chunk-MDN5WZXA.js +1544 -0
- package/dist/chunk-MGCDP6VU.js +928 -0
- package/dist/chunk-NCX7X6G2.js +8681 -0
- package/dist/chunk-OF54BPVD.js +913 -0
- package/dist/chunk-OWSN2Q3Q.js +690 -0
- package/dist/chunk-PRRB5TTA.js +406 -0
- package/dist/chunk-PXWVQF76.js +4086 -0
- package/dist/chunk-PYCOIDT2.js +812 -0
- package/dist/chunk-PZCSADOV.js +928 -0
- package/dist/chunk-Q2XBVS2K.js +1038 -0
- package/dist/chunk-QDZRXWN5.js +1776 -0
- package/dist/chunk-RNWOZ6WQ.js +913 -0
- package/dist/chunk-ROLFT4CJ.js +1693 -0
- package/dist/chunk-SLTJRZ2N.js +266 -0
- package/dist/chunk-SRUS5XSU.js +4088 -0
- package/dist/chunk-TKCA3WZ5.js +5409 -0
- package/dist/chunk-TNRMXYI2.js +1650 -0
- package/dist/chunk-TQB3GJGM.js +9763 -0
- package/dist/chunk-TUFGXG6K.js +510 -0
- package/dist/chunk-U6KMTGQJ.js +632 -0
- package/dist/chunk-VMGJQST6.js +8681 -0
- package/dist/chunk-X4F4TCG4.js +5470 -0
- package/dist/chunk-ZIFROE75.js +1544 -0
- package/dist/chunk-ZIJQYHSQ.js +1204 -0
- package/dist/combat/index.cjs +4 -4
- package/dist/combat/index.d.cts +1 -1
- package/dist/combat/index.d.ts +1 -1
- package/dist/combat/index.js +1 -1
- package/dist/ecs/index.cjs +1 -1
- package/dist/ecs/index.js +1 -1
- package/dist/environment/index.cjs +14 -14
- package/dist/environment/index.d.cts +1 -1
- package/dist/environment/index.d.ts +1 -1
- package/dist/environment/index.js +1 -1
- package/dist/gpu/index.cjs +4810 -0
- package/dist/gpu/index.js +3714 -0
- package/dist/hologram/index.cjs +27 -1
- package/dist/hologram/index.js +1 -1
- package/dist/index-B2PIsAmR.d.cts +2180 -0
- package/dist/index-B2PIsAmR.d.ts +2180 -0
- package/dist/index-BHySEPX7.d.cts +2921 -0
- package/dist/index-BJV21zuy.d.cts +341 -0
- package/dist/index-BJV21zuy.d.ts +341 -0
- package/dist/index-BQutTphC.d.cts +790 -0
- package/dist/index-ByIq2XrS.d.cts +3910 -0
- package/dist/index-BysHjDSO.d.cts +224 -0
- package/dist/index-BysHjDSO.d.ts +224 -0
- package/dist/index-CKwAJGck.d.ts +455 -0
- package/dist/index-CUl3QstQ.d.cts +3006 -0
- package/dist/index-CUl3QstQ.d.ts +3006 -0
- package/dist/index-CmYtNiI-.d.cts +953 -0
- package/dist/index-CmYtNiI-.d.ts +953 -0
- package/dist/index-CnRzWxi_.d.cts +522 -0
- package/dist/index-CnRzWxi_.d.ts +522 -0
- package/dist/index-CwRWbSC7.d.ts +2921 -0
- package/dist/index-CxKIBstO.d.ts +790 -0
- package/dist/index-DJ6-R8vh.d.cts +455 -0
- package/dist/index-DQKisbcI.d.cts +4968 -0
- package/dist/index-DQKisbcI.d.ts +4968 -0
- package/dist/index-DRT2zJez.d.ts +3910 -0
- package/dist/index-DfNLiAka.d.cts +192 -0
- package/dist/index-DfNLiAka.d.ts +192 -0
- package/dist/index-nMvkoRm8.d.cts +405 -0
- package/dist/index-nMvkoRm8.d.ts +405 -0
- package/dist/index-s9yOFU37.d.cts +604 -0
- package/dist/index-s9yOFU37.d.ts +604 -0
- package/dist/index.cjs +22966 -6960
- package/dist/index.d.cts +864 -20
- package/dist/index.d.ts +864 -20
- package/dist/index.js +3062 -48
- package/dist/input/index.cjs +1 -1
- package/dist/input/index.js +1 -1
- package/dist/orbital/index.cjs +3 -3
- package/dist/orbital/index.d.cts +1 -1
- package/dist/orbital/index.d.ts +1 -1
- package/dist/orbital/index.js +1 -1
- package/dist/particles/index.cjs +16 -16
- package/dist/particles/index.d.cts +1 -1
- package/dist/particles/index.d.ts +1 -1
- package/dist/particles/index.js +1 -1
- package/dist/physics/index.cjs +2377 -21
- package/dist/physics/index.d.cts +1 -1
- package/dist/physics/index.d.ts +1 -1
- package/dist/physics/index.js +35 -1
- package/dist/postfx/index.cjs +3491 -0
- package/dist/postfx/index.js +93 -0
- package/dist/procedural/index.cjs +1 -1
- package/dist/procedural/index.js +1 -1
- package/dist/puppeteer-5VF6KDVO.js +52197 -0
- package/dist/puppeteer-IZVZ3SG4.js +52197 -0
- package/dist/rendering/index.cjs +33 -32
- package/dist/rendering/index.d.cts +1 -1
- package/dist/rendering/index.d.ts +1 -1
- package/dist/rendering/index.js +8 -6
- package/dist/runtime/index.cjs +23 -13
- package/dist/runtime/index.d.cts +1 -1
- package/dist/runtime/index.d.ts +1 -1
- package/dist/runtime/index.js +8 -6
- package/dist/runtime/protocols/index.cjs +349 -0
- package/dist/runtime/protocols/index.js +15 -0
- package/dist/scene/index.cjs +8 -8
- package/dist/scene/index.d.cts +1 -1
- package/dist/scene/index.d.ts +1 -1
- package/dist/scene/index.js +1 -1
- package/dist/shader/index.cjs +3087 -0
- package/dist/shader/index.js +3044 -0
- package/dist/simulation/index.cjs +10680 -0
- package/dist/simulation/index.d.cts +3 -0
- package/dist/simulation/index.d.ts +3 -0
- package/dist/simulation/index.js +307 -0
- package/dist/spatial/index.cjs +2443 -0
- package/dist/spatial/index.d.cts +1545 -0
- package/dist/spatial/index.d.ts +1545 -0
- package/dist/spatial/index.js +2400 -0
- package/dist/terrain/index.cjs +1 -1
- package/dist/terrain/index.d.cts +1 -1
- package/dist/terrain/index.d.ts +1 -1
- package/dist/terrain/index.js +1 -1
- package/dist/transformers.node-4NKAPD5U.js +45620 -0
- package/dist/vm/index.cjs +7 -8
- package/dist/vm/index.d.cts +1 -1
- package/dist/vm/index.d.ts +1 -1
- package/dist/vm/index.js +1 -1
- package/dist/vm-bridge/index.cjs +2 -2
- package/dist/vm-bridge/index.d.cts +2 -2
- package/dist/vm-bridge/index.d.ts +2 -2
- package/dist/vm-bridge/index.js +1 -1
- package/dist/vr/index.cjs +6 -6
- package/dist/vr/index.js +1 -1
- package/dist/world/index.cjs +3 -3
- package/dist/world/index.d.cts +1 -1
- package/dist/world/index.d.ts +1 -1
- package/dist/world/index.js +1 -1
- package/package.json +53 -21
- package/LICENSE +0 -21
|
@@ -0,0 +1,1106 @@
|
|
|
1
|
+
import {
|
|
2
|
+
__export
|
|
3
|
+
} from "./chunk-AKLW2MUS.js";
|
|
4
|
+
|
|
5
|
+
// src/hologram/index.ts
|
|
6
|
+
var hologram_exports = {};
|
|
7
|
+
__export(hologram_exports, {
|
|
8
|
+
DepthEstimationService: () => DepthEstimationService,
|
|
9
|
+
GIFDecomposer: () => GIFDecomposer,
|
|
10
|
+
GIFDisposalMethod: () => GIFDisposalMethod,
|
|
11
|
+
MVHEVCCompiler: () => MVHEVCCompiler,
|
|
12
|
+
ModelCache: () => ModelCache,
|
|
13
|
+
QuiltCompiler: () => QuiltCompiler,
|
|
14
|
+
TemporalSmoother: () => TemporalSmoother,
|
|
15
|
+
WebCodecsDepthPipeline: () => WebCodecsDepthPipeline,
|
|
16
|
+
depthToNormalMap: () => depthToNormalMap,
|
|
17
|
+
detectBestBackend: () => detectBestBackend
|
|
18
|
+
});
|
|
19
|
+
|
|
20
|
+
// src/hologram/DepthEstimationService.ts
|
|
21
|
+
var DEFAULT_MODEL_ID = "depth-anything/Depth-Anything-V2-Small-hf";
|
|
22
|
+
var DEFAULT_MAX_RESOLUTION = 512;
|
|
23
|
+
var DEFAULT_TEMPORAL_ALPHA = 0.8;
|
|
24
|
+
var CACHE_DB_NAME = "holoscript-ml-models";
|
|
25
|
+
var CACHE_STORE_NAME = "depth-models";
|
|
26
|
+
async function detectBestBackend() {
|
|
27
|
+
if (typeof navigator !== "undefined" && "gpu" in navigator) {
|
|
28
|
+
try {
|
|
29
|
+
const adapter = await navigator.gpu.requestAdapter();
|
|
30
|
+
if (adapter) return "webgpu";
|
|
31
|
+
} catch {
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
if (typeof WebAssembly !== "undefined") {
|
|
35
|
+
return "wasm";
|
|
36
|
+
}
|
|
37
|
+
return "cpu";
|
|
38
|
+
}
|
|
39
|
+
function depthToNormalMap(depthMap, width, height) {
|
|
40
|
+
const normalMap = new Float32Array(width * height * 3);
|
|
41
|
+
for (let y = 1; y < height - 1; y++) {
|
|
42
|
+
for (let x = 1; x < width - 1; x++) {
|
|
43
|
+
const tl = depthMap[(y - 1) * width + (x - 1)];
|
|
44
|
+
const t = depthMap[(y - 1) * width + x];
|
|
45
|
+
const tr = depthMap[(y - 1) * width + (x + 1)];
|
|
46
|
+
const l = depthMap[y * width + (x - 1)];
|
|
47
|
+
const r = depthMap[y * width + (x + 1)];
|
|
48
|
+
const bl = depthMap[(y + 1) * width + (x - 1)];
|
|
49
|
+
const b = depthMap[(y + 1) * width + x];
|
|
50
|
+
const br = depthMap[(y + 1) * width + (x + 1)];
|
|
51
|
+
const dx = tr + 2 * r + br - (tl + 2 * l + bl);
|
|
52
|
+
const dy = bl + 2 * b + br - (tl + 2 * t + tr);
|
|
53
|
+
const dz = 1;
|
|
54
|
+
const len = Math.sqrt(dx * dx + dy * dy + dz * dz);
|
|
55
|
+
const idx = (y * width + x) * 3;
|
|
56
|
+
normalMap[idx] = dx / len * 0.5 + 0.5;
|
|
57
|
+
normalMap[idx + 1] = dy / len * 0.5 + 0.5;
|
|
58
|
+
normalMap[idx + 2] = dz / len * 0.5 + 0.5;
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
for (let x = 0; x < width; x++) {
|
|
62
|
+
const topIdx = x * 3;
|
|
63
|
+
const botIdx = ((height - 1) * width + x) * 3;
|
|
64
|
+
normalMap[topIdx] = 0.5;
|
|
65
|
+
normalMap[topIdx + 1] = 0.5;
|
|
66
|
+
normalMap[topIdx + 2] = 1;
|
|
67
|
+
normalMap[botIdx] = 0.5;
|
|
68
|
+
normalMap[botIdx + 1] = 0.5;
|
|
69
|
+
normalMap[botIdx + 2] = 1;
|
|
70
|
+
}
|
|
71
|
+
for (let y = 0; y < height; y++) {
|
|
72
|
+
const leftIdx = y * width * 3;
|
|
73
|
+
const rightIdx = (y * width + width - 1) * 3;
|
|
74
|
+
normalMap[leftIdx] = 0.5;
|
|
75
|
+
normalMap[leftIdx + 1] = 0.5;
|
|
76
|
+
normalMap[leftIdx + 2] = 1;
|
|
77
|
+
normalMap[rightIdx] = 0.5;
|
|
78
|
+
normalMap[rightIdx + 1] = 0.5;
|
|
79
|
+
normalMap[rightIdx + 2] = 1;
|
|
80
|
+
}
|
|
81
|
+
return normalMap;
|
|
82
|
+
}
|
|
83
|
+
var ModelCache = class {
|
|
84
|
+
db = null;
|
|
85
|
+
async open() {
|
|
86
|
+
if (this.db) return;
|
|
87
|
+
if (typeof indexedDB === "undefined") return;
|
|
88
|
+
return new Promise((resolve, reject) => {
|
|
89
|
+
const request = indexedDB.open(CACHE_DB_NAME, 1);
|
|
90
|
+
request.onupgradeneeded = () => {
|
|
91
|
+
const db = request.result;
|
|
92
|
+
if (!db.objectStoreNames.contains(CACHE_STORE_NAME)) {
|
|
93
|
+
db.createObjectStore(CACHE_STORE_NAME);
|
|
94
|
+
}
|
|
95
|
+
};
|
|
96
|
+
request.onsuccess = () => {
|
|
97
|
+
this.db = request.result;
|
|
98
|
+
resolve();
|
|
99
|
+
};
|
|
100
|
+
request.onerror = () => reject(request.error);
|
|
101
|
+
});
|
|
102
|
+
}
|
|
103
|
+
async get(key) {
|
|
104
|
+
if (!this.db) return null;
|
|
105
|
+
return new Promise((resolve, reject) => {
|
|
106
|
+
const tx = this.db.transaction(CACHE_STORE_NAME, "readonly");
|
|
107
|
+
const store = tx.objectStore(CACHE_STORE_NAME);
|
|
108
|
+
const request = store.get(key);
|
|
109
|
+
request.onsuccess = () => resolve(request.result ?? null);
|
|
110
|
+
request.onerror = () => reject(request.error);
|
|
111
|
+
});
|
|
112
|
+
}
|
|
113
|
+
async set(key, value) {
|
|
114
|
+
if (!this.db) return;
|
|
115
|
+
return new Promise((resolve, reject) => {
|
|
116
|
+
const tx = this.db.transaction(CACHE_STORE_NAME, "readwrite");
|
|
117
|
+
const store = tx.objectStore(CACHE_STORE_NAME);
|
|
118
|
+
const request = store.put(value, key);
|
|
119
|
+
request.onsuccess = () => resolve();
|
|
120
|
+
request.onerror = () => reject(request.error);
|
|
121
|
+
});
|
|
122
|
+
}
|
|
123
|
+
async has(key) {
|
|
124
|
+
const value = await this.get(key);
|
|
125
|
+
return value !== null;
|
|
126
|
+
}
|
|
127
|
+
close() {
|
|
128
|
+
this.db?.close();
|
|
129
|
+
this.db = null;
|
|
130
|
+
}
|
|
131
|
+
};
|
|
132
|
+
var TemporalSmoother = class {
|
|
133
|
+
previousDepth = null;
|
|
134
|
+
alpha;
|
|
135
|
+
constructor(alpha = DEFAULT_TEMPORAL_ALPHA) {
|
|
136
|
+
this.alpha = Math.max(0, Math.min(1, alpha));
|
|
137
|
+
}
|
|
138
|
+
/**
|
|
139
|
+
* Smooth the current depth map against the previous frame.
|
|
140
|
+
* Returns a new smoothed array (does not mutate input).
|
|
141
|
+
*/
|
|
142
|
+
smooth(currentDepth) {
|
|
143
|
+
if (!this.previousDepth || this.previousDepth.length !== currentDepth.length) {
|
|
144
|
+
this.previousDepth = new Float32Array(currentDepth);
|
|
145
|
+
return new Float32Array(currentDepth);
|
|
146
|
+
}
|
|
147
|
+
const smoothed = new Float32Array(currentDepth.length);
|
|
148
|
+
for (let i = 0; i < currentDepth.length; i++) {
|
|
149
|
+
smoothed[i] = this.alpha * currentDepth[i] + (1 - this.alpha) * this.previousDepth[i];
|
|
150
|
+
}
|
|
151
|
+
this.previousDepth = new Float32Array(smoothed);
|
|
152
|
+
return smoothed;
|
|
153
|
+
}
|
|
154
|
+
reset() {
|
|
155
|
+
this.previousDepth = null;
|
|
156
|
+
}
|
|
157
|
+
};
|
|
158
|
+
var GIFDisposalMethod = /* @__PURE__ */ ((GIFDisposalMethod2) => {
|
|
159
|
+
GIFDisposalMethod2[GIFDisposalMethod2["Unspecified"] = 0] = "Unspecified";
|
|
160
|
+
GIFDisposalMethod2[GIFDisposalMethod2["DoNotDispose"] = 1] = "DoNotDispose";
|
|
161
|
+
GIFDisposalMethod2[GIFDisposalMethod2["RestoreBackground"] = 2] = "RestoreBackground";
|
|
162
|
+
GIFDisposalMethod2[GIFDisposalMethod2["RestorePrevious"] = 3] = "RestorePrevious";
|
|
163
|
+
return GIFDisposalMethod2;
|
|
164
|
+
})(GIFDisposalMethod || {});
|
|
165
|
+
var GIFDecomposer = class {
|
|
166
|
+
config;
|
|
167
|
+
constructor(config = {}) {
|
|
168
|
+
this.config = {
|
|
169
|
+
maxFrames: config.maxFrames ?? 500,
|
|
170
|
+
targetWidth: config.targetWidth ?? 0
|
|
171
|
+
};
|
|
172
|
+
}
|
|
173
|
+
/**
|
|
174
|
+
* Decompose raw GIF frame data into full composited RGBA frames.
|
|
175
|
+
* Each frame in the input array should have: data (Uint8ClampedArray),
|
|
176
|
+
* width, height, left, top, disposalMethod, delayMs.
|
|
177
|
+
*/
|
|
178
|
+
decompose(rawFrames, gifWidth, gifHeight) {
|
|
179
|
+
const maxFrames = Math.min(rawFrames.length, this.config.maxFrames);
|
|
180
|
+
const frames = [];
|
|
181
|
+
let canvas = new Uint8ClampedArray(gifWidth * gifHeight * 4);
|
|
182
|
+
let previousCanvas = new Uint8ClampedArray(gifWidth * gifHeight * 4);
|
|
183
|
+
for (let i = 0; i < maxFrames; i++) {
|
|
184
|
+
const raw = rawFrames[i];
|
|
185
|
+
const savedCanvas = new Uint8ClampedArray(canvas);
|
|
186
|
+
for (let y = 0; y < raw.height; y++) {
|
|
187
|
+
for (let x = 0; x < raw.width; x++) {
|
|
188
|
+
const srcIdx = (y * raw.width + x) * 4;
|
|
189
|
+
const dstX = raw.left + x;
|
|
190
|
+
const dstY = raw.top + y;
|
|
191
|
+
if (dstX >= gifWidth || dstY >= gifHeight) continue;
|
|
192
|
+
const dstIdx = (dstY * gifWidth + dstX) * 4;
|
|
193
|
+
const alpha = raw.data[srcIdx + 3];
|
|
194
|
+
if (alpha > 0) {
|
|
195
|
+
canvas[dstIdx] = raw.data[srcIdx];
|
|
196
|
+
canvas[dstIdx + 1] = raw.data[srcIdx + 1];
|
|
197
|
+
canvas[dstIdx + 2] = raw.data[srcIdx + 2];
|
|
198
|
+
canvas[dstIdx + 3] = alpha;
|
|
199
|
+
}
|
|
200
|
+
}
|
|
201
|
+
}
|
|
202
|
+
frames.push({
|
|
203
|
+
data: new Uint8ClampedArray(canvas),
|
|
204
|
+
width: gifWidth,
|
|
205
|
+
height: gifHeight,
|
|
206
|
+
delayMs: raw.delayMs || 100,
|
|
207
|
+
index: i
|
|
208
|
+
});
|
|
209
|
+
switch (raw.disposalMethod) {
|
|
210
|
+
case 2 /* RestoreBackground */:
|
|
211
|
+
for (let y = 0; y < raw.height; y++) {
|
|
212
|
+
for (let x = 0; x < raw.width; x++) {
|
|
213
|
+
const dstX = raw.left + x;
|
|
214
|
+
const dstY = raw.top + y;
|
|
215
|
+
if (dstX >= gifWidth || dstY >= gifHeight) continue;
|
|
216
|
+
const dstIdx = (dstY * gifWidth + dstX) * 4;
|
|
217
|
+
canvas[dstIdx] = 0;
|
|
218
|
+
canvas[dstIdx + 1] = 0;
|
|
219
|
+
canvas[dstIdx + 2] = 0;
|
|
220
|
+
canvas[dstIdx + 3] = 0;
|
|
221
|
+
}
|
|
222
|
+
}
|
|
223
|
+
break;
|
|
224
|
+
case 3 /* RestorePrevious */:
|
|
225
|
+
canvas = new Uint8ClampedArray(previousCanvas);
|
|
226
|
+
break;
|
|
227
|
+
case 1 /* DoNotDispose */:
|
|
228
|
+
case 0 /* Unspecified */:
|
|
229
|
+
default:
|
|
230
|
+
break;
|
|
231
|
+
}
|
|
232
|
+
previousCanvas = savedCanvas;
|
|
233
|
+
}
|
|
234
|
+
return frames;
|
|
235
|
+
}
|
|
236
|
+
};
|
|
237
|
+
var DepthEstimationService = class _DepthEstimationService {
|
|
238
|
+
static instance = null;
|
|
239
|
+
pipeline = null;
|
|
240
|
+
config;
|
|
241
|
+
modelCache = new ModelCache();
|
|
242
|
+
_initialized = false;
|
|
243
|
+
_initializing = null;
|
|
244
|
+
_backend = "cpu";
|
|
245
|
+
constructor(config = {}) {
|
|
246
|
+
this.config = {
|
|
247
|
+
modelId: config.modelId ?? DEFAULT_MODEL_ID,
|
|
248
|
+
backend: config.backend ?? "webgpu",
|
|
249
|
+
maxResolution: config.maxResolution ?? DEFAULT_MAX_RESOLUTION,
|
|
250
|
+
enableCache: config.enableCache ?? true,
|
|
251
|
+
onProgress: config.onProgress ?? (() => {
|
|
252
|
+
})
|
|
253
|
+
};
|
|
254
|
+
}
|
|
255
|
+
static getInstance(config) {
|
|
256
|
+
if (!_DepthEstimationService.instance) {
|
|
257
|
+
_DepthEstimationService.instance = new _DepthEstimationService(config);
|
|
258
|
+
}
|
|
259
|
+
return _DepthEstimationService.instance;
|
|
260
|
+
}
|
|
261
|
+
static resetInstance() {
|
|
262
|
+
if (_DepthEstimationService.instance) {
|
|
263
|
+
_DepthEstimationService.instance.dispose();
|
|
264
|
+
_DepthEstimationService.instance = null;
|
|
265
|
+
}
|
|
266
|
+
}
|
|
267
|
+
get initialized() {
|
|
268
|
+
return this._initialized;
|
|
269
|
+
}
|
|
270
|
+
get backend() {
|
|
271
|
+
return this._backend;
|
|
272
|
+
}
|
|
273
|
+
/**
|
|
274
|
+
* Initialize the depth estimation pipeline.
|
|
275
|
+
* Downloads model on first call, loads from IndexedDB cache on subsequent calls.
|
|
276
|
+
* Safe to call multiple times — returns cached promise if already initializing.
|
|
277
|
+
*/
|
|
278
|
+
async initialize(config) {
|
|
279
|
+
if (this._initialized) return;
|
|
280
|
+
if (this._initializing) return this._initializing;
|
|
281
|
+
if (config) {
|
|
282
|
+
Object.assign(this.config, config);
|
|
283
|
+
}
|
|
284
|
+
this._initializing = this._doInitialize();
|
|
285
|
+
await this._initializing;
|
|
286
|
+
this._initializing = null;
|
|
287
|
+
}
|
|
288
|
+
async _doInitialize() {
|
|
289
|
+
this._backend = this.config.backend ?? await detectBestBackend();
|
|
290
|
+
if (this.config.enableCache) {
|
|
291
|
+
try {
|
|
292
|
+
await this.modelCache.open();
|
|
293
|
+
} catch {
|
|
294
|
+
}
|
|
295
|
+
}
|
|
296
|
+
try {
|
|
297
|
+
let transformers;
|
|
298
|
+
try {
|
|
299
|
+
transformers = await import("./transformers.node-4NKAPD5U.js");
|
|
300
|
+
} catch {
|
|
301
|
+
this._initialized = true;
|
|
302
|
+
return;
|
|
303
|
+
}
|
|
304
|
+
const { pipeline: createPipeline } = transformers;
|
|
305
|
+
this.pipeline = await createPipeline("depth-estimation", this.config.modelId, {
|
|
306
|
+
device: this._backend === "cpu" ? void 0 : this._backend,
|
|
307
|
+
progress_callback: (progress) => {
|
|
308
|
+
if (progress?.progress !== void 0) {
|
|
309
|
+
this.config.onProgress(progress.progress / 100);
|
|
310
|
+
}
|
|
311
|
+
}
|
|
312
|
+
});
|
|
313
|
+
} catch {
|
|
314
|
+
}
|
|
315
|
+
this._initialized = true;
|
|
316
|
+
}
|
|
317
|
+
/**
|
|
318
|
+
* Estimate depth from a single image.
|
|
319
|
+
* Returns both depth map and derived normal map.
|
|
320
|
+
*
|
|
321
|
+
* @param imageData - ImageData, HTMLImageElement, HTMLCanvasElement, or image URL
|
|
322
|
+
*/
|
|
323
|
+
async estimateDepth(imageData) {
|
|
324
|
+
if (!this._initialized) {
|
|
325
|
+
throw new Error("DepthEstimationService not initialized. Call initialize() first.");
|
|
326
|
+
}
|
|
327
|
+
const startTime = performance.now();
|
|
328
|
+
const { width, height, data } = imageData;
|
|
329
|
+
const scale = Math.min(1, this.config.maxResolution / Math.max(width, height));
|
|
330
|
+
const outW = Math.round(width * scale);
|
|
331
|
+
const outH = Math.round(height * scale);
|
|
332
|
+
let depthMap;
|
|
333
|
+
if (this.pipeline) {
|
|
334
|
+
depthMap = await this._runPipelineInference(data, width, height, outW, outH);
|
|
335
|
+
} else {
|
|
336
|
+
depthMap = this._generatePlaceholderDepth(data, width, height, outW, outH);
|
|
337
|
+
}
|
|
338
|
+
const normalMap = depthToNormalMap(depthMap, outW, outH);
|
|
339
|
+
const inferenceMs = performance.now() - startTime;
|
|
340
|
+
return {
|
|
341
|
+
depthMap,
|
|
342
|
+
normalMap,
|
|
343
|
+
width: outW,
|
|
344
|
+
height: outH,
|
|
345
|
+
backend: this._backend,
|
|
346
|
+
inferenceMs
|
|
347
|
+
};
|
|
348
|
+
}
|
|
349
|
+
/**
|
|
350
|
+
* Estimate depth for a sequence of frames with temporal smoothing.
|
|
351
|
+
* Maintains coherence across frames via EMA filter.
|
|
352
|
+
*
|
|
353
|
+
* @see P.150.01: Temporal Depth Smoothing pattern
|
|
354
|
+
*/
|
|
355
|
+
async estimateDepthSequence(frames, config) {
|
|
356
|
+
const alpha = config?.temporalAlpha ?? DEFAULT_TEMPORAL_ALPHA;
|
|
357
|
+
const smoother = new TemporalSmoother(alpha);
|
|
358
|
+
const results = [];
|
|
359
|
+
for (const frame of frames) {
|
|
360
|
+
const raw = await this.estimateDepth(frame);
|
|
361
|
+
const smoothedDepth = smoother.smooth(raw.depthMap);
|
|
362
|
+
const smoothedNormals = depthToNormalMap(smoothedDepth, raw.width, raw.height);
|
|
363
|
+
results.push({
|
|
364
|
+
...raw,
|
|
365
|
+
depthMap: smoothedDepth,
|
|
366
|
+
normalMap: smoothedNormals
|
|
367
|
+
});
|
|
368
|
+
}
|
|
369
|
+
return results;
|
|
370
|
+
}
|
|
371
|
+
/**
|
|
372
|
+
* Run real Depth Anything V2 inference via Transformers.js pipeline.
|
|
373
|
+
* Converts RGBA pixel data to a RawImage, runs the pipeline, and
|
|
374
|
+
* extracts the depth tensor as a normalized Float32Array.
|
|
375
|
+
*/
|
|
376
|
+
async _runPipelineInference(data, srcW, srcH, outW, outH) {
|
|
377
|
+
const input = { data, width: srcW, height: srcH };
|
|
378
|
+
const output = await this.pipeline(input);
|
|
379
|
+
const rawDepth = output.predicted_depth?.data ?? output.depth?.data ?? new Float32Array(0);
|
|
380
|
+
if (rawDepth.length === 0) {
|
|
381
|
+
return this._generatePlaceholderDepth(data, srcW, srcH, outW, outH);
|
|
382
|
+
}
|
|
383
|
+
let minVal = Infinity, maxVal = -Infinity;
|
|
384
|
+
for (let i = 0; i < rawDepth.length; i++) {
|
|
385
|
+
if (rawDepth[i] < minVal) minVal = rawDepth[i];
|
|
386
|
+
if (rawDepth[i] > maxVal) maxVal = rawDepth[i];
|
|
387
|
+
}
|
|
388
|
+
const range = maxVal - minVal || 1;
|
|
389
|
+
const pipelineW = output.predicted_depth?.dims?.[1] ?? output.depth?.width ?? srcW;
|
|
390
|
+
const pipelineH = output.predicted_depth?.dims?.[0] ?? output.depth?.height ?? srcH;
|
|
391
|
+
const depthMap = new Float32Array(outW * outH);
|
|
392
|
+
const scaleX = pipelineW / outW;
|
|
393
|
+
const scaleY = pipelineH / outH;
|
|
394
|
+
for (let y = 0; y < outH; y++) {
|
|
395
|
+
for (let x = 0; x < outW; x++) {
|
|
396
|
+
const pX = Math.min(Math.floor(x * scaleX), pipelineW - 1);
|
|
397
|
+
const pY = Math.min(Math.floor(y * scaleY), pipelineH - 1);
|
|
398
|
+
const val = rawDepth[pY * pipelineW + pX];
|
|
399
|
+
depthMap[y * outW + x] = (val - minVal) / range;
|
|
400
|
+
}
|
|
401
|
+
}
|
|
402
|
+
return depthMap;
|
|
403
|
+
}
|
|
404
|
+
/**
|
|
405
|
+
* Fallback depth generation using luminance as a proxy.
|
|
406
|
+
* Used when Transformers.js is not installed or pipeline fails.
|
|
407
|
+
*/
|
|
408
|
+
_generatePlaceholderDepth(data, srcW, srcH, outW, outH) {
|
|
409
|
+
const depthMap = new Float32Array(outW * outH);
|
|
410
|
+
const scaleX = srcW / outW;
|
|
411
|
+
const scaleY = srcH / outH;
|
|
412
|
+
for (let y = 0; y < outH; y++) {
|
|
413
|
+
for (let x = 0; x < outW; x++) {
|
|
414
|
+
const srcX = Math.floor(x * scaleX);
|
|
415
|
+
const srcY = Math.floor(y * scaleY);
|
|
416
|
+
const srcIdx = (srcY * srcW + srcX) * 4;
|
|
417
|
+
const r = data[srcIdx] / 255;
|
|
418
|
+
const g = data[srcIdx + 1] / 255;
|
|
419
|
+
const b = data[srcIdx + 2] / 255;
|
|
420
|
+
const luminance = 0.299 * r + 0.587 * g + 0.114 * b;
|
|
421
|
+
depthMap[y * outW + x] = 1 - luminance;
|
|
422
|
+
}
|
|
423
|
+
}
|
|
424
|
+
return depthMap;
|
|
425
|
+
}
|
|
426
|
+
dispose() {
|
|
427
|
+
this.pipeline = null;
|
|
428
|
+
this._initialized = false;
|
|
429
|
+
this._initializing = null;
|
|
430
|
+
this.modelCache.close();
|
|
431
|
+
}
|
|
432
|
+
};
|
|
433
|
+
|
|
434
|
+
// src/hologram/QuiltCompiler.ts
|
|
435
|
+
var DEVICE_PRESETS = {
|
|
436
|
+
go: {
|
|
437
|
+
views: 48,
|
|
438
|
+
columns: 8,
|
|
439
|
+
rows: 6,
|
|
440
|
+
resolution: [3360, 3360],
|
|
441
|
+
baseline: 0.04
|
|
442
|
+
},
|
|
443
|
+
"16inch": {
|
|
444
|
+
views: 48,
|
|
445
|
+
columns: 8,
|
|
446
|
+
rows: 6,
|
|
447
|
+
resolution: [3360, 3360],
|
|
448
|
+
baseline: 0.06
|
|
449
|
+
},
|
|
450
|
+
"27inch": {
|
|
451
|
+
views: 48,
|
|
452
|
+
columns: 8,
|
|
453
|
+
rows: 6,
|
|
454
|
+
resolution: [4096, 4096],
|
|
455
|
+
baseline: 0.08
|
|
456
|
+
},
|
|
457
|
+
"65inch": {
|
|
458
|
+
views: 100,
|
|
459
|
+
columns: 10,
|
|
460
|
+
rows: 10,
|
|
461
|
+
resolution: [8192, 8192],
|
|
462
|
+
baseline: 0.1
|
|
463
|
+
}
|
|
464
|
+
};
|
|
465
|
+
var DEFAULT_CONFIG = {
|
|
466
|
+
views: 48,
|
|
467
|
+
columns: 8,
|
|
468
|
+
rows: 6,
|
|
469
|
+
resolution: [3360, 3360],
|
|
470
|
+
baseline: 0.06,
|
|
471
|
+
device: "16inch",
|
|
472
|
+
focusDistance: 2
|
|
473
|
+
};
|
|
474
|
+
var QuiltCompiler = class {
|
|
475
|
+
compile(composition, agentToken, outputPath) {
|
|
476
|
+
void agentToken;
|
|
477
|
+
void outputPath;
|
|
478
|
+
const quiltResult = this.compileQuilt(composition);
|
|
479
|
+
return quiltResult.rendererCode;
|
|
480
|
+
}
|
|
481
|
+
/**
|
|
482
|
+
* Compile a HoloComposition to full quilt output with tile parameters.
|
|
483
|
+
*/
|
|
484
|
+
compileQuilt(composition, overrides) {
|
|
485
|
+
const config = this.resolveConfig(composition, overrides);
|
|
486
|
+
const tiles = this.generateTiles(config);
|
|
487
|
+
const rendererCode = this.generateRendererCode(composition, config, tiles);
|
|
488
|
+
const tileWidth = Math.floor(config.resolution[0] / config.columns);
|
|
489
|
+
const tileHeight = Math.floor(config.resolution[1] / config.rows);
|
|
490
|
+
return {
|
|
491
|
+
config,
|
|
492
|
+
tiles,
|
|
493
|
+
rendererCode,
|
|
494
|
+
metadata: {
|
|
495
|
+
quiltAspect: config.resolution[0] / config.resolution[1],
|
|
496
|
+
tileWidth,
|
|
497
|
+
tileHeight,
|
|
498
|
+
numViews: config.views
|
|
499
|
+
}
|
|
500
|
+
};
|
|
501
|
+
}
|
|
502
|
+
/**
|
|
503
|
+
* Resolve quilt configuration from composition @quilt trait and device presets.
|
|
504
|
+
*/
|
|
505
|
+
resolveConfig(composition, overrides) {
|
|
506
|
+
let config = { ...DEFAULT_CONFIG };
|
|
507
|
+
for (const obj of composition.objects) {
|
|
508
|
+
const quiltTrait = obj.traits?.find((t) => t.name === "quilt");
|
|
509
|
+
if (quiltTrait?.config) {
|
|
510
|
+
const p = quiltTrait.config;
|
|
511
|
+
if (typeof p["views"] === "number") config.views = p["views"];
|
|
512
|
+
if (typeof p["columns"] === "number") config.columns = p["columns"];
|
|
513
|
+
if (typeof p["rows"] === "number") config.rows = p["rows"];
|
|
514
|
+
if (Array.isArray(p["resolution"])) config.resolution = p["resolution"];
|
|
515
|
+
if (typeof p["baseline"] === "number") config.baseline = p["baseline"];
|
|
516
|
+
if (typeof p["device"] === "string" && p["device"] in DEVICE_PRESETS) {
|
|
517
|
+
config = {
|
|
518
|
+
...config,
|
|
519
|
+
...DEVICE_PRESETS[p["device"]],
|
|
520
|
+
device: p["device"]
|
|
521
|
+
};
|
|
522
|
+
}
|
|
523
|
+
}
|
|
524
|
+
const lgTrait = obj.traits?.find((t) => t.name === "looking_glass");
|
|
525
|
+
if (lgTrait?.config) {
|
|
526
|
+
const device = lgTrait.config["device"];
|
|
527
|
+
if (device && device in DEVICE_PRESETS) {
|
|
528
|
+
config = {
|
|
529
|
+
...config,
|
|
530
|
+
...DEVICE_PRESETS[device],
|
|
531
|
+
device
|
|
532
|
+
};
|
|
533
|
+
}
|
|
534
|
+
}
|
|
535
|
+
}
|
|
536
|
+
if (overrides) {
|
|
537
|
+
Object.assign(config, overrides);
|
|
538
|
+
}
|
|
539
|
+
return config;
|
|
540
|
+
}
|
|
541
|
+
/**
|
|
542
|
+
* Generate tile parameters for the camera rig.
|
|
543
|
+
* Uses view shearing (asymmetric frustum) instead of camera rotation
|
|
544
|
+
* to prevent toe-in artifacts.
|
|
545
|
+
*
|
|
546
|
+
* @see P.151.01: Multi-View Camera Rig pattern
|
|
547
|
+
*/
|
|
548
|
+
generateTiles(config) {
|
|
549
|
+
const tiles = [];
|
|
550
|
+
const halfBaseline = config.baseline / 2;
|
|
551
|
+
for (let i = 0; i < config.views; i++) {
|
|
552
|
+
const t = config.views > 1 ? i / (config.views - 1) : 0.5;
|
|
553
|
+
const cameraOffset = -halfBaseline + t * config.baseline;
|
|
554
|
+
const viewShear = -cameraOffset / config.focusDistance;
|
|
555
|
+
tiles.push({
|
|
556
|
+
index: i,
|
|
557
|
+
column: i % config.columns,
|
|
558
|
+
row: Math.floor(i / config.columns),
|
|
559
|
+
cameraOffset,
|
|
560
|
+
viewShear
|
|
561
|
+
});
|
|
562
|
+
}
|
|
563
|
+
return tiles;
|
|
564
|
+
}
|
|
565
|
+
/**
|
|
566
|
+
* Generate R3F/Three.js renderer code for the quilt camera rig.
|
|
567
|
+
*/
|
|
568
|
+
generateRendererCode(composition, config, tiles) {
|
|
569
|
+
const tileW = Math.floor(config.resolution[0] / config.columns);
|
|
570
|
+
const tileH = Math.floor(config.resolution[1] / config.rows);
|
|
571
|
+
const sceneObjects = composition.objects.map((obj) => this.objectToJSX(obj)).join("\n ");
|
|
572
|
+
return `// QuiltCompiler output \u2014 ${config.views} views for ${config.device} Looking Glass
|
|
573
|
+
// Tile grid: ${config.columns}x${config.rows} @ ${tileW}x${tileH}px each
|
|
574
|
+
// Total resolution: ${config.resolution[0]}x${config.resolution[1]}
|
|
575
|
+
|
|
576
|
+
import { useRef, useMemo } from 'react';
|
|
577
|
+
import { Canvas, useThree, useFrame } from '@react-three/fiber';
|
|
578
|
+
import * as THREE from 'three';
|
|
579
|
+
|
|
580
|
+
const QUILT_CONFIG = ${JSON.stringify(config, null, 2)};
|
|
581
|
+
|
|
582
|
+
const TILES = ${JSON.stringify(
|
|
583
|
+
tiles.map((t) => ({
|
|
584
|
+
index: t.index,
|
|
585
|
+
col: t.column,
|
|
586
|
+
row: t.row,
|
|
587
|
+
offset: Math.round(t.cameraOffset * 1e4) / 1e4,
|
|
588
|
+
shear: Math.round(t.viewShear * 1e4) / 1e4
|
|
589
|
+
})),
|
|
590
|
+
null,
|
|
591
|
+
2
|
|
592
|
+
)};
|
|
593
|
+
|
|
594
|
+
function QuiltCamera({ tileIndex, children }) {
|
|
595
|
+
const { gl, size } = useThree();
|
|
596
|
+
const cameraRef = useRef();
|
|
597
|
+
const tile = TILES[tileIndex];
|
|
598
|
+
|
|
599
|
+
useFrame(() => {
|
|
600
|
+
if (!cameraRef.current) return;
|
|
601
|
+
const cam = cameraRef.current;
|
|
602
|
+
cam.position.x = tile.offset;
|
|
603
|
+
cam.projectionMatrix.elements[8] = tile.shear;
|
|
604
|
+
cam.updateProjectionMatrix();
|
|
605
|
+
});
|
|
606
|
+
|
|
607
|
+
return (
|
|
608
|
+
<perspectiveCamera ref={cameraRef} fov={14} near={0.1} far={100}>
|
|
609
|
+
{children}
|
|
610
|
+
</perspectiveCamera>
|
|
611
|
+
);
|
|
612
|
+
}
|
|
613
|
+
|
|
614
|
+
function QuiltScene() {
|
|
615
|
+
return (
|
|
616
|
+
<group>
|
|
617
|
+
${sceneObjects}
|
|
618
|
+
</group>
|
|
619
|
+
);
|
|
620
|
+
}
|
|
621
|
+
|
|
622
|
+
export function QuiltRenderer() {
|
|
623
|
+
const tileW = ${tileW};
|
|
624
|
+
const tileH = ${tileH};
|
|
625
|
+
const renderTarget = useMemo(() =>
|
|
626
|
+
new THREE.WebGLRenderTarget(${config.resolution[0]}, ${config.resolution[1]}),
|
|
627
|
+
[]
|
|
628
|
+
);
|
|
629
|
+
|
|
630
|
+
return (
|
|
631
|
+
<Canvas gl={{ preserveDrawingBuffer: true }}>
|
|
632
|
+
<QuiltScene />
|
|
633
|
+
</Canvas>
|
|
634
|
+
);
|
|
635
|
+
}
|
|
636
|
+
`;
|
|
637
|
+
}
|
|
638
|
+
/**
|
|
639
|
+
* Convert a HoloScript object to JSX for the quilt scene.
|
|
640
|
+
*/
|
|
641
|
+
objectToJSX(obj) {
|
|
642
|
+
const getProp = (key) => obj.properties.find((p) => p.key === key)?.value;
|
|
643
|
+
const pos = getProp("position");
|
|
644
|
+
const rot = getProp("rotation");
|
|
645
|
+
const scale = getProp("scale");
|
|
646
|
+
const color = getProp("color") ?? "#888888";
|
|
647
|
+
const geo = getProp("geometry") ?? "box";
|
|
648
|
+
const posStr = Array.isArray(pos) ? `[${pos.join(", ")}]` : "[0, 0, 0]";
|
|
649
|
+
const rotStr = Array.isArray(rot) ? `[${rot.map((r) => Number(r) * Math.PI / 180).join(", ")}]` : void 0;
|
|
650
|
+
const scaleStr = Array.isArray(scale) ? `[${scale.join(", ")}]` : void 0;
|
|
651
|
+
const geoMap = {
|
|
652
|
+
box: "boxGeometry",
|
|
653
|
+
sphere: "sphereGeometry",
|
|
654
|
+
cylinder: "cylinderGeometry",
|
|
655
|
+
torus: "torusGeometry",
|
|
656
|
+
plane: "planeGeometry"
|
|
657
|
+
};
|
|
658
|
+
const geoTag = geoMap[geo] ?? "boxGeometry";
|
|
659
|
+
return `<mesh position={${posStr}}${rotStr ? ` rotation={${rotStr}}` : ""}${scaleStr ? ` scale={${scaleStr}}` : ""}>
|
|
660
|
+
<${geoTag} />
|
|
661
|
+
<meshStandardMaterial color="${color}" />
|
|
662
|
+
</mesh>`;
|
|
663
|
+
}
|
|
664
|
+
};
|
|
665
|
+
|
|
666
|
+
// src/hologram/MVHEVCCompiler.ts
|
|
667
|
+
var DEFAULT_CONFIG2 = {
|
|
668
|
+
ipd: 0.065,
|
|
669
|
+
resolution: [1920, 1080],
|
|
670
|
+
fps: 30,
|
|
671
|
+
convergenceDistance: 2,
|
|
672
|
+
fovDegrees: 90,
|
|
673
|
+
quality: "high",
|
|
674
|
+
container: "mov",
|
|
675
|
+
disparityScale: 1
|
|
676
|
+
};
|
|
677
|
+
var QUALITY_BITRATE = {
|
|
678
|
+
low: 1e7,
|
|
679
|
+
// 10 Mbps per eye
|
|
680
|
+
medium: 25e6,
|
|
681
|
+
// 25 Mbps per eye
|
|
682
|
+
high: 5e7
|
|
683
|
+
// 50 Mbps per eye
|
|
684
|
+
};
|
|
685
|
+
var MVHEVCCompiler = class {
|
|
686
|
+
compile(composition, agentToken, outputPath) {
|
|
687
|
+
void agentToken;
|
|
688
|
+
void outputPath;
|
|
689
|
+
const result = this.compileMVHEVC(composition);
|
|
690
|
+
return result.swiftCode;
|
|
691
|
+
}
|
|
692
|
+
/**
|
|
693
|
+
* Full MV-HEVC compilation with stereo rig, Swift code, and mux command.
|
|
694
|
+
*/
|
|
695
|
+
compileMVHEVC(composition, overrides) {
|
|
696
|
+
const config = this.resolveConfig(composition, overrides);
|
|
697
|
+
const views = this.generateStereoViews(config);
|
|
698
|
+
const swiftCode = this.generateSwiftCode(composition, config);
|
|
699
|
+
const muxCommand = this.generateMuxCommand(config);
|
|
700
|
+
return {
|
|
701
|
+
config,
|
|
702
|
+
views,
|
|
703
|
+
swiftCode,
|
|
704
|
+
muxCommand,
|
|
705
|
+
metadata: {
|
|
706
|
+
stereoMode: "multiview-hevc",
|
|
707
|
+
baseline: config.ipd,
|
|
708
|
+
convergence: config.convergenceDistance,
|
|
709
|
+
horizontalFOV: config.fovDegrees
|
|
710
|
+
}
|
|
711
|
+
};
|
|
712
|
+
}
|
|
713
|
+
/**
|
|
714
|
+
* Generate left/right stereo view parameters.
|
|
715
|
+
* Uses half-IPD offset with view shearing for convergence.
|
|
716
|
+
*/
|
|
717
|
+
generateStereoViews(config) {
|
|
718
|
+
const halfIPD = config.ipd * config.disparityScale / 2;
|
|
719
|
+
const shearFactor = 1 / config.convergenceDistance;
|
|
720
|
+
return [
|
|
721
|
+
{
|
|
722
|
+
eye: "left",
|
|
723
|
+
cameraOffset: -halfIPD,
|
|
724
|
+
viewShear: halfIPD * shearFactor,
|
|
725
|
+
layerIndex: 0
|
|
726
|
+
// Base layer
|
|
727
|
+
},
|
|
728
|
+
{
|
|
729
|
+
eye: "right",
|
|
730
|
+
cameraOffset: halfIPD,
|
|
731
|
+
viewShear: -halfIPD * shearFactor,
|
|
732
|
+
layerIndex: 1
|
|
733
|
+
// Enhancement layer
|
|
734
|
+
}
|
|
735
|
+
];
|
|
736
|
+
}
|
|
737
|
+
/**
|
|
738
|
+
* Extract MV-HEVC config from composition @spatial_video traits.
|
|
739
|
+
*/
|
|
740
|
+
resolveConfig(composition, overrides) {
|
|
741
|
+
const config = { ...DEFAULT_CONFIG2 };
|
|
742
|
+
for (const obj of composition.objects) {
|
|
743
|
+
const svTrait = obj.traits?.find((t) => t.name === "spatial_video");
|
|
744
|
+
if (svTrait?.config) {
|
|
745
|
+
const p = svTrait.config;
|
|
746
|
+
if (typeof p["ipd"] === "number") config.ipd = p["ipd"];
|
|
747
|
+
if (Array.isArray(p["resolution"])) config.resolution = p["resolution"];
|
|
748
|
+
if (typeof p["fps"] === "number") config.fps = p["fps"];
|
|
749
|
+
if (typeof p["convergence"] === "number") config.convergenceDistance = p["convergence"];
|
|
750
|
+
if (typeof p["fov"] === "number") config.fovDegrees = p["fov"];
|
|
751
|
+
if (typeof p["quality"] === "string")
|
|
752
|
+
config.quality = p["quality"];
|
|
753
|
+
}
|
|
754
|
+
}
|
|
755
|
+
if (overrides) Object.assign(config, overrides);
|
|
756
|
+
return config;
|
|
757
|
+
}
|
|
758
|
+
/**
|
|
759
|
+
* Generate Swift code for spatial video playback on Apple Vision Pro.
|
|
760
|
+
*/
|
|
761
|
+
generateSwiftCode(composition, config) {
|
|
762
|
+
const sceneName = composition.name || "SpatialVideoScene";
|
|
763
|
+
const bitrate = QUALITY_BITRATE[config.quality] ?? QUALITY_BITRATE.high;
|
|
764
|
+
return `// MVHEVCCompiler output \u2014 Spatial Video for Apple Vision Pro
|
|
765
|
+
// Stereo baseline: ${config.ipd * 1e3}mm IPD, ${config.resolution[0]}x${config.resolution[1]} per eye
|
|
766
|
+
// Convergence: ${config.convergenceDistance}m, FOV: ${config.fovDegrees}\xB0, ${config.fps}fps
|
|
767
|
+
|
|
768
|
+
import SwiftUI
|
|
769
|
+
import RealityKit
|
|
770
|
+
import AVFoundation
|
|
771
|
+
import AVKit
|
|
772
|
+
|
|
773
|
+
// MARK: - Stereo Camera Rig
|
|
774
|
+
|
|
775
|
+
struct StereoCameraConfig {
|
|
776
|
+
let ipd: Float = ${config.ipd}
|
|
777
|
+
let convergenceDistance: Float = ${config.convergenceDistance}
|
|
778
|
+
let fovDegrees: Float = ${config.fovDegrees}
|
|
779
|
+
let resolution: SIMD2<Int> = [${config.resolution[0]}, ${config.resolution[1]}]
|
|
780
|
+
let fps: Int = ${config.fps}
|
|
781
|
+
let bitrate: Int = ${bitrate}
|
|
782
|
+
}
|
|
783
|
+
|
|
784
|
+
// MARK: - Spatial Video Player
|
|
785
|
+
|
|
786
|
+
struct ${sceneName}: View {
|
|
787
|
+
@State private var player: AVPlayer?
|
|
788
|
+
|
|
789
|
+
var body: some View {
|
|
790
|
+
RealityView { content in
|
|
791
|
+
// Spatial video entity with VideoPlayerComponent
|
|
792
|
+
let entity = Entity()
|
|
793
|
+
|
|
794
|
+
if let url = Bundle.main.url(forResource: "${sceneName.toLowerCase()}", withExtension: "${config.container}") {
|
|
795
|
+
let player = AVPlayer(url: url)
|
|
796
|
+
|
|
797
|
+
// Configure for MV-HEVC spatial video playback
|
|
798
|
+
var videoComponent = VideoPlayerComponent(avPlayer: player)
|
|
799
|
+
entity.components.set(videoComponent)
|
|
800
|
+
|
|
801
|
+
// Position the video surface
|
|
802
|
+
entity.position = [0, 1.5, -2]
|
|
803
|
+
entity.scale = [${(config.resolution[0] / config.resolution[1]).toFixed(2)}, 1.0, 1.0]
|
|
804
|
+
|
|
805
|
+
content.add(entity)
|
|
806
|
+
self.player = player
|
|
807
|
+
}
|
|
808
|
+
}
|
|
809
|
+
.onAppear {
|
|
810
|
+
player?.play()
|
|
811
|
+
}
|
|
812
|
+
}
|
|
813
|
+
}
|
|
814
|
+
|
|
815
|
+
// MARK: - Stereo Rendering Pipeline
|
|
816
|
+
|
|
817
|
+
struct StereoRenderPipeline {
|
|
818
|
+
let config = StereoCameraConfig()
|
|
819
|
+
|
|
820
|
+
/// Render left and right eye views for MV-HEVC encoding
|
|
821
|
+
func renderStereoFrame(scene: Entity, timestamp: TimeInterval) -> (left: CGImage, right: CGImage)? {
|
|
822
|
+
let halfIPD = config.ipd / 2
|
|
823
|
+
|
|
824
|
+
// Left eye: offset camera by -halfIPD
|
|
825
|
+
let leftView = renderEyeView(
|
|
826
|
+
scene: scene,
|
|
827
|
+
cameraOffset: SIMD3<Float>(-halfIPD, 0, 0),
|
|
828
|
+
viewShear: halfIPD / config.convergenceDistance,
|
|
829
|
+
timestamp: timestamp
|
|
830
|
+
)
|
|
831
|
+
|
|
832
|
+
// Right eye: offset camera by +halfIPD
|
|
833
|
+
let rightView = renderEyeView(
|
|
834
|
+
scene: scene,
|
|
835
|
+
cameraOffset: SIMD3<Float>(halfIPD, 0, 0),
|
|
836
|
+
viewShear: -halfIPD / config.convergenceDistance,
|
|
837
|
+
timestamp: timestamp
|
|
838
|
+
)
|
|
839
|
+
|
|
840
|
+
guard let left = leftView, let right = rightView else { return nil }
|
|
841
|
+
return (left, right)
|
|
842
|
+
}
|
|
843
|
+
|
|
844
|
+
private func renderEyeView(
|
|
845
|
+
scene: Entity,
|
|
846
|
+
cameraOffset: SIMD3<Float>,
|
|
847
|
+
viewShear: Float,
|
|
848
|
+
timestamp: TimeInterval
|
|
849
|
+
) -> CGImage? {
|
|
850
|
+
// Configure asymmetric frustum (view shearing, not toe-in)
|
|
851
|
+
// Prevents vertical parallax artifacts
|
|
852
|
+
// Implementation: offset camera position, shear projection matrix column 2
|
|
853
|
+
return nil // Platform-specific Metal rendering
|
|
854
|
+
}
|
|
855
|
+
}
|
|
856
|
+
|
|
857
|
+
// MARK: - AVAssetWriter MV-HEVC Muxer
|
|
858
|
+
|
|
859
|
+
extension StereoRenderPipeline {
|
|
860
|
+
/// Create AVAssetWriter configured for MV-HEVC output
|
|
861
|
+
func createMVHEVCWriter(outputURL: URL) throws -> AVAssetWriter {
|
|
862
|
+
let writer = try AVAssetWriter(outputURL: outputURL, fileType: .mov)
|
|
863
|
+
|
|
864
|
+
// Base layer (left eye) \u2014 HEVC Main Profile
|
|
865
|
+
let baseSettings: [String: Any] = [
|
|
866
|
+
AVVideoCodecKey: AVVideoCodecType.hevc,
|
|
867
|
+
AVVideoWidthKey: config.resolution.x,
|
|
868
|
+
AVVideoHeightKey: config.resolution.y,
|
|
869
|
+
AVVideoCompressionPropertiesKey: [
|
|
870
|
+
AVVideoAverageBitRateKey: config.bitrate,
|
|
871
|
+
AVVideoProfileLevelKey: "HEVC_Main_AutoLevel",
|
|
872
|
+
] as [String: Any],
|
|
873
|
+
]
|
|
874
|
+
let baseInput = AVAssetWriterInput(mediaType: .video, outputSettings: baseSettings)
|
|
875
|
+
baseInput.expectsMediaDataInRealTime = false
|
|
876
|
+
writer.add(baseInput)
|
|
877
|
+
|
|
878
|
+
// Enhancement layer (right eye) \u2014 MV-HEVC Stereo Profile
|
|
879
|
+
// Tagged with kCMTagStereoInterpretation for spatial video
|
|
880
|
+
let enhancementSettings: [String: Any] = [
|
|
881
|
+
AVVideoCodecKey: AVVideoCodecType.hevc,
|
|
882
|
+
AVVideoWidthKey: config.resolution.x,
|
|
883
|
+
AVVideoHeightKey: config.resolution.y,
|
|
884
|
+
AVVideoCompressionPropertiesKey: [
|
|
885
|
+
AVVideoAverageBitRateKey: config.bitrate,
|
|
886
|
+
AVVideoProfileLevelKey: "HEVC_Main_AutoLevel",
|
|
887
|
+
] as [String: Any],
|
|
888
|
+
]
|
|
889
|
+
let enhancementInput = AVAssetWriterInput(mediaType: .video, outputSettings: enhancementSettings)
|
|
890
|
+
enhancementInput.expectsMediaDataInRealTime = false
|
|
891
|
+
writer.add(enhancementInput)
|
|
892
|
+
|
|
893
|
+
return writer
|
|
894
|
+
}
|
|
895
|
+
}
|
|
896
|
+
`;
|
|
897
|
+
}
|
|
898
|
+
/**
|
|
899
|
+
* Generate FFmpeg command for muxing separate L/R HEVC streams into MV-HEVC.
|
|
900
|
+
*/
|
|
901
|
+
generateMuxCommand(config) {
|
|
902
|
+
const bitrate = Math.round(
|
|
903
|
+
(QUALITY_BITRATE[config.quality] ?? QUALITY_BITRATE.high) / 1e6
|
|
904
|
+
);
|
|
905
|
+
return [
|
|
906
|
+
"ffmpeg",
|
|
907
|
+
"-i left_eye.hevc -i right_eye.hevc",
|
|
908
|
+
`-c:v hevc -b:v ${bitrate}M`,
|
|
909
|
+
"-tag:v hvc1",
|
|
910
|
+
`-r ${config.fps}`,
|
|
911
|
+
`-s ${config.resolution[0]}x${config.resolution[1]}`,
|
|
912
|
+
"-movflags +faststart",
|
|
913
|
+
"-brand mp42",
|
|
914
|
+
// MV-HEVC signaling via SEI messages
|
|
915
|
+
"-metadata:s:v:0 stereo_mode=multiview_hevc",
|
|
916
|
+
'-metadata:s:v:0 "handler_name=Left Eye"',
|
|
917
|
+
'-metadata:s:v:1 "handler_name=Right Eye"',
|
|
918
|
+
`-f ${config.container}`,
|
|
919
|
+
`output_spatial.${config.container}`
|
|
920
|
+
].join(" \\\n ");
|
|
921
|
+
}
|
|
922
|
+
};
|
|
923
|
+
|
|
924
|
+
// src/hologram/WebCodecsDepthPipeline.ts
|
|
925
|
+
var DEFAULT_CONFIG3 = {
|
|
926
|
+
maxFps: 30,
|
|
927
|
+
maxDepthResolution: 512,
|
|
928
|
+
temporalAlpha: 0.8,
|
|
929
|
+
codec: "vp9",
|
|
930
|
+
onFrame: void 0,
|
|
931
|
+
onError: void 0
|
|
932
|
+
};
|
|
933
|
+
var CODEC_STRINGS = {
|
|
934
|
+
h264: "avc1.42E01E",
|
|
935
|
+
vp9: "vp09.00.10.08",
|
|
936
|
+
av1: "av01.0.04M.08"
|
|
937
|
+
};
|
|
938
|
+
var WebCodecsDepthPipeline = class _WebCodecsDepthPipeline {
|
|
939
|
+
decoder = null;
|
|
940
|
+
// VideoDecoder
|
|
941
|
+
config;
|
|
942
|
+
depthService;
|
|
943
|
+
canvas = null;
|
|
944
|
+
ctx = null;
|
|
945
|
+
_running = false;
|
|
946
|
+
_disposed = false;
|
|
947
|
+
lastProcessTime = 0;
|
|
948
|
+
minFrameInterval;
|
|
949
|
+
// Stats
|
|
950
|
+
_framesDecoded = 0;
|
|
951
|
+
_framesProcessed = 0;
|
|
952
|
+
_framesSkipped = 0;
|
|
953
|
+
_totalDecodeMs = 0;
|
|
954
|
+
_totalInferenceMs = 0;
|
|
955
|
+
constructor(config) {
|
|
956
|
+
this.config = { ...DEFAULT_CONFIG3, ...config };
|
|
957
|
+
this.minFrameInterval = 1e3 / this.config.maxFps;
|
|
958
|
+
this.depthService = DepthEstimationService.getInstance({
|
|
959
|
+
maxResolution: this.config.maxDepthResolution
|
|
960
|
+
});
|
|
961
|
+
}
|
|
962
|
+
/**
|
|
963
|
+
* Check if WebCodecs API is available in the current environment.
|
|
964
|
+
*/
|
|
965
|
+
static isSupported() {
|
|
966
|
+
return typeof globalThis !== "undefined" && "VideoDecoder" in globalThis && "VideoFrame" in globalThis;
|
|
967
|
+
}
|
|
968
|
+
/**
|
|
969
|
+
* Initialize the pipeline: set up VideoDecoder and DepthEstimationService.
|
|
970
|
+
*/
|
|
971
|
+
async initialize(config) {
|
|
972
|
+
if (config) Object.assign(this.config, config);
|
|
973
|
+
if (!_WebCodecsDepthPipeline.isSupported()) {
|
|
974
|
+
throw new Error("WebCodecs API not supported in this browser");
|
|
975
|
+
}
|
|
976
|
+
await this.depthService.initialize({
|
|
977
|
+
maxResolution: this.config.maxDepthResolution
|
|
978
|
+
});
|
|
979
|
+
this.canvas = new OffscreenCanvas(
|
|
980
|
+
this.config.maxDepthResolution,
|
|
981
|
+
this.config.maxDepthResolution
|
|
982
|
+
);
|
|
983
|
+
this.ctx = this.canvas.getContext("2d");
|
|
984
|
+
const VideoDecoderClass = globalThis.VideoDecoder;
|
|
985
|
+
this.decoder = new VideoDecoderClass({
|
|
986
|
+
output: (frame) => this._handleDecodedFrame(frame),
|
|
987
|
+
error: (err) => {
|
|
988
|
+
this.config.onError?.(err);
|
|
989
|
+
}
|
|
990
|
+
});
|
|
991
|
+
const codecString = CODEC_STRINGS[this.config.codec] ?? CODEC_STRINGS.vp9;
|
|
992
|
+
this.decoder.configure({
|
|
993
|
+
codec: codecString,
|
|
994
|
+
hardwareAcceleration: "prefer-hardware"
|
|
995
|
+
});
|
|
996
|
+
this._running = true;
|
|
997
|
+
}
|
|
998
|
+
/**
|
|
999
|
+
* Feed an encoded video chunk to the decoder.
|
|
1000
|
+
* The chunk will be decoded and processed for depth asynchronously.
|
|
1001
|
+
*/
|
|
1002
|
+
feedChunk(chunk) {
|
|
1003
|
+
if (!this._running || this._disposed || !this.decoder) return;
|
|
1004
|
+
if (this.decoder.state !== "configured") return;
|
|
1005
|
+
this.decoder.decode(chunk);
|
|
1006
|
+
}
|
|
1007
|
+
/**
|
|
1008
|
+
* Process a raw VideoFrame directly (for cases where decoding is handled externally).
|
|
1009
|
+
*/
|
|
1010
|
+
async processFrame(frame) {
|
|
1011
|
+
return this._handleDecodedFrame(frame);
|
|
1012
|
+
}
|
|
1013
|
+
/**
|
|
1014
|
+
* Get pipeline statistics.
|
|
1015
|
+
*/
|
|
1016
|
+
get stats() {
|
|
1017
|
+
return {
|
|
1018
|
+
framesDecoded: this._framesDecoded,
|
|
1019
|
+
framesProcessed: this._framesProcessed,
|
|
1020
|
+
framesSkipped: this._framesSkipped,
|
|
1021
|
+
avgDecodeMs: this._framesDecoded > 0 ? this._totalDecodeMs / this._framesDecoded : 0,
|
|
1022
|
+
avgInferenceMs: this._framesProcessed > 0 ? this._totalInferenceMs / this._framesProcessed : 0,
|
|
1023
|
+
running: this._running
|
|
1024
|
+
};
|
|
1025
|
+
}
|
|
1026
|
+
/**
|
|
1027
|
+
* Handle a decoded VideoFrame: rate-limit, extract pixels, run depth inference.
|
|
1028
|
+
*/
|
|
1029
|
+
async _handleDecodedFrame(frame) {
|
|
1030
|
+
const decodeStart = performance.now();
|
|
1031
|
+
this._framesDecoded++;
|
|
1032
|
+
const now = performance.now();
|
|
1033
|
+
if (now - this.lastProcessTime < this.minFrameInterval) {
|
|
1034
|
+
this._framesSkipped++;
|
|
1035
|
+
frame.close();
|
|
1036
|
+
return null;
|
|
1037
|
+
}
|
|
1038
|
+
this.lastProcessTime = now;
|
|
1039
|
+
this._totalDecodeMs += performance.now() - decodeStart;
|
|
1040
|
+
try {
|
|
1041
|
+
const bitmap = await createImageBitmap(frame);
|
|
1042
|
+
const w = Math.min(bitmap.width, this.config.maxDepthResolution);
|
|
1043
|
+
const h = Math.min(bitmap.height, this.config.maxDepthResolution);
|
|
1044
|
+
if (this.canvas && (this.canvas.width !== w || this.canvas.height !== h)) {
|
|
1045
|
+
this.canvas.width = w;
|
|
1046
|
+
this.canvas.height = h;
|
|
1047
|
+
}
|
|
1048
|
+
this.ctx?.drawImage(bitmap, 0, 0, w, h);
|
|
1049
|
+
bitmap.close();
|
|
1050
|
+
const imageData = this.ctx?.getImageData(0, 0, w, h);
|
|
1051
|
+
if (!imageData) {
|
|
1052
|
+
frame.close();
|
|
1053
|
+
return null;
|
|
1054
|
+
}
|
|
1055
|
+
const inferenceStart = performance.now();
|
|
1056
|
+
const result = await this.depthService.estimateDepth(imageData);
|
|
1057
|
+
this._totalInferenceMs += performance.now() - inferenceStart;
|
|
1058
|
+
this._framesProcessed++;
|
|
1059
|
+
this.config.onFrame?.(result, this._framesProcessed, frame.timestamp ?? 0);
|
|
1060
|
+
frame.close();
|
|
1061
|
+
return result;
|
|
1062
|
+
} catch (err) {
|
|
1063
|
+
frame.close();
|
|
1064
|
+
this.config.onError?.(err instanceof Error ? err : new Error(String(err)));
|
|
1065
|
+
return null;
|
|
1066
|
+
}
|
|
1067
|
+
}
|
|
1068
|
+
/**
|
|
1069
|
+
* Flush any remaining frames in the decoder.
|
|
1070
|
+
*/
|
|
1071
|
+
async flush() {
|
|
1072
|
+
if (this.decoder?.state === "configured") {
|
|
1073
|
+
await this.decoder.flush();
|
|
1074
|
+
}
|
|
1075
|
+
}
|
|
1076
|
+
/**
|
|
1077
|
+
* Stop the pipeline and release all resources.
|
|
1078
|
+
*/
|
|
1079
|
+
dispose() {
|
|
1080
|
+
this._disposed = true;
|
|
1081
|
+
this._running = false;
|
|
1082
|
+
if (this.decoder?.state !== "closed") {
|
|
1083
|
+
try {
|
|
1084
|
+
this.decoder?.close();
|
|
1085
|
+
} catch {
|
|
1086
|
+
}
|
|
1087
|
+
}
|
|
1088
|
+
this.decoder = null;
|
|
1089
|
+
this.canvas = null;
|
|
1090
|
+
this.ctx = null;
|
|
1091
|
+
}
|
|
1092
|
+
};
|
|
1093
|
+
|
|
1094
|
+
export {
|
|
1095
|
+
detectBestBackend,
|
|
1096
|
+
depthToNormalMap,
|
|
1097
|
+
ModelCache,
|
|
1098
|
+
TemporalSmoother,
|
|
1099
|
+
GIFDisposalMethod,
|
|
1100
|
+
GIFDecomposer,
|
|
1101
|
+
DepthEstimationService,
|
|
1102
|
+
QuiltCompiler,
|
|
1103
|
+
MVHEVCCompiler,
|
|
1104
|
+
WebCodecsDepthPipeline,
|
|
1105
|
+
hologram_exports
|
|
1106
|
+
};
|