@kubohiroya/turbowarp-camera-source 0.3.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.ja.md +29 -4
- package/README.md +33 -3
- package/dist/camera-source.js +193 -0
- package/package.json +1 -1
package/README.ja.md
CHANGED
|
@@ -26,13 +26,13 @@ TurboWarp-Camera-Sourceは、MediaDevicesのカメラストリームを名前付
|
|
|
26
26
|
次のURLをunsandboxed custom extensionとして読み込みます。
|
|
27
27
|
|
|
28
28
|
```text
|
|
29
|
-
https://cdn.jsdelivr.net/npm/@kubohiroya/turbowarp-camera-source@0.
|
|
29
|
+
https://cdn.jsdelivr.net/npm/@kubohiroya/turbowarp-camera-source@0.4.0/dist/camera-source.js
|
|
30
30
|
```
|
|
31
31
|
|
|
32
32
|
npm hostでは次を使います。
|
|
33
33
|
|
|
34
34
|
```bash
|
|
35
|
-
pnpm add @kubohiroya/turbowarp-camera-source@0.
|
|
35
|
+
pnpm add @kubohiroya/turbowarp-camera-source@0.4.0
|
|
36
36
|
```
|
|
37
37
|
|
|
38
38
|
## Quick start
|
|
@@ -72,16 +72,41 @@ const qrLease = await cameraSource.acquireCamera({
|
|
|
72
72
|
cameraId: "qr",
|
|
73
73
|
deviceId: qrDeviceId,
|
|
74
74
|
});
|
|
75
|
+
const previewLease = await cameraSource.acquireCamera({
|
|
76
|
+
owner: "camera-preview",
|
|
77
|
+
cameraId: "pose",
|
|
78
|
+
preview: true,
|
|
79
|
+
mirrored: true,
|
|
80
|
+
});
|
|
75
81
|
const frame = qrLease.getFrameSource();
|
|
82
|
+
await previewLease.release();
|
|
76
83
|
await qrLease.release();
|
|
77
84
|
await poseLease.release();
|
|
78
85
|
```
|
|
79
86
|
|
|
80
|
-
`
|
|
87
|
+
`preview: true`を指定したleaseは、GPU-backed stage previewを利用します。専用video skinは共有
|
|
88
|
+
`HTMLVideoElement`を`texImage2D(video)`でアップロードし、preview drawableのX scaleで左右反転します。
|
|
89
|
+
`drawImage()`、`getImageData()`、CPUでのフレームピクセル走査は行いません。既定値は`false`で、同じ
|
|
90
|
+
カメラの最後のpreview leaseが解放されるまで表示を維持します。
|
|
91
|
+
複数のpreview leaseが同じカメラを共有する場合、1つ以上の有効なleaseが`mirrored: true`を要求している間は
|
|
92
|
+
previewを左右反転します。
|
|
93
|
+
|
|
94
|
+
`frame.element`はpreviewを左右反転した場合も未加工の`HTMLVideoElement`です。そのためWebGPUや
|
|
95
|
+
WebCodecsのconsumerは、CPU canvasやpreviewを経由せず、同じsourceを
|
|
96
|
+
`GPUQueue.copyExternalImageToTexture()`や短時間だけ保持する`VideoFrame`へ渡せます。consumerが生成した
|
|
97
|
+
`VideoFrame`はconsumer自身が`close()`する必要があります。
|
|
98
|
+
|
|
99
|
+
この表示経路は明示的なCPU readbackを避けますが、end-to-endの完全なzero-copyは保証しません。ブラウザ内部の
|
|
100
|
+
色変換やGPU転送が発生する可能性があります。またpreviewはprivateかつnoninteractiveで、表示専用です。
|
|
101
|
+
Scratchのtouchingまたはcolor sensing用の画像sourceとしては扱いません。
|
|
102
|
+
|
|
103
|
+
`cameraId`は`pose`や`qr`のような作品内の役割名です。`deviceId`を指定すると、その役割をブラウザが公開する特定のカメラデバイスへ割り当てられます。各カメラは最後のleaseが解放されるまで維持されます。
|
|
81
104
|
|
|
82
105
|
## 互換性
|
|
83
106
|
|
|
84
|
-
Extension ID
|
|
107
|
+
0.4.0ではopt-inのGPU-backed video previewを追加します。Extension ID、ブロックopcode、camera lease
|
|
108
|
+
ownership、device selection、release semanticsは変更しません。`preview: true`を指定しないconsumerは、
|
|
109
|
+
従来のcamera-source動作を維持します。
|
|
85
110
|
|
|
86
111
|
## 開発
|
|
87
112
|
|
package/README.md
CHANGED
|
@@ -30,13 +30,13 @@ or select separate cameras for separate roles.
|
|
|
30
30
|
Load this URL as an unsandboxed custom extension:
|
|
31
31
|
|
|
32
32
|
```text
|
|
33
|
-
https://cdn.jsdelivr.net/npm/@kubohiroya/turbowarp-camera-source@0.
|
|
33
|
+
https://cdn.jsdelivr.net/npm/@kubohiroya/turbowarp-camera-source@0.4.0/dist/camera-source.js
|
|
34
34
|
```
|
|
35
35
|
|
|
36
36
|
For npm hosts:
|
|
37
37
|
|
|
38
38
|
```bash
|
|
39
|
-
pnpm add @kubohiroya/turbowarp-camera-source@0.
|
|
39
|
+
pnpm add @kubohiroya/turbowarp-camera-source@0.4.0
|
|
40
40
|
```
|
|
41
41
|
|
|
42
42
|
## Quick Start
|
|
@@ -154,11 +154,40 @@ const qrLease = await cameraSource.acquireCamera({
|
|
|
154
154
|
cameraId: "qr",
|
|
155
155
|
deviceId: qrDeviceId,
|
|
156
156
|
});
|
|
157
|
+
const previewLease = await cameraSource.acquireCamera({
|
|
158
|
+
owner: "camera-preview",
|
|
159
|
+
cameraId: "pose",
|
|
160
|
+
preview: true,
|
|
161
|
+
mirrored: true,
|
|
162
|
+
});
|
|
163
|
+
const frame = qrLease.getFrameSource();
|
|
164
|
+
await previewLease.release();
|
|
165
|
+
await qrLease.release();
|
|
166
|
+
await poseLease.release();
|
|
157
167
|
```
|
|
158
168
|
|
|
169
|
+
`preview: true` opts that lease into the GPU-backed stage preview. The dedicated video skin uploads
|
|
170
|
+
the shared `HTMLVideoElement` with `texImage2D(video)` and mirrors by changing the preview drawable's
|
|
171
|
+
X scale. It does not call `drawImage()`, `getImageData()`, or scan frame pixels on the CPU. The option
|
|
172
|
+
defaults to `false`; the preview remains visible until the final preview lease for that camera is
|
|
173
|
+
released.
|
|
174
|
+
If multiple preview leases share a camera, the preview is mirrored while any active preview lease
|
|
175
|
+
requests `mirrored: true`.
|
|
176
|
+
|
|
177
|
+
`frame.element` is the original, unmodified `HTMLVideoElement`, even when the preview is mirrored.
|
|
178
|
+
WebGPU and WebCodecs consumers can therefore use that same source without going through the preview
|
|
179
|
+
or a CPU canvas, for example with `GPUQueue.copyExternalImageToTexture()` or a short-lived
|
|
180
|
+
`VideoFrame`. Consumers own and must close any `VideoFrame` they create.
|
|
181
|
+
|
|
182
|
+
The display path avoids explicit CPU readback, but it does not promise end-to-end zero-copy:
|
|
183
|
+
browser-internal color conversion and GPU transfer may still occur. The preview is private and
|
|
184
|
+
noninteractive; it is intended for display, not Scratch touching or color-sensing queries.
|
|
185
|
+
|
|
159
186
|
## Compatibility
|
|
160
187
|
|
|
161
|
-
|
|
188
|
+
Version 0.4.0 adds the opt-in GPU-backed video preview without changing the extension ID,
|
|
189
|
+
block opcodes, camera lease ownership, device selection, or release semantics. Consumers that do
|
|
190
|
+
not request `preview: true` retain the existing camera-source behavior.
|
|
162
191
|
|
|
163
192
|
## Development
|
|
164
193
|
|
|
@@ -194,6 +223,7 @@ The generated JavaScript is a single, non-minified TurboWarp extension file with
|
|
|
194
223
|
- `src/config.ts`: extension metadata
|
|
195
224
|
- `src/block-definitions.json`: canonical block metadata used by both the extension and README generator
|
|
196
225
|
- `src/extension.ts`: extension implementation
|
|
226
|
+
- `src/video-preview.ts`: GPU-backed `HTMLVideoElement` preview skin and drawable lifecycle
|
|
197
227
|
- `src/extension-manifest.ts`: canonical manifest generator and Vite output plugin
|
|
198
228
|
- `src/index.ts`: extension registration entry point
|
|
199
229
|
- `src/globals.d.ts`: Scratch API declarations used by the project
|
package/dist/camera-source.js
CHANGED
|
@@ -103,6 +103,169 @@
|
|
|
103
103
|
]
|
|
104
104
|
};
|
|
105
105
|
//#endregion
|
|
106
|
+
//#region src/video-preview.ts
|
|
107
|
+
var videoLayer = "video";
|
|
108
|
+
var haveCurrentData = 2;
|
|
109
|
+
function videoSkinClass(renderer) {
|
|
110
|
+
const BaseSkin = renderer.exports?.Skin;
|
|
111
|
+
if (typeof BaseSkin !== "function") throw new Error("Camera preview requires renderer.exports.Skin.");
|
|
112
|
+
return class VideoSkin extends BaseSkin {
|
|
113
|
+
constructor(id, skinRenderer, video, onFrame) {
|
|
114
|
+
super(id, skinRenderer);
|
|
115
|
+
this.texture = null;
|
|
116
|
+
this.textureSize = [0, 0];
|
|
117
|
+
this.dirty = true;
|
|
118
|
+
this.disposed = false;
|
|
119
|
+
this.videoFrameCallbackId = null;
|
|
120
|
+
this.animationFrameId = null;
|
|
121
|
+
this.lastCurrentTime = NaN;
|
|
122
|
+
this.handleVideoFrame = () => {
|
|
123
|
+
if (this.disposed) return;
|
|
124
|
+
this.dirty = true;
|
|
125
|
+
this.onFrame(this.syncMetrics());
|
|
126
|
+
this.scheduleFrame();
|
|
127
|
+
};
|
|
128
|
+
this.handleAnimationFrame = () => {
|
|
129
|
+
if (this.disposed) return;
|
|
130
|
+
if (this.video.currentTime !== this.lastCurrentTime) {
|
|
131
|
+
this.dirty = true;
|
|
132
|
+
this.onFrame(this.syncMetrics());
|
|
133
|
+
}
|
|
134
|
+
this.scheduleFrame();
|
|
135
|
+
};
|
|
136
|
+
this.video = video;
|
|
137
|
+
this.onFrame = onFrame;
|
|
138
|
+
this.private = true;
|
|
139
|
+
this.syncMetrics();
|
|
140
|
+
this.scheduleFrame();
|
|
141
|
+
}
|
|
142
|
+
get size() {
|
|
143
|
+
return this.textureSize;
|
|
144
|
+
}
|
|
145
|
+
getTexture() {
|
|
146
|
+
if (this.disposed || this.video.readyState < haveCurrentData || this.video.videoWidth === 0 || this.video.videoHeight === 0) return null;
|
|
147
|
+
if (this.syncMetrics()) this.onFrame(true);
|
|
148
|
+
if (!this.dirty && this.video.currentTime === this.lastCurrentTime) return this.texture;
|
|
149
|
+
const gl = renderer.gl;
|
|
150
|
+
if (!this.texture) {
|
|
151
|
+
this.texture = gl.createTexture();
|
|
152
|
+
if (!this.texture) throw new Error("Camera preview could not create a WebGL texture.");
|
|
153
|
+
gl.bindTexture(gl.TEXTURE_2D, this.texture);
|
|
154
|
+
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE);
|
|
155
|
+
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE);
|
|
156
|
+
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR);
|
|
157
|
+
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.LINEAR);
|
|
158
|
+
} else gl.bindTexture(gl.TEXTURE_2D, this.texture);
|
|
159
|
+
gl.pixelStorei(gl.UNPACK_PREMULTIPLY_ALPHA_WEBGL, true);
|
|
160
|
+
try {
|
|
161
|
+
gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, this.video);
|
|
162
|
+
} finally {
|
|
163
|
+
gl.pixelStorei(gl.UNPACK_PREMULTIPLY_ALPHA_WEBGL, false);
|
|
164
|
+
}
|
|
165
|
+
this.dirty = false;
|
|
166
|
+
this.lastCurrentTime = this.video.currentTime;
|
|
167
|
+
return this.texture;
|
|
168
|
+
}
|
|
169
|
+
dispose() {
|
|
170
|
+
this.disposed = true;
|
|
171
|
+
if (this.videoFrameCallbackId !== null && this.video.cancelVideoFrameCallback) this.video.cancelVideoFrameCallback(this.videoFrameCallbackId);
|
|
172
|
+
if (this.animationFrameId !== null && typeof globalThis.cancelAnimationFrame === "function") globalThis.cancelAnimationFrame(this.animationFrameId);
|
|
173
|
+
if (this.texture) renderer.gl.deleteTexture(this.texture);
|
|
174
|
+
this.texture = null;
|
|
175
|
+
super.dispose();
|
|
176
|
+
}
|
|
177
|
+
useNearest() {
|
|
178
|
+
return false;
|
|
179
|
+
}
|
|
180
|
+
updateSilhouette() {}
|
|
181
|
+
syncMetrics() {
|
|
182
|
+
const width = this.video.videoWidth;
|
|
183
|
+
const height = this.video.videoHeight;
|
|
184
|
+
if (width === this.textureSize[0] && height === this.textureSize[1]) return false;
|
|
185
|
+
this.textureSize = [width, height];
|
|
186
|
+
this.rotationCenter[0] = width / 2;
|
|
187
|
+
this.rotationCenter[1] = height / 2;
|
|
188
|
+
return true;
|
|
189
|
+
}
|
|
190
|
+
scheduleFrame() {
|
|
191
|
+
if (typeof this.video.requestVideoFrameCallback === "function") this.videoFrameCallbackId = this.video.requestVideoFrameCallback(this.handleVideoFrame);
|
|
192
|
+
else if (typeof globalThis.requestAnimationFrame === "function") this.animationFrameId = globalThis.requestAnimationFrame(this.handleAnimationFrame);
|
|
193
|
+
}
|
|
194
|
+
};
|
|
195
|
+
}
|
|
196
|
+
function assertRenderer(renderer) {
|
|
197
|
+
if (!renderer || !Array.isArray(renderer._allSkins) || !Number.isInteger(renderer._nextSkinId) || typeof renderer.createDrawable !== "function" || typeof renderer.destroyDrawable !== "function" || typeof renderer.destroySkin !== "function" || typeof renderer.getNativeSize !== "function") throw new Error("Camera preview requires a compatible TurboWarp renderer.");
|
|
198
|
+
}
|
|
199
|
+
function createVideoPreview(renderer, video, mirrored, requestRedraw) {
|
|
200
|
+
assertRenderer(renderer);
|
|
201
|
+
const VideoSkin = videoSkinClass(renderer);
|
|
202
|
+
const skinId = renderer._nextSkinId++;
|
|
203
|
+
let drawableId;
|
|
204
|
+
let disposed = false;
|
|
205
|
+
let previewMirrored = mirrored;
|
|
206
|
+
let lastLayout = "";
|
|
207
|
+
const updateLayout = () => {
|
|
208
|
+
if (drawableId === void 0 || video.videoWidth === 0 || video.videoHeight === 0) return;
|
|
209
|
+
const [stageWidth, stageHeight] = renderer.getNativeSize();
|
|
210
|
+
const layout = [
|
|
211
|
+
video.videoWidth,
|
|
212
|
+
video.videoHeight,
|
|
213
|
+
stageWidth,
|
|
214
|
+
stageHeight,
|
|
215
|
+
previewMirrored
|
|
216
|
+
].join(":");
|
|
217
|
+
if (layout === lastLayout) return;
|
|
218
|
+
const scale = Math.max(stageWidth / video.videoWidth, stageHeight / video.videoHeight) * 100;
|
|
219
|
+
renderer.updateDrawableScale(drawableId, [previewMirrored ? -scale : scale, scale]);
|
|
220
|
+
lastLayout = layout;
|
|
221
|
+
};
|
|
222
|
+
const skin = new VideoSkin(skinId, renderer, video, (metricsChanged) => {
|
|
223
|
+
if (metricsChanged) skin.emitWasAltered();
|
|
224
|
+
updateLayout();
|
|
225
|
+
requestRedraw();
|
|
226
|
+
});
|
|
227
|
+
renderer._allSkins[skinId] = skin;
|
|
228
|
+
try {
|
|
229
|
+
drawableId = renderer.createDrawable(videoLayer);
|
|
230
|
+
if (drawableId === void 0) throw new Error("Camera preview could not create a video drawable.");
|
|
231
|
+
renderer.updateDrawableSkinId(drawableId, skinId);
|
|
232
|
+
renderer.updateDrawablePosition(drawableId, [0, 0]);
|
|
233
|
+
updateLayout();
|
|
234
|
+
renderer.updateDrawableVisible(drawableId, true);
|
|
235
|
+
renderer.markSkinAsPrivate?.(skinId);
|
|
236
|
+
renderer.markDrawableAsNoninteractive?.(drawableId);
|
|
237
|
+
requestRedraw();
|
|
238
|
+
} catch (error) {
|
|
239
|
+
try {
|
|
240
|
+
if (drawableId !== void 0) renderer.destroyDrawable(drawableId, videoLayer);
|
|
241
|
+
} finally {
|
|
242
|
+
renderer.destroySkin(skinId);
|
|
243
|
+
}
|
|
244
|
+
throw error;
|
|
245
|
+
}
|
|
246
|
+
return Object.freeze({
|
|
247
|
+
setMirrored: (nextMirrored) => {
|
|
248
|
+
if (disposed || previewMirrored === nextMirrored) return;
|
|
249
|
+
previewMirrored = nextMirrored;
|
|
250
|
+
updateLayout();
|
|
251
|
+
requestRedraw();
|
|
252
|
+
},
|
|
253
|
+
dispose: () => {
|
|
254
|
+
if (disposed) return;
|
|
255
|
+
disposed = true;
|
|
256
|
+
try {
|
|
257
|
+
if (drawableId !== void 0) renderer.destroyDrawable(drawableId, videoLayer);
|
|
258
|
+
} finally {
|
|
259
|
+
try {
|
|
260
|
+
renderer.destroySkin(skinId);
|
|
261
|
+
} finally {
|
|
262
|
+
requestRedraw();
|
|
263
|
+
}
|
|
264
|
+
}
|
|
265
|
+
}
|
|
266
|
+
});
|
|
267
|
+
}
|
|
268
|
+
//#endregion
|
|
106
269
|
//#region src/extension.ts
|
|
107
270
|
var blockDefinitions = block_definitions_default.blocks;
|
|
108
271
|
var defaultCameraId = "default";
|
|
@@ -175,6 +338,18 @@
|
|
|
175
338
|
if (session.startPromise) await session.startPromise;
|
|
176
339
|
else if (!session.stream) await this.start(session, options);
|
|
177
340
|
session.leases.add(token);
|
|
341
|
+
try {
|
|
342
|
+
if (options.preview === true) {
|
|
343
|
+
session.previewLeases.set(token, options.mirrored === true);
|
|
344
|
+
this.ensurePreview(session);
|
|
345
|
+
session.preview?.setMirrored(this.previewMirrored(session));
|
|
346
|
+
}
|
|
347
|
+
} catch (error) {
|
|
348
|
+
session.leases.delete(token);
|
|
349
|
+
session.previewLeases.delete(token);
|
|
350
|
+
if (session.leases.size === 0) this.stopCameraSession(session.cameraId);
|
|
351
|
+
throw error;
|
|
352
|
+
}
|
|
178
353
|
let released = false;
|
|
179
354
|
return Object.freeze({
|
|
180
355
|
getFrameSource: () => this.getFrameSource(session),
|
|
@@ -182,6 +357,11 @@
|
|
|
182
357
|
if (released) return;
|
|
183
358
|
released = true;
|
|
184
359
|
session.leases.delete(token);
|
|
360
|
+
session.previewLeases.delete(token);
|
|
361
|
+
if (session.previewLeases.size === 0) {
|
|
362
|
+
session.preview?.dispose();
|
|
363
|
+
session.preview = null;
|
|
364
|
+
} else session.preview?.setMirrored(this.previewMirrored(session));
|
|
185
365
|
if (session.leases.size === 0) this.stopCameraSession(session.cameraId);
|
|
186
366
|
}
|
|
187
367
|
});
|
|
@@ -211,8 +391,10 @@
|
|
|
211
391
|
const session = {
|
|
212
392
|
cameraId,
|
|
213
393
|
leases: /* @__PURE__ */ new Set(),
|
|
394
|
+
previewLeases: /* @__PURE__ */ new Map(),
|
|
214
395
|
stream: null,
|
|
215
396
|
video: null,
|
|
397
|
+
preview: null,
|
|
216
398
|
startPromise: null,
|
|
217
399
|
mirrored: false,
|
|
218
400
|
activeDeviceId: ""
|
|
@@ -244,6 +426,14 @@
|
|
|
244
426
|
const settings = (session.stream?.getVideoTracks()[0] ?? null)?.getSettings();
|
|
245
427
|
session.activeDeviceId = typeof settings?.deviceId === "string" ? settings.deviceId : "";
|
|
246
428
|
}
|
|
429
|
+
ensurePreview(session) {
|
|
430
|
+
if (session.preview || !session.video) return;
|
|
431
|
+
const runtime = Scratch.vm.runtime;
|
|
432
|
+
session.preview = createVideoPreview(runtime.renderer, session.video, this.previewMirrored(session), () => runtime.requestRedraw?.());
|
|
433
|
+
}
|
|
434
|
+
previewMirrored(session) {
|
|
435
|
+
return [...session.previewLeases.values()].some(Boolean);
|
|
436
|
+
}
|
|
247
437
|
getFrameSource(session) {
|
|
248
438
|
if (!session.video || !session.stream) throw new Error("Shared camera is not running.");
|
|
249
439
|
return Object.freeze({
|
|
@@ -258,13 +448,16 @@
|
|
|
258
448
|
stopCameraSession(cameraId) {
|
|
259
449
|
const session = this.sessions.get(cameraId);
|
|
260
450
|
if (!session) return;
|
|
451
|
+
session.preview?.dispose();
|
|
261
452
|
session.stream?.getTracks().forEach((track) => track.stop());
|
|
262
453
|
if (session.video) session.video.srcObject = null;
|
|
263
454
|
session.stream = null;
|
|
264
455
|
session.video = null;
|
|
265
456
|
session.startPromise = null;
|
|
457
|
+
session.preview = null;
|
|
266
458
|
session.activeDeviceId = "";
|
|
267
459
|
session.leases.clear();
|
|
460
|
+
session.previewLeases.clear();
|
|
268
461
|
this.sessions.delete(cameraId);
|
|
269
462
|
this.blockLeases.delete(cameraId);
|
|
270
463
|
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@kubohiroya/turbowarp-camera-source",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.4.0",
|
|
4
4
|
"author": "Hiroya Kubo <hiroya@cuc.ac.jp>",
|
|
5
5
|
"description": "A TurboWarp capability extension for sharing one MediaDevices camera stream across extensions.",
|
|
6
6
|
"type": "module",
|