@kubohiroya/turbowarp-realtime-motion-capture 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (62) hide show
  1. package/LICENSE +373 -0
  2. package/README.ja.md +89 -0
  3. package/README.md +113 -0
  4. package/THIRD_PARTY_NOTICES.md +31 -0
  5. package/config/feature-flags.ts +28 -0
  6. package/config/qr-config.ts +19 -0
  7. package/dist/extension-manifest.json +664 -0
  8. package/dist/turbowarp-realtime-motion-capture.js +82631 -0
  9. package/docs/architecture.ja.md +256 -0
  10. package/docs/architecture.md +294 -0
  11. package/docs/index.html +83 -0
  12. package/docs/turbowarp-extension-api.ja.md +157 -0
  13. package/docs/turbowarp-extension-api.md +1033 -0
  14. package/package.json +105 -0
  15. package/schemas/camera-calibration-v1.json +209 -0
  16. package/schemas/extension-manifest.schema.json +49 -0
  17. package/schemas/performance-dsl-v1.json +72 -0
  18. package/schemas/pose-frame-2d-v1.json +626 -0
  19. package/schemas/pose-frame-2d-v2.json +723 -0
  20. package/schemas/pose-frame-3d-v1.json +714 -0
  21. package/schemas/session-policy-v1.json +138 -0
  22. package/src/avatar/aframe-port.ts +39 -0
  23. package/src/avatar/controller.ts +541 -0
  24. package/src/avatar/kalidokit-adapter.ts +193 -0
  25. package/src/avatar/types.ts +119 -0
  26. package/src/block-definitions.json +810 -0
  27. package/src/calibration/controller.ts +564 -0
  28. package/src/calibration/opencv-backend.ts +347 -0
  29. package/src/calibration/types.ts +53 -0
  30. package/src/config.ts +13 -0
  31. package/src/extension-manifest.ts +182 -0
  32. package/src/extension.ts +913 -0
  33. package/src/frame-sync/controller.ts +416 -0
  34. package/src/frame-sync/detector.ts +278 -0
  35. package/src/frame-sync/pattern-display.ts +161 -0
  36. package/src/frame-sync/pattern.ts +88 -0
  37. package/src/frame-sync/time-source.ts +36 -0
  38. package/src/frame-sync/types.ts +39 -0
  39. package/src/frame-sync/video-frame-pump.ts +116 -0
  40. package/src/fusion/controller.ts +554 -0
  41. package/src/fusion/fuse.ts +385 -0
  42. package/src/fusion/geometry.ts +474 -0
  43. package/src/fusion/glow-stick.ts +173 -0
  44. package/src/fusion/identity.ts +112 -0
  45. package/src/fusion/jitter-buffer.ts +439 -0
  46. package/src/fusion/types.ts +103 -0
  47. package/src/globals.d.ts +57 -0
  48. package/src/index.ts +8 -0
  49. package/src/markers/canvas-sampler.ts +59 -0
  50. package/src/markers/color.ts +53 -0
  51. package/src/markers/sampler.ts +185 -0
  52. package/src/markers/types.ts +49 -0
  53. package/src/pose/controller.ts +388 -0
  54. package/src/pose/pose-frame.ts +128 -0
  55. package/src/pose/tfjs-movenet.ts +34 -0
  56. package/src/pose/types.ts +116 -0
  57. package/src/protocol/codec.ts +248 -0
  58. package/src/protocol/schemas.ts +283 -0
  59. package/src/qr-courier.ts +289 -0
  60. package/src/qr-svg.ts +39 -0
  61. package/src/sprite-skin.ts +112 -0
  62. package/src/webrtc-capability.ts +38 -0
package/schemas/pose-frame-2d-v1.json ADDED
@@ -0,0 +1,626 @@
1
+ {
2
+ "additionalProperties": false,
3
+ "$id": "https://kubohiroya.github.io/multiview-pose/schema/pose-frame-2d-v1.json",
4
+ "type": "object",
5
+ "required": [
6
+ "schema",
7
+ "version",
8
+ "cameraId",
9
+ "peerId",
10
+ "sequence",
11
+ "captureTimestampUs",
12
+ "frameWidth",
13
+ "frameHeight",
14
+ "calibrationId",
15
+ "persons"
16
+ ],
17
+ "properties": {
18
+ "schema": {
19
+ "const": "twrmc/pose-frame-2d",
20
+ "type": "string"
21
+ },
22
+ "version": {
23
+ "const": 1,
24
+ "type": "number"
25
+ },
26
+ "cameraId": {
27
+ "minLength": 1,
28
+ "maxLength": 64,
29
+ "pattern": "^[A-Za-z0-9._-]+$",
30
+ "type": "string"
31
+ },
32
+ "peerId": {
33
+ "minLength": 1,
34
+ "maxLength": 64,
35
+ "pattern": "^[A-Za-z0-9._-]+$",
36
+ "type": "string"
37
+ },
38
+ "sequence": {
39
+ "minimum": 0,
40
+ "maximum": 9007199254740991,
41
+ "type": "integer"
42
+ },
43
+ "captureTimestampUs": {
44
+ "minimum": 0,
45
+ "maximum": 9007199254740991,
46
+ "type": "integer"
47
+ },
48
+ "frameWidth": {
49
+ "minimum": 1,
50
+ "maximum": 16384,
51
+ "type": "integer"
52
+ },
53
+ "frameHeight": {
54
+ "minimum": 1,
55
+ "maximum": 16384,
56
+ "type": "integer"
57
+ },
58
+ "calibrationId": {
59
+ "minLength": 1,
60
+ "maxLength": 64,
61
+ "pattern": "^[A-Za-z0-9._-]+$",
62
+ "type": "string"
63
+ },
64
+ "persons": {
65
+ "maxItems": 6,
66
+ "type": "array",
67
+ "items": {
68
+ "additionalProperties": false,
69
+ "type": "object",
70
+ "required": [
71
+ "trackingId",
72
+ "score",
73
+ "keypoints"
74
+ ],
75
+ "properties": {
76
+ "trackingId": {
77
+ "minLength": 1,
78
+ "maxLength": 64,
79
+ "pattern": "^[A-Za-z0-9._-]+$",
80
+ "type": "string"
81
+ },
82
+ "score": {
83
+ "minimum": 0,
84
+ "maximum": 1,
85
+ "type": "number"
86
+ },
87
+ "keypoints": {
88
+ "type": "array",
89
+ "items": [
90
+ {
91
+ "additionalProperties": false,
92
+ "type": "object",
93
+ "required": [
94
+ "id",
95
+ "x",
96
+ "y",
97
+ "score"
98
+ ],
99
+ "properties": {
100
+ "id": {
101
+ "const": "nose",
102
+ "type": "string"
103
+ },
104
+ "x": {
105
+ "minimum": -1000000,
106
+ "maximum": 1000000,
107
+ "type": "number"
108
+ },
109
+ "y": {
110
+ "minimum": -1000000,
111
+ "maximum": 1000000,
112
+ "type": "number"
113
+ },
114
+ "score": {
115
+ "minimum": 0,
116
+ "maximum": 1,
117
+ "type": "number"
118
+ }
119
+ }
120
+ },
121
+ {
122
+ "additionalProperties": false,
123
+ "type": "object",
124
+ "required": [
125
+ "id",
126
+ "x",
127
+ "y",
128
+ "score"
129
+ ],
130
+ "properties": {
131
+ "id": {
132
+ "const": "left_eye",
133
+ "type": "string"
134
+ },
135
+ "x": {
136
+ "minimum": -1000000,
137
+ "maximum": 1000000,
138
+ "type": "number"
139
+ },
140
+ "y": {
141
+ "minimum": -1000000,
142
+ "maximum": 1000000,
143
+ "type": "number"
144
+ },
145
+ "score": {
146
+ "minimum": 0,
147
+ "maximum": 1,
148
+ "type": "number"
149
+ }
150
+ }
151
+ },
152
+ {
153
+ "additionalProperties": false,
154
+ "type": "object",
155
+ "required": [
156
+ "id",
157
+ "x",
158
+ "y",
159
+ "score"
160
+ ],
161
+ "properties": {
162
+ "id": {
163
+ "const": "right_eye",
164
+ "type": "string"
165
+ },
166
+ "x": {
167
+ "minimum": -1000000,
168
+ "maximum": 1000000,
169
+ "type": "number"
170
+ },
171
+ "y": {
172
+ "minimum": -1000000,
173
+ "maximum": 1000000,
174
+ "type": "number"
175
+ },
176
+ "score": {
177
+ "minimum": 0,
178
+ "maximum": 1,
179
+ "type": "number"
180
+ }
181
+ }
182
+ },
183
+ {
184
+ "additionalProperties": false,
185
+ "type": "object",
186
+ "required": [
187
+ "id",
188
+ "x",
189
+ "y",
190
+ "score"
191
+ ],
192
+ "properties": {
193
+ "id": {
194
+ "const": "left_ear",
195
+ "type": "string"
196
+ },
197
+ "x": {
198
+ "minimum": -1000000,
199
+ "maximum": 1000000,
200
+ "type": "number"
201
+ },
202
+ "y": {
203
+ "minimum": -1000000,
204
+ "maximum": 1000000,
205
+ "type": "number"
206
+ },
207
+ "score": {
208
+ "minimum": 0,
209
+ "maximum": 1,
210
+ "type": "number"
211
+ }
212
+ }
213
+ },
214
+ {
215
+ "additionalProperties": false,
216
+ "type": "object",
217
+ "required": [
218
+ "id",
219
+ "x",
220
+ "y",
221
+ "score"
222
+ ],
223
+ "properties": {
224
+ "id": {
225
+ "const": "right_ear",
226
+ "type": "string"
227
+ },
228
+ "x": {
229
+ "minimum": -1000000,
230
+ "maximum": 1000000,
231
+ "type": "number"
232
+ },
233
+ "y": {
234
+ "minimum": -1000000,
235
+ "maximum": 1000000,
236
+ "type": "number"
237
+ },
238
+ "score": {
239
+ "minimum": 0,
240
+ "maximum": 1,
241
+ "type": "number"
242
+ }
243
+ }
244
+ },
245
+ {
246
+ "additionalProperties": false,
247
+ "type": "object",
248
+ "required": [
249
+ "id",
250
+ "x",
251
+ "y",
252
+ "score"
253
+ ],
254
+ "properties": {
255
+ "id": {
256
+ "const": "left_shoulder",
257
+ "type": "string"
258
+ },
259
+ "x": {
260
+ "minimum": -1000000,
261
+ "maximum": 1000000,
262
+ "type": "number"
263
+ },
264
+ "y": {
265
+ "minimum": -1000000,
266
+ "maximum": 1000000,
267
+ "type": "number"
268
+ },
269
+ "score": {
270
+ "minimum": 0,
271
+ "maximum": 1,
272
+ "type": "number"
273
+ }
274
+ }
275
+ },
276
+ {
277
+ "additionalProperties": false,
278
+ "type": "object",
279
+ "required": [
280
+ "id",
281
+ "x",
282
+ "y",
283
+ "score"
284
+ ],
285
+ "properties": {
286
+ "id": {
287
+ "const": "right_shoulder",
288
+ "type": "string"
289
+ },
290
+ "x": {
291
+ "minimum": -1000000,
292
+ "maximum": 1000000,
293
+ "type": "number"
294
+ },
295
+ "y": {
296
+ "minimum": -1000000,
297
+ "maximum": 1000000,
298
+ "type": "number"
299
+ },
300
+ "score": {
301
+ "minimum": 0,
302
+ "maximum": 1,
303
+ "type": "number"
304
+ }
305
+ }
306
+ },
307
+ {
308
+ "additionalProperties": false,
309
+ "type": "object",
310
+ "required": [
311
+ "id",
312
+ "x",
313
+ "y",
314
+ "score"
315
+ ],
316
+ "properties": {
317
+ "id": {
318
+ "const": "left_elbow",
319
+ "type": "string"
320
+ },
321
+ "x": {
322
+ "minimum": -1000000,
323
+ "maximum": 1000000,
324
+ "type": "number"
325
+ },
326
+ "y": {
327
+ "minimum": -1000000,
328
+ "maximum": 1000000,
329
+ "type": "number"
330
+ },
331
+ "score": {
332
+ "minimum": 0,
333
+ "maximum": 1,
334
+ "type": "number"
335
+ }
336
+ }
337
+ },
338
+ {
339
+ "additionalProperties": false,
340
+ "type": "object",
341
+ "required": [
342
+ "id",
343
+ "x",
344
+ "y",
345
+ "score"
346
+ ],
347
+ "properties": {
348
+ "id": {
349
+ "const": "right_elbow",
350
+ "type": "string"
351
+ },
352
+ "x": {
353
+ "minimum": -1000000,
354
+ "maximum": 1000000,
355
+ "type": "number"
356
+ },
357
+ "y": {
358
+ "minimum": -1000000,
359
+ "maximum": 1000000,
360
+ "type": "number"
361
+ },
362
+ "score": {
363
+ "minimum": 0,
364
+ "maximum": 1,
365
+ "type": "number"
366
+ }
367
+ }
368
+ },
369
+ {
370
+ "additionalProperties": false,
371
+ "type": "object",
372
+ "required": [
373
+ "id",
374
+ "x",
375
+ "y",
376
+ "score"
377
+ ],
378
+ "properties": {
379
+ "id": {
380
+ "const": "left_wrist",
381
+ "type": "string"
382
+ },
383
+ "x": {
384
+ "minimum": -1000000,
385
+ "maximum": 1000000,
386
+ "type": "number"
387
+ },
388
+ "y": {
389
+ "minimum": -1000000,
390
+ "maximum": 1000000,
391
+ "type": "number"
392
+ },
393
+ "score": {
394
+ "minimum": 0,
395
+ "maximum": 1,
396
+ "type": "number"
397
+ }
398
+ }
399
+ },
400
+ {
401
+ "additionalProperties": false,
402
+ "type": "object",
403
+ "required": [
404
+ "id",
405
+ "x",
406
+ "y",
407
+ "score"
408
+ ],
409
+ "properties": {
410
+ "id": {
411
+ "const": "right_wrist",
412
+ "type": "string"
413
+ },
414
+ "x": {
415
+ "minimum": -1000000,
416
+ "maximum": 1000000,
417
+ "type": "number"
418
+ },
419
+ "y": {
420
+ "minimum": -1000000,
421
+ "maximum": 1000000,
422
+ "type": "number"
423
+ },
424
+ "score": {
425
+ "minimum": 0,
426
+ "maximum": 1,
427
+ "type": "number"
428
+ }
429
+ }
430
+ },
431
+ {
432
+ "additionalProperties": false,
433
+ "type": "object",
434
+ "required": [
435
+ "id",
436
+ "x",
437
+ "y",
438
+ "score"
439
+ ],
440
+ "properties": {
441
+ "id": {
442
+ "const": "left_hip",
443
+ "type": "string"
444
+ },
445
+ "x": {
446
+ "minimum": -1000000,
447
+ "maximum": 1000000,
448
+ "type": "number"
449
+ },
450
+ "y": {
451
+ "minimum": -1000000,
452
+ "maximum": 1000000,
453
+ "type": "number"
454
+ },
455
+ "score": {
456
+ "minimum": 0,
457
+ "maximum": 1,
458
+ "type": "number"
459
+ }
460
+ }
461
+ },
462
+ {
463
+ "additionalProperties": false,
464
+ "type": "object",
465
+ "required": [
466
+ "id",
467
+ "x",
468
+ "y",
469
+ "score"
470
+ ],
471
+ "properties": {
472
+ "id": {
473
+ "const": "right_hip",
474
+ "type": "string"
475
+ },
476
+ "x": {
477
+ "minimum": -1000000,
478
+ "maximum": 1000000,
479
+ "type": "number"
480
+ },
481
+ "y": {
482
+ "minimum": -1000000,
483
+ "maximum": 1000000,
484
+ "type": "number"
485
+ },
486
+ "score": {
487
+ "minimum": 0,
488
+ "maximum": 1,
489
+ "type": "number"
490
+ }
491
+ }
492
+ },
493
+ {
494
+ "additionalProperties": false,
495
+ "type": "object",
496
+ "required": [
497
+ "id",
498
+ "x",
499
+ "y",
500
+ "score"
501
+ ],
502
+ "properties": {
503
+ "id": {
504
+ "const": "left_knee",
505
+ "type": "string"
506
+ },
507
+ "x": {
508
+ "minimum": -1000000,
509
+ "maximum": 1000000,
510
+ "type": "number"
511
+ },
512
+ "y": {
513
+ "minimum": -1000000,
514
+ "maximum": 1000000,
515
+ "type": "number"
516
+ },
517
+ "score": {
518
+ "minimum": 0,
519
+ "maximum": 1,
520
+ "type": "number"
521
+ }
522
+ }
523
+ },
524
+ {
525
+ "additionalProperties": false,
526
+ "type": "object",
527
+ "required": [
528
+ "id",
529
+ "x",
530
+ "y",
531
+ "score"
532
+ ],
533
+ "properties": {
534
+ "id": {
535
+ "const": "right_knee",
536
+ "type": "string"
537
+ },
538
+ "x": {
539
+ "minimum": -1000000,
540
+ "maximum": 1000000,
541
+ "type": "number"
542
+ },
543
+ "y": {
544
+ "minimum": -1000000,
545
+ "maximum": 1000000,
546
+ "type": "number"
547
+ },
548
+ "score": {
549
+ "minimum": 0,
550
+ "maximum": 1,
551
+ "type": "number"
552
+ }
553
+ }
554
+ },
555
+ {
556
+ "additionalProperties": false,
557
+ "type": "object",
558
+ "required": [
559
+ "id",
560
+ "x",
561
+ "y",
562
+ "score"
563
+ ],
564
+ "properties": {
565
+ "id": {
566
+ "const": "left_ankle",
567
+ "type": "string"
568
+ },
569
+ "x": {
570
+ "minimum": -1000000,
571
+ "maximum": 1000000,
572
+ "type": "number"
573
+ },
574
+ "y": {
575
+ "minimum": -1000000,
576
+ "maximum": 1000000,
577
+ "type": "number"
578
+ },
579
+ "score": {
580
+ "minimum": 0,
581
+ "maximum": 1,
582
+ "type": "number"
583
+ }
584
+ }
585
+ },
586
+ {
587
+ "additionalProperties": false,
588
+ "type": "object",
589
+ "required": [
590
+ "id",
591
+ "x",
592
+ "y",
593
+ "score"
594
+ ],
595
+ "properties": {
596
+ "id": {
597
+ "const": "right_ankle",
598
+ "type": "string"
599
+ },
600
+ "x": {
601
+ "minimum": -1000000,
602
+ "maximum": 1000000,
603
+ "type": "number"
604
+ },
605
+ "y": {
606
+ "minimum": -1000000,
607
+ "maximum": 1000000,
608
+ "type": "number"
609
+ },
610
+ "score": {
611
+ "minimum": 0,
612
+ "maximum": 1,
613
+ "type": "number"
614
+ }
615
+ }
616
+ }
617
+ ],
618
+ "additionalItems": false,
619
+ "minItems": 17,
620
+ "maxItems": 17
621
+ }
622
+ }
623
+ }
624
+ }
625
+ }
626
+ }