@twick/2d 0.14.0 → 1.14.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (186) hide show
  1. package/LICENSE +21 -21
  2. package/editor/editor/tsconfig.build.tsbuildinfo +1 -1
  3. package/lib/components/Audio.d.ts.map +1 -1
  4. package/lib/components/Audio.js +33 -3
  5. package/lib/components/CodeBlock.d.ts +1 -1
  6. package/lib/components/Img.js +23 -23
  7. package/lib/components/Line.js +31 -31
  8. package/lib/components/Media.d.ts +6 -0
  9. package/lib/components/Media.d.ts.map +1 -1
  10. package/lib/components/Media.js +277 -61
  11. package/lib/components/Node.d.ts +1 -1
  12. package/lib/components/Path.d.ts +1 -1
  13. package/lib/components/SVG.d.ts +1 -1
  14. package/lib/components/Shape.d.ts +1 -1
  15. package/lib/components/Spline.js +25 -25
  16. package/lib/components/Video.d.ts +0 -1
  17. package/lib/components/Video.d.ts.map +1 -1
  18. package/lib/components/Video.js +70 -65
  19. package/lib/tsconfig.build.tsbuildinfo +1 -1
  20. package/package.json +4 -5
  21. package/src/editor/NodeInspectorConfig.tsx +76 -76
  22. package/src/editor/PreviewOverlayConfig.tsx +67 -67
  23. package/src/editor/Provider.tsx +93 -93
  24. package/src/editor/SceneGraphTabConfig.tsx +81 -81
  25. package/src/editor/icons/CircleIcon.tsx +7 -7
  26. package/src/editor/icons/CodeBlockIcon.tsx +8 -8
  27. package/src/editor/icons/CurveIcon.tsx +7 -7
  28. package/src/editor/icons/GridIcon.tsx +7 -7
  29. package/src/editor/icons/IconMap.ts +35 -35
  30. package/src/editor/icons/ImgIcon.tsx +8 -8
  31. package/src/editor/icons/LayoutIcon.tsx +9 -9
  32. package/src/editor/icons/LineIcon.tsx +7 -7
  33. package/src/editor/icons/NodeIcon.tsx +7 -7
  34. package/src/editor/icons/RayIcon.tsx +7 -7
  35. package/src/editor/icons/RectIcon.tsx +7 -7
  36. package/src/editor/icons/ShapeIcon.tsx +7 -7
  37. package/src/editor/icons/TxtIcon.tsx +8 -8
  38. package/src/editor/icons/VideoIcon.tsx +7 -7
  39. package/src/editor/icons/View2DIcon.tsx +10 -10
  40. package/src/editor/index.ts +17 -17
  41. package/src/editor/tree/DetachedRoot.tsx +23 -23
  42. package/src/editor/tree/NodeElement.tsx +74 -74
  43. package/src/editor/tree/TreeElement.tsx +72 -72
  44. package/src/editor/tree/TreeRoot.tsx +10 -10
  45. package/src/editor/tree/ViewRoot.tsx +20 -20
  46. package/src/editor/tree/index.module.scss +38 -38
  47. package/src/editor/tree/index.ts +3 -3
  48. package/src/editor/tsconfig.build.json +5 -5
  49. package/src/editor/tsconfig.json +12 -12
  50. package/src/editor/tsdoc.json +4 -4
  51. package/src/editor/vite-env.d.ts +1 -1
  52. package/src/lib/code/CodeCursor.ts +445 -445
  53. package/src/lib/code/CodeDiffer.ts +78 -78
  54. package/src/lib/code/CodeFragment.ts +97 -97
  55. package/src/lib/code/CodeHighlighter.ts +75 -75
  56. package/src/lib/code/CodeMetrics.ts +47 -47
  57. package/src/lib/code/CodeRange.test.ts +74 -74
  58. package/src/lib/code/CodeRange.ts +216 -216
  59. package/src/lib/code/CodeScope.ts +101 -101
  60. package/src/lib/code/CodeSelection.ts +24 -24
  61. package/src/lib/code/CodeSignal.ts +327 -327
  62. package/src/lib/code/CodeTokenizer.ts +54 -54
  63. package/src/lib/code/DefaultHighlightStyle.ts +98 -98
  64. package/src/lib/code/LezerHighlighter.ts +113 -113
  65. package/src/lib/code/diff.test.ts +311 -311
  66. package/src/lib/code/diff.ts +319 -319
  67. package/src/lib/code/extractRange.ts +126 -126
  68. package/src/lib/code/index.ts +13 -13
  69. package/src/lib/components/Audio.ts +168 -131
  70. package/src/lib/components/Bezier.ts +105 -105
  71. package/src/lib/components/Circle.ts +266 -266
  72. package/src/lib/components/Code.ts +526 -526
  73. package/src/lib/components/CodeBlock.ts +576 -576
  74. package/src/lib/components/CubicBezier.ts +112 -112
  75. package/src/lib/components/Curve.ts +455 -455
  76. package/src/lib/components/Grid.ts +135 -135
  77. package/src/lib/components/Icon.ts +96 -96
  78. package/src/lib/components/Img.ts +319 -319
  79. package/src/lib/components/Knot.ts +157 -157
  80. package/src/lib/components/Latex.ts +122 -122
  81. package/src/lib/components/Layout.ts +1092 -1092
  82. package/src/lib/components/Line.ts +429 -429
  83. package/src/lib/components/Media.ts +576 -346
  84. package/src/lib/components/Node.ts +1940 -1940
  85. package/src/lib/components/Path.ts +137 -137
  86. package/src/lib/components/Polygon.ts +171 -171
  87. package/src/lib/components/QuadBezier.ts +100 -100
  88. package/src/lib/components/Ray.ts +125 -125
  89. package/src/lib/components/Rect.ts +187 -187
  90. package/src/lib/components/Rive.ts +156 -156
  91. package/src/lib/components/SVG.ts +797 -797
  92. package/src/lib/components/Shape.ts +143 -143
  93. package/src/lib/components/Spline.ts +344 -344
  94. package/src/lib/components/Txt.test.tsx +81 -81
  95. package/src/lib/components/Txt.ts +203 -203
  96. package/src/lib/components/TxtLeaf.ts +205 -205
  97. package/src/lib/components/Video.ts +461 -462
  98. package/src/lib/components/View2D.ts +98 -98
  99. package/src/lib/components/__tests__/children.test.tsx +142 -142
  100. package/src/lib/components/__tests__/clone.test.tsx +126 -126
  101. package/src/lib/components/__tests__/generatorTest.ts +28 -28
  102. package/src/lib/components/__tests__/mockScene2D.ts +45 -45
  103. package/src/lib/components/__tests__/query.test.tsx +122 -122
  104. package/src/lib/components/__tests__/state.test.tsx +60 -60
  105. package/src/lib/components/index.ts +28 -28
  106. package/src/lib/components/types.ts +35 -35
  107. package/src/lib/curves/ArcSegment.ts +159 -159
  108. package/src/lib/curves/CircleSegment.ts +77 -77
  109. package/src/lib/curves/CubicBezierSegment.ts +78 -78
  110. package/src/lib/curves/CurveDrawingInfo.ts +11 -11
  111. package/src/lib/curves/CurvePoint.ts +15 -15
  112. package/src/lib/curves/CurveProfile.ts +7 -7
  113. package/src/lib/curves/KnotInfo.ts +10 -10
  114. package/src/lib/curves/LineSegment.ts +62 -62
  115. package/src/lib/curves/Polynomial.ts +355 -355
  116. package/src/lib/curves/Polynomial2D.ts +62 -62
  117. package/src/lib/curves/PolynomialSegment.ts +124 -124
  118. package/src/lib/curves/QuadBezierSegment.ts +64 -64
  119. package/src/lib/curves/Segment.ts +17 -17
  120. package/src/lib/curves/UniformPolynomialCurveSampler.ts +94 -94
  121. package/src/lib/curves/createCurveProfileLerp.ts +471 -471
  122. package/src/lib/curves/getBezierSplineProfile.ts +223 -223
  123. package/src/lib/curves/getCircleProfile.ts +86 -86
  124. package/src/lib/curves/getPathProfile.ts +178 -178
  125. package/src/lib/curves/getPointAtDistance.ts +21 -21
  126. package/src/lib/curves/getPolylineProfile.test.ts +21 -21
  127. package/src/lib/curves/getPolylineProfile.ts +89 -89
  128. package/src/lib/curves/getRectProfile.ts +139 -139
  129. package/src/lib/curves/index.ts +16 -16
  130. package/src/lib/decorators/canvasStyleSignal.ts +16 -16
  131. package/src/lib/decorators/colorSignal.ts +9 -9
  132. package/src/lib/decorators/compound.ts +72 -72
  133. package/src/lib/decorators/computed.ts +18 -18
  134. package/src/lib/decorators/defaultStyle.ts +18 -18
  135. package/src/lib/decorators/filtersSignal.ts +136 -136
  136. package/src/lib/decorators/index.ts +10 -10
  137. package/src/lib/decorators/initializers.ts +32 -32
  138. package/src/lib/decorators/nodeName.ts +13 -13
  139. package/src/lib/decorators/signal.test.ts +90 -90
  140. package/src/lib/decorators/signal.ts +345 -345
  141. package/src/lib/decorators/spacingSignal.ts +15 -15
  142. package/src/lib/decorators/vector2Signal.ts +30 -30
  143. package/src/lib/globals.d.ts +2 -2
  144. package/src/lib/index.ts +8 -8
  145. package/src/lib/jsx-dev-runtime.ts +2 -2
  146. package/src/lib/jsx-runtime.ts +46 -46
  147. package/src/lib/parse-svg-path.d.ts +14 -14
  148. package/src/lib/partials/Filter.ts +180 -180
  149. package/src/lib/partials/Gradient.ts +102 -102
  150. package/src/lib/partials/Pattern.ts +34 -34
  151. package/src/lib/partials/ShaderConfig.ts +117 -117
  152. package/src/lib/partials/index.ts +4 -4
  153. package/src/lib/partials/types.ts +58 -58
  154. package/src/lib/scenes/Scene2D.ts +242 -242
  155. package/src/lib/scenes/index.ts +3 -3
  156. package/src/lib/scenes/makeScene2D.ts +16 -16
  157. package/src/lib/scenes/useScene2D.ts +6 -6
  158. package/src/lib/tsconfig.build.json +5 -5
  159. package/src/lib/tsconfig.json +10 -10
  160. package/src/lib/tsdoc.json +4 -4
  161. package/src/lib/utils/CanvasUtils.ts +306 -306
  162. package/src/lib/utils/diff.test.ts +453 -453
  163. package/src/lib/utils/diff.ts +148 -148
  164. package/src/lib/utils/index.ts +2 -2
  165. package/src/lib/utils/is.ts +11 -11
  166. package/src/lib/utils/makeSignalExtensions.ts +30 -30
  167. package/src/lib/utils/video/declarations.d.ts +1 -1
  168. package/src/lib/utils/video/ffmpeg-client.ts +50 -50
  169. package/src/lib/utils/video/mp4-parser-manager.ts +72 -72
  170. package/src/lib/utils/video/parser/index.ts +1 -1
  171. package/src/lib/utils/video/parser/parser.ts +257 -257
  172. package/src/lib/utils/video/parser/sampler.ts +72 -72
  173. package/src/lib/utils/video/parser/segment.ts +302 -302
  174. package/src/lib/utils/video/parser/sink.ts +29 -29
  175. package/src/lib/utils/video/parser/utils.ts +31 -31
  176. package/src/tsconfig.base.json +19 -19
  177. package/src/tsconfig.build.json +8 -8
  178. package/src/tsconfig.json +5 -5
  179. package/tsconfig.project.json +7 -7
  180. package/lib/components/utils/waitUntil.d.ts +0 -7
  181. package/lib/components/utils/waitUntil.d.ts.map +0 -1
  182. package/lib/components/utils/waitUntil.js +0 -15
  183. package/lib/utils/waitUntil.d.ts +0 -7
  184. package/lib/utils/waitUntil.d.ts.map +0 -1
  185. package/lib/utils/waitUntil.js +0 -15
  186. package/src/lib/utils/waitUntil.ts +0 -18
@@ -1,462 +1,461 @@
1
- import type {SerializedVector2, SignalValue, SimpleSignal} from '@twick/core';
2
- import {BBox, DependencyContext, PlaybackState} from '@twick/core';
3
- import Hls from 'hls.js';
4
- import {computed, initial, nodeName, signal} from '../decorators';
5
- import type {DesiredLength} from '../partials';
6
- import {drawImage} from '../utils';
7
- import {ImageCommunication} from '../utils/video/ffmpeg-client';
8
- import {dropExtractor, getFrame} from '../utils/video/mp4-parser-manager';
9
- import type {MediaProps} from './Media';
10
- import {Media} from './Media';
11
- import {waitUntil} from '../utils/waitUntil';
12
-
13
- export interface VideoProps extends MediaProps {
14
- /**
15
- * {@inheritDoc Video.alpha}
16
- */
17
- alpha?: SignalValue<number>;
18
- /**
19
- * {@inheritDoc Video.smoothing}
20
- */
21
- smoothing?: SignalValue<boolean>;
22
- /**
23
- * {@inheritDoc Video.decoder}
24
- */
25
- decoder?: SignalValue<'web' | 'ffmpeg' | 'slow' | null>;
26
- }
27
-
28
- @nodeName('Video')
29
- export class Video extends Media {
30
- /**
31
- * The alpha value of this video.
32
- *
33
- * @remarks
34
- * Unlike opacity, the alpha value affects only the video itself, leaving the
35
- * fill, stroke, and children intact.
36
- */
37
- @initial(1)
38
- @signal()
39
- public declare readonly alpha: SimpleSignal<number, this>;
40
-
41
- /**
42
- * Whether the video should be smoothed.
43
- *
44
- * @remarks
45
- * When disabled, the video will be scaled using the nearest neighbor
46
- * interpolation with no smoothing. The resulting video will appear pixelated.
47
- *
48
- * @defaultValue true
49
- */
50
- @initial(true)
51
- @signal()
52
- public declare readonly smoothing: SimpleSignal<boolean, this>;
53
-
54
- /**
55
- * Which decoder to use during rendering. The `web` decoder is the fastest
56
- * but only supports MP4 files. The `ffmpeg` decoder is slower and more resource
57
- * intensive but supports more formats. The `slow` decoder is the slowest but
58
- * supports all formats.
59
- *
60
- * @defaultValue null
61
- */
62
- @initial(null)
63
- @signal()
64
- public declare readonly decoder: SimpleSignal<
65
- 'web' | 'ffmpeg' | 'slow' | null,
66
- this
67
- >;
68
-
69
- public detectedFileType: 'mp4' | 'webm' | 'hls' | 'mov' | 'unknown' =
70
- 'unknown';
71
- private fileTypeWasDetected: boolean = false;
72
-
73
- private static readonly pool: Record<string, HTMLVideoElement> = {};
74
-
75
- private static readonly imageCommunication = !import.meta.hot
76
- ? null
77
- : new ImageCommunication();
78
-
79
- public constructor(props: VideoProps) {
80
- super(props);
81
- }
82
-
83
- protected override desiredSize(): SerializedVector2<DesiredLength> {
84
- const custom = super.desiredSize();
85
- if (custom.x === null && custom.y === null) {
86
- const image = this.video();
87
- return {
88
- x: image.videoWidth,
89
- y: image.videoHeight,
90
- };
91
- }
92
-
93
- return custom;
94
- }
95
-
96
- protected mediaElement(): HTMLVideoElement {
97
- return this.video();
98
- }
99
-
100
- protected seekedMedia(): HTMLVideoElement {
101
- return this.seekedVideo();
102
- }
103
-
104
- protected fastSeekedMedia(): HTMLVideoElement {
105
- return this.fastSeekedVideo();
106
- }
107
-
108
- @computed()
109
- private video(): HTMLVideoElement {
110
- const src = this.src();
111
- const key = `${this.key}/${src}`;
112
- let video = Video.pool[key];
113
- if (!video) {
114
- video = document.createElement('video');
115
- video.crossOrigin = 'anonymous';
116
- video.preload = 'metadata';
117
- video.playsInline = true;
118
- video.setAttribute('webkit-playsinline', 'true');
119
- video.setAttribute('playsinline', 'true');
120
-
121
- // Set initial volume
122
- video.volume = this.getVolume();
123
-
124
- const parsedSrc = new URL(src, window.location.origin);
125
- if (parsedSrc.pathname.endsWith('.m3u8')) {
126
- const hls = new Hls();
127
- hls.loadSource(src);
128
- hls.attachMedia(video);
129
- } else {
130
- video.src = src;
131
- }
132
-
133
- // Add metadata event listeners
134
- video.addEventListener('loadedmetadata', () => {
135
- if (video.duration === Infinity || video.duration === 0) {
136
- // For iOS, we need to seek to the end to get the duration
137
- video.currentTime = 24 * 60 * 60; // 24 hours
138
- }
139
- });
140
-
141
- video.addEventListener('seeked', () => {
142
- if (video.duration === Infinity || video.duration === 0) {
143
- // If we still don't have duration, try a different approach
144
- video.currentTime = 0;
145
- }
146
- });
147
-
148
- // Add durationchange event listener
149
- video.addEventListener('durationchange', () => {
150
- if (video.duration === Infinity || video.duration === 0) {
151
- // Try to force duration calculation
152
- video.currentTime = 0.1;
153
- }
154
- });
155
-
156
- // Add loadeddata event listener
157
- video.addEventListener('loadeddata', () => {
158
- if (video.duration === Infinity || video.duration === 0) {
159
- // Try to force duration calculation
160
- video.currentTime = 0.1;
161
- }
162
- });
163
-
164
- // Add canplay event listener
165
- video.addEventListener('canplay', () => {
166
- if (video.duration === Infinity || video.duration === 0) {
167
- // Try to force duration calculation
168
- video.currentTime = 0.1;
169
- }
170
- });
171
-
172
- Video.pool[key] = video;
173
- }
174
-
175
- // Update volume whenever video is accessed
176
- video.volume = this.getVolume();
177
-
178
- const weNeedToWait = this.waitForCanPlayNecessary(video);
179
- if (!weNeedToWait) {
180
- return video;
181
- }
182
-
183
- DependencyContext.collectPromise(
184
- new Promise<void>(resolve => {
185
- this.waitForCanPlay(video, resolve);
186
- }),
187
- );
188
-
189
- return video;
190
- }
191
-
192
- @computed()
193
- protected seekedVideo(): HTMLVideoElement {
194
- const video = this.video();
195
- const time = this.clampTime(this.time());
196
-
197
- video.playbackRate = this.playbackRate();
198
-
199
- if (!video.paused) {
200
- video.pause();
201
- }
202
-
203
- if (this.lastTime === time) {
204
- return video;
205
- }
206
-
207
- this.setCurrentTime(time);
208
-
209
- return video;
210
- }
211
-
212
- @computed()
213
- protected fastSeekedVideo(): HTMLVideoElement {
214
- const video = this.video();
215
- const time = this.clampTime(this.time());
216
-
217
- video.playbackRate = this.playbackRate();
218
-
219
- if (this.lastTime === time) {
220
- return video;
221
- }
222
-
223
- const playing =
224
- this.playing() && time < video.duration && video.playbackRate > 0;
225
- if (playing) {
226
- if (video.paused) {
227
- DependencyContext.collectPromise(video.play());
228
- }
229
- } else {
230
- if (!video.paused) {
231
- video.pause();
232
- }
233
- }
234
-
235
- // reseek when video is out of sync by more than one second
236
- if (Math.abs(video.currentTime - time) > 1) {
237
- this.setCurrentTime(time);
238
- } else if (!playing) {
239
- video.currentTime = time;
240
- }
241
-
242
- return video;
243
- }
244
-
245
- protected lastFrame: ImageBitmap | null = null;
246
-
247
- protected async webcodecSeekedVideo(): Promise<CanvasImageSource> {
248
- const video = this.video();
249
- const time = this.clampTime(this.time());
250
-
251
- video.playbackRate = this.playbackRate();
252
-
253
- if (this.lastFrame && this.lastTime === time) {
254
- return this.lastFrame;
255
- }
256
-
257
- const fps = this.view().fps() / this.playbackRate();
258
- return getFrame(this.key, video.src, time, fps);
259
- }
260
-
261
- protected async ffmpegSeekedVideo(): Promise<ImageBitmap> {
262
- const video = this.video();
263
- const time = this.clampTime(this.time());
264
- const duration = this.getDuration();
265
-
266
- video.playbackRate = this.playbackRate();
267
-
268
- if (this.lastFrame && this.lastTime === time) {
269
- return this.lastFrame;
270
- }
271
-
272
- const fps = this.view().fps() / this.playbackRate();
273
-
274
- if (!Video.imageCommunication) {
275
- throw new Error('ServerSeekedVideo can only be used with HMR.');
276
- }
277
-
278
- const frame = await Video.imageCommunication.getFrame(
279
- this.key,
280
- video.src,
281
- time,
282
- duration,
283
- fps,
284
- );
285
- this.lastFrame = frame;
286
- this.lastTime = time;
287
-
288
- return frame;
289
- }
290
-
291
- protected async seekFunction() {
292
- const playbackState = this.view().playbackState();
293
-
294
- // During playback
295
- if (
296
- playbackState === PlaybackState.Playing ||
297
- playbackState === PlaybackState.Presenting
298
- ) {
299
- return this.fastSeekedVideo();
300
- }
301
-
302
- if (playbackState === PlaybackState.Paused) {
303
- return this.seekedVideo();
304
- }
305
-
306
- // During rendering, if set explicitly
307
- if (this.decoder() === 'slow') {
308
- return this.seekedVideo();
309
- }
310
-
311
- if (this.decoder() === 'ffmpeg') {
312
- return this.ffmpegSeekedVideo();
313
- }
314
-
315
- if (this.decoder() === 'web') {
316
- return this.webcodecSeekedVideo();
317
- }
318
-
319
- if (!this.fileTypeWasDetected) {
320
- this.detectFileType();
321
- }
322
-
323
- // If not set explicitly, use detected file type to determine decoder
324
- if (this.detectedFileType === 'webm') {
325
- return this.ffmpegSeekedVideo();
326
- }
327
-
328
- if (this.detectedFileType === 'hls') {
329
- return this.seekedVideo();
330
- }
331
-
332
- return this.webcodecSeekedVideo();
333
- }
334
-
335
- protected override async draw(context: CanvasRenderingContext2D) {
336
- this.drawShape(context);
337
- const alpha = this.alpha();
338
- if (alpha > 0) {
339
- const video = await this.seekFunction();
340
-
341
- const box = BBox.fromSizeCentered(this.computedSize());
342
- context.save();
343
- context.clip(this.getPath());
344
- if (alpha < 1) {
345
- context.globalAlpha *= alpha;
346
- }
347
- context.imageSmoothingEnabled = this.smoothing();
348
- drawImage(context, video, box);
349
- context.restore();
350
- }
351
-
352
- if (this.clip()) {
353
- context.clip(this.getPath());
354
- }
355
-
356
- await this.drawChildren(context);
357
- }
358
-
359
- protected override applyFlex() {
360
- super.applyFlex();
361
- const video = this.video();
362
- this.element.style.aspectRatio = (
363
- this.ratio() ?? video.videoWidth / video.videoHeight
364
- ).toString();
365
- }
366
-
367
- public override remove() {
368
- super.remove();
369
- dropExtractor(this.key, this.video().src);
370
- return this;
371
- }
372
-
373
- private handleUnknownFileType(src: string) {
374
- console.warn(
375
- `WARNING: Could not detect file type of video (${src}), will default to using mp4 decoder. If your video file is not an mp4 file, this will lead to an error - to fix this, reencode your video as an mp4 file (better performance) or specify a different decoder: https://docs.re.video/common-issues/slow-rendering#use-mp4-decoder`,
376
- );
377
- this.detectedFileType = 'unknown';
378
- this.fileTypeWasDetected = true;
379
- }
380
-
381
- private detectFileType() {
382
- return DependencyContext.collectPromise(
383
- (async () => {
384
- const src = this.src();
385
- const extension = src.split('?')[0].split('.').pop()?.toLowerCase();
386
-
387
- if (
388
- extension === 'mp4' ||
389
- extension === 'webm' ||
390
- extension === 'mov'
391
- ) {
392
- this.detectedFileType = extension;
393
- this.fileTypeWasDetected = true;
394
- return;
395
- }
396
-
397
- if (extension === 'm3u8') {
398
- this.detectedFileType = 'hls';
399
- this.fileTypeWasDetected = true;
400
- return;
401
- }
402
-
403
- if (!src.startsWith('http://') && !src.startsWith('https://')) {
404
- this.handleUnknownFileType(src);
405
- return;
406
- }
407
-
408
- const response = await fetch(src, {method: 'HEAD'});
409
- const contentType = response.headers.get('Content-Type');
410
-
411
- if (!contentType) {
412
- this.handleUnknownFileType(src);
413
- return;
414
- }
415
-
416
- if (contentType.includes('video/mp4')) {
417
- this.detectedFileType = 'mp4';
418
- this.fileTypeWasDetected = true;
419
- return;
420
- }
421
-
422
- if (contentType.includes('video/webm')) {
423
- this.detectedFileType = 'webm';
424
- this.fileTypeWasDetected = true;
425
- return;
426
- }
427
-
428
- if (contentType.includes('video/quicktime')) {
429
- this.detectedFileType = 'mov';
430
- this.fileTypeWasDetected = true;
431
- return;
432
- }
433
-
434
- if (
435
- contentType.includes('application/vnd.apple.mpegurl') ||
436
- contentType.includes('application/x-mpegURL')
437
- ) {
438
- this.detectedFileType = 'hls';
439
- this.fileTypeWasDetected = true;
440
- return;
441
- }
442
-
443
- this.handleUnknownFileType(src);
444
- })(),
445
- );
446
- }
447
-
448
- public *waitForMetadata() {
449
- const video = this.video();
450
-
451
- // If duration is already available and valid, return immediately
452
- if (video.duration > 0 && video.duration !== Infinity) {
453
- return;
454
- }
455
-
456
- // Try to force duration calculation
457
- video.currentTime = 0.1;
458
-
459
- // Wait for metadata to be loaded with a valid duration
460
- yield* waitUntil(() => video.duration > 0 && video.duration !== Infinity);
461
- }
462
- }
1
+ import type {SerializedVector2, SignalValue, SimpleSignal} from '@twick/core';
2
+ import {BBox, DependencyContext, PlaybackState} from '@twick/core';
3
+ import Hls from 'hls.js';
4
+ import {computed, initial, nodeName, signal} from '../decorators';
5
+ import type {DesiredLength} from '../partials';
6
+ import {drawImage} from '../utils';
7
+ import {ImageCommunication} from '../utils/video/ffmpeg-client';
8
+ import {dropExtractor, getFrame} from '../utils/video/mp4-parser-manager';
9
+ import type {MediaProps} from './Media';
10
+ import {Media} from './Media';
11
+
12
+ export interface VideoProps extends MediaProps {
13
+ /**
14
+ * {@inheritDoc Video.alpha}
15
+ */
16
+ alpha?: SignalValue<number>;
17
+ /**
18
+ * {@inheritDoc Video.smoothing}
19
+ */
20
+ smoothing?: SignalValue<boolean>;
21
+ /**
22
+ * {@inheritDoc Video.decoder}
23
+ */
24
+ decoder?: SignalValue<'web' | 'ffmpeg' | 'slow' | null>;
25
+ }
26
+
27
+ @nodeName('Video')
28
+ export class Video extends Media {
29
+ /**
30
+ * The alpha value of this video.
31
+ *
32
+ * @remarks
33
+ * Unlike opacity, the alpha value affects only the video itself, leaving the
34
+ * fill, stroke, and children intact.
35
+ */
36
+ @initial(1)
37
+ @signal()
38
+ public declare readonly alpha: SimpleSignal<number, this>;
39
+
40
+ /**
41
+ * Whether the video should be smoothed.
42
+ *
43
+ * @remarks
44
+ * When disabled, the video will be scaled using the nearest neighbor
45
+ * interpolation with no smoothing. The resulting video will appear pixelated.
46
+ *
47
+ * @defaultValue true
48
+ */
49
+ @initial(true)
50
+ @signal()
51
+ public declare readonly smoothing: SimpleSignal<boolean, this>;
52
+
53
+ /**
54
+ * Which decoder to use during rendering. The `web` decoder is the fastest
55
+ * but only supports MP4 files. The `ffmpeg` decoder is slower and more resource
56
+ * intensive but supports more formats. The `slow` decoder is the slowest but
57
+ * supports all formats.
58
+ *
59
+ * @defaultValue null
60
+ */
61
+ @initial(null)
62
+ @signal()
63
+ public declare readonly decoder: SimpleSignal<
64
+ 'web' | 'ffmpeg' | 'slow' | null,
65
+ this
66
+ >;
67
+
68
+ public detectedFileType: 'mp4' | 'webm' | 'hls' | 'mov' | 'unknown' =
69
+ 'unknown';
70
+ private fileTypeWasDetected: boolean = false;
71
+
72
+ private static readonly pool: Record<string, HTMLVideoElement> = {};
73
+
74
+ private static readonly imageCommunication = !import.meta.hot
75
+ ? null
76
+ : new ImageCommunication();
77
+
78
+ public constructor(props: VideoProps) {
79
+ super(props);
80
+ }
81
+
82
+ protected override desiredSize(): SerializedVector2<DesiredLength> {
83
+ const custom = super.desiredSize();
84
+ if (custom.x === null && custom.y === null) {
85
+ const image = this.video();
86
+ return {
87
+ x: image.videoWidth,
88
+ y: image.videoHeight,
89
+ };
90
+ }
91
+
92
+ return custom;
93
+ }
94
+
95
+ protected mediaElement(): HTMLVideoElement {
96
+ return this.video();
97
+ }
98
+
99
+ protected seekedMedia(): HTMLVideoElement {
100
+ return this.seekedVideo();
101
+ }
102
+
103
+ protected fastSeekedMedia(): HTMLVideoElement {
104
+ return this.fastSeekedVideo();
105
+ }
106
+
107
+ @computed()
108
+ private video(): HTMLVideoElement {
109
+ const src = this.src();
110
+
111
+ // Use a temporary key for undefined src to avoid conflicts
112
+ const key = `${this.key}/${src || 'pending'}`;
113
+
114
+ let video = Video.pool[key];
115
+ if (!video) {
116
+ video = document.createElement('video');
117
+ video.crossOrigin = 'anonymous';
118
+
119
+ // Only set src if it's valid, otherwise leave it empty
120
+ if (src && src !== 'undefined') {
121
+ try {
122
+ const parsedSrc = new URL(src, window.location.origin);
123
+
124
+ if (parsedSrc.pathname.endsWith('.m3u8')) {
125
+ const hls = new Hls();
126
+ hls.loadSource(src);
127
+ hls.attachMedia(video);
128
+ } else {
129
+ video.src = src;
130
+ }
131
+ } catch (error) {
132
+ // Fallback to direct assignment
133
+ video.src = src;
134
+ }
135
+ }
136
+
137
+ Video.pool[key] = video;
138
+ } else if (src && src !== 'undefined' && video.src !== src) {
139
+ // Update existing video element if src has changed and is now valid
140
+ try {
141
+ const parsedSrc = new URL(src, window.location.origin);
142
+
143
+ if (parsedSrc.pathname.endsWith('.m3u8')) {
144
+ const hls = new Hls();
145
+ hls.loadSource(src);
146
+ hls.attachMedia(video);
147
+ } else {
148
+ video.src = src;
149
+ }
150
+ } catch (error) {
151
+ // Fallback to direct assignment
152
+ video.src = src;
153
+ }
154
+
155
+ // Move video to correct pool key
156
+ delete Video.pool[key];
157
+ const newKey = `${this.key}/${src}`;
158
+ Video.pool[newKey] = video;
159
+ }
160
+
161
+ // If src is still undefined, wait for it to become available
162
+ if (!src || src === 'undefined') {
163
+ DependencyContext.collectPromise(
164
+ new Promise<void>(resolve => {
165
+ // Check periodically for valid src
166
+ const checkSrc = () => {
167
+ const currentSrc = this.src();
168
+ if (currentSrc && currentSrc !== 'undefined') {
169
+ resolve();
170
+ } else {
171
+ setTimeout(checkSrc, 10);
172
+ }
173
+ };
174
+ checkSrc();
175
+ }),
176
+ );
177
+ }
178
+
179
+ const weNeedToWait = this.waitForCanPlayNecessary(video);
180
+
181
+ if (!weNeedToWait) {
182
+ return video;
183
+ }
184
+
185
+ DependencyContext.collectPromise(
186
+ new Promise<void>(resolve => {
187
+ this.waitForCanPlay(video, resolve);
188
+ }),
189
+ );
190
+
191
+ return video;
192
+ }
193
+
194
+ @computed()
195
+ protected seekedVideo(): HTMLVideoElement {
196
+ const video = this.video();
197
+ const time = this.clampTime(this.time());
198
+
199
+ video.playbackRate = this.playbackRate();
200
+
201
+ if (!video.paused) {
202
+ video.pause();
203
+ }
204
+
205
+ if (this.lastTime === time) {
206
+ return video;
207
+ }
208
+
209
+ this.setCurrentTime(time);
210
+
211
+ return video;
212
+ }
213
+
214
+ @computed()
215
+ protected fastSeekedVideo(): HTMLVideoElement {
216
+ const video = this.video();
217
+ const time = this.clampTime(this.time());
218
+
219
+ video.playbackRate = this.playbackRate();
220
+
221
+ if (this.lastTime === time) {
222
+ return video;
223
+ }
224
+
225
+ const playing =
226
+ this.playing() && time < video.duration && video.playbackRate > 0;
227
+
228
+ if (playing) {
229
+ if (video.paused) {
230
+ DependencyContext.collectPromise(video.play());
231
+ }
232
+ } else {
233
+ if (!video.paused) {
234
+ video.pause();
235
+ }
236
+ }
237
+
238
+ // reseek when video is out of sync by more than one second
239
+ if (Math.abs(video.currentTime - time) > 1) {
240
+ this.setCurrentTime(time);
241
+ } else if (!playing) {
242
+ video.currentTime = time;
243
+ }
244
+
245
+ return video;
246
+ }
247
+
248
+ protected lastFrame: ImageBitmap | null = null;
249
+
250
+ protected async webcodecSeekedVideo(): Promise<CanvasImageSource> {
251
+ const video = this.video();
252
+ const time = this.clampTime(this.time());
253
+
254
+ video.playbackRate = this.playbackRate();
255
+
256
+ if (this.lastFrame && this.lastTime === time) {
257
+ return this.lastFrame;
258
+ }
259
+
260
+ const fps = this.view().fps() / this.playbackRate();
261
+ return getFrame(this.key, video.src, time, fps);
262
+ }
263
+
264
+ protected async ffmpegSeekedVideo(): Promise<ImageBitmap> {
265
+ const video = this.video();
266
+ const time = this.clampTime(this.time());
267
+ const duration = this.getDuration();
268
+
269
+ video.playbackRate = this.playbackRate();
270
+
271
+ if (this.lastFrame && this.lastTime === time) {
272
+ return this.lastFrame;
273
+ }
274
+
275
+ const fps = this.view().fps() / this.playbackRate();
276
+
277
+ if (!Video.imageCommunication) {
278
+ throw new Error('ServerSeekedVideo can only be used with HMR.');
279
+ }
280
+
281
+ const frame = await Video.imageCommunication.getFrame(
282
+ this.key,
283
+ video.src,
284
+ time,
285
+ duration,
286
+ fps,
287
+ );
288
+ this.lastFrame = frame;
289
+ this.lastTime = time;
290
+
291
+ return frame;
292
+ }
293
+
294
+ protected async seekFunction() {
295
+ const playbackState = this.view().playbackState();
296
+
297
+ // During playback
298
+ if (
299
+ playbackState === PlaybackState.Playing ||
300
+ playbackState === PlaybackState.Presenting
301
+ ) {
302
+ return this.fastSeekedVideo();
303
+ }
304
+
305
+ if (playbackState === PlaybackState.Paused) {
306
+ return this.seekedVideo();
307
+ }
308
+
309
+ // During rendering, if set explicitly
310
+ if (this.decoder() === 'slow') {
311
+ return this.seekedVideo();
312
+ }
313
+
314
+ if (this.decoder() === 'ffmpeg') {
315
+ return this.ffmpegSeekedVideo();
316
+ }
317
+
318
+ if (this.decoder() === 'web') {
319
+ return this.webcodecSeekedVideo();
320
+ }
321
+
322
+ if (!this.fileTypeWasDetected) {
323
+ this.detectFileType();
324
+ }
325
+
326
+ // If not set explicitly, use detected file type to determine decoder
327
+ if (this.detectedFileType === 'webm') {
328
+ return this.ffmpegSeekedVideo();
329
+ }
330
+
331
+ if (this.detectedFileType === 'hls') {
332
+ return this.seekedVideo();
333
+ }
334
+
335
+ return this.webcodecSeekedVideo();
336
+ }
337
+
338
+ protected override async draw(context: CanvasRenderingContext2D) {
339
+ // Auto-start playback if Twick is playing but media isn't
340
+ this.autoPlayBasedOnTwick();
341
+
342
+ this.drawShape(context);
343
+ const alpha = this.alpha();
344
+ if (alpha > 0) {
345
+ const video = await this.seekFunction();
346
+
347
+ const box = BBox.fromSizeCentered(this.computedSize());
348
+ context.save();
349
+ context.clip(this.getPath());
350
+ if (alpha < 1) {
351
+ context.globalAlpha *= alpha;
352
+ }
353
+ context.imageSmoothingEnabled = this.smoothing();
354
+ drawImage(context, video, box);
355
+ context.restore();
356
+ }
357
+
358
+ if (this.clip()) {
359
+ context.clip(this.getPath());
360
+ }
361
+
362
+ await this.drawChildren(context);
363
+ }
364
+
365
+ protected override applyFlex() {
366
+ super.applyFlex();
367
+ try {
368
+ const video = this.video();
369
+ // Only set aspect ratio if video element is available and has valid dimensions
370
+ if (video && video.videoWidth > 0 && video.videoHeight > 0) {
371
+ this.element.style.aspectRatio = (
372
+ this.ratio() ?? video.videoWidth / video.videoHeight
373
+ ).toString();
374
+ }
375
+ } catch (error) {
376
+ // If video element is not ready yet, skip setting aspect ratio
377
+ // It will be set later when the video becomes available
378
+ }
379
+ }
380
+
381
+ public override remove() {
382
+ super.remove();
383
+ dropExtractor(this.key, this.src());
384
+ return this;
385
+ }
386
+
387
+ private handleUnknownFileType(src: string) {
388
+ console.warn(
389
+ `WARNING: Could not detect file type of video (${src}), will default to using mp4 decoder. If your video file is not an mp4 file, this will lead to an error - to fix this, reencode your video as an mp4 file (better performance) or specify a different decoder: https://docs.re.video/common-issues/slow-rendering#use-mp4-decoder`,
390
+ );
391
+ this.detectedFileType = 'unknown';
392
+ this.fileTypeWasDetected = true;
393
+ }
394
+
395
+ private detectFileType() {
396
+ return DependencyContext.collectPromise(
397
+ (async () => {
398
+ const src = this.src();
399
+ const extension = src.split('?')[0].split('.').pop()?.toLowerCase();
400
+
401
+ if (
402
+ extension === 'mp4' ||
403
+ extension === 'webm' ||
404
+ extension === 'mov'
405
+ ) {
406
+ this.detectedFileType = extension;
407
+ this.fileTypeWasDetected = true;
408
+ return;
409
+ }
410
+
411
+ if (extension === 'm3u8') {
412
+ this.detectedFileType = 'hls';
413
+ this.fileTypeWasDetected = true;
414
+ return;
415
+ }
416
+
417
+ if (!src.startsWith('http://') && !src.startsWith('https://')) {
418
+ this.handleUnknownFileType(src);
419
+ return;
420
+ }
421
+
422
+ const response = await fetch(src, {method: 'HEAD'});
423
+ const contentType = response.headers.get('Content-Type');
424
+
425
+ if (!contentType) {
426
+ this.handleUnknownFileType(src);
427
+ return;
428
+ }
429
+
430
+ if (contentType.includes('video/mp4')) {
431
+ this.detectedFileType = 'mp4';
432
+ this.fileTypeWasDetected = true;
433
+ return;
434
+ }
435
+
436
+ if (contentType.includes('video/webm')) {
437
+ this.detectedFileType = 'webm';
438
+ this.fileTypeWasDetected = true;
439
+ return;
440
+ }
441
+
442
+ if (contentType.includes('video/quicktime')) {
443
+ this.detectedFileType = 'mov';
444
+ this.fileTypeWasDetected = true;
445
+ return;
446
+ }
447
+
448
+ if (
449
+ contentType.includes('application/vnd.apple.mpegurl') ||
450
+ contentType.includes('application/x-mpegURL')
451
+ ) {
452
+ this.detectedFileType = 'hls';
453
+ this.fileTypeWasDetected = true;
454
+ return;
455
+ }
456
+
457
+ this.handleUnknownFileType(src);
458
+ })(),
459
+ );
460
+ }
461
+ }