@twick/2d 0.13.0 → 0.14.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (171) hide show
  1. package/LICENSE +21 -0
  2. package/editor/editor/tsconfig.build.tsbuildinfo +1 -1
  3. package/lib/components/Audio.js +3 -3
  4. package/lib/components/Img.js +23 -23
  5. package/lib/components/Line.js +31 -31
  6. package/lib/components/Media.d.ts +1 -1
  7. package/lib/components/Media.d.ts.map +1 -1
  8. package/lib/components/Media.js +26 -26
  9. package/lib/components/Spline.js +25 -25
  10. package/lib/components/Video.js +3 -3
  11. package/lib/tsconfig.build.tsbuildinfo +1 -1
  12. package/package.json +5 -4
  13. package/src/editor/NodeInspectorConfig.tsx +76 -76
  14. package/src/editor/PreviewOverlayConfig.tsx +67 -67
  15. package/src/editor/Provider.tsx +93 -93
  16. package/src/editor/SceneGraphTabConfig.tsx +81 -81
  17. package/src/editor/icons/CircleIcon.tsx +7 -7
  18. package/src/editor/icons/CodeBlockIcon.tsx +8 -8
  19. package/src/editor/icons/CurveIcon.tsx +7 -7
  20. package/src/editor/icons/GridIcon.tsx +7 -7
  21. package/src/editor/icons/IconMap.ts +35 -35
  22. package/src/editor/icons/ImgIcon.tsx +8 -8
  23. package/src/editor/icons/LayoutIcon.tsx +9 -9
  24. package/src/editor/icons/LineIcon.tsx +7 -7
  25. package/src/editor/icons/NodeIcon.tsx +7 -7
  26. package/src/editor/icons/RayIcon.tsx +7 -7
  27. package/src/editor/icons/RectIcon.tsx +7 -7
  28. package/src/editor/icons/ShapeIcon.tsx +7 -7
  29. package/src/editor/icons/TxtIcon.tsx +8 -8
  30. package/src/editor/icons/VideoIcon.tsx +7 -7
  31. package/src/editor/icons/View2DIcon.tsx +10 -10
  32. package/src/editor/index.ts +17 -17
  33. package/src/editor/tree/DetachedRoot.tsx +23 -23
  34. package/src/editor/tree/NodeElement.tsx +74 -74
  35. package/src/editor/tree/TreeElement.tsx +72 -72
  36. package/src/editor/tree/TreeRoot.tsx +10 -10
  37. package/src/editor/tree/ViewRoot.tsx +20 -20
  38. package/src/editor/tree/index.module.scss +38 -38
  39. package/src/editor/tree/index.ts +3 -3
  40. package/src/editor/tsconfig.build.json +5 -5
  41. package/src/editor/tsconfig.json +12 -12
  42. package/src/editor/tsdoc.json +4 -4
  43. package/src/editor/vite-env.d.ts +1 -1
  44. package/src/lib/code/CodeCursor.ts +445 -445
  45. package/src/lib/code/CodeDiffer.ts +78 -78
  46. package/src/lib/code/CodeFragment.ts +97 -97
  47. package/src/lib/code/CodeHighlighter.ts +75 -75
  48. package/src/lib/code/CodeMetrics.ts +47 -47
  49. package/src/lib/code/CodeRange.test.ts +74 -74
  50. package/src/lib/code/CodeRange.ts +216 -216
  51. package/src/lib/code/CodeScope.ts +101 -101
  52. package/src/lib/code/CodeSelection.ts +24 -24
  53. package/src/lib/code/CodeSignal.ts +327 -327
  54. package/src/lib/code/CodeTokenizer.ts +54 -54
  55. package/src/lib/code/DefaultHighlightStyle.ts +98 -98
  56. package/src/lib/code/LezerHighlighter.ts +113 -113
  57. package/src/lib/code/diff.test.ts +311 -311
  58. package/src/lib/code/diff.ts +319 -319
  59. package/src/lib/code/extractRange.ts +126 -126
  60. package/src/lib/code/index.ts +13 -13
  61. package/src/lib/components/Audio.ts +168 -168
  62. package/src/lib/components/Bezier.ts +105 -105
  63. package/src/lib/components/Circle.ts +266 -266
  64. package/src/lib/components/Code.ts +526 -526
  65. package/src/lib/components/CodeBlock.ts +576 -576
  66. package/src/lib/components/CubicBezier.ts +112 -112
  67. package/src/lib/components/Curve.ts +455 -455
  68. package/src/lib/components/Grid.ts +135 -135
  69. package/src/lib/components/Icon.ts +96 -96
  70. package/src/lib/components/Img.ts +319 -319
  71. package/src/lib/components/Knot.ts +157 -157
  72. package/src/lib/components/Latex.ts +122 -122
  73. package/src/lib/components/Layout.ts +1092 -1092
  74. package/src/lib/components/Line.ts +429 -429
  75. package/src/lib/components/Media.ts +576 -576
  76. package/src/lib/components/Node.ts +1940 -1940
  77. package/src/lib/components/Path.ts +137 -137
  78. package/src/lib/components/Polygon.ts +171 -171
  79. package/src/lib/components/QuadBezier.ts +100 -100
  80. package/src/lib/components/Ray.ts +125 -125
  81. package/src/lib/components/Rect.ts +187 -187
  82. package/src/lib/components/Rive.ts +156 -156
  83. package/src/lib/components/SVG.ts +797 -797
  84. package/src/lib/components/Shape.ts +143 -143
  85. package/src/lib/components/Spline.ts +344 -344
  86. package/src/lib/components/Txt.test.tsx +81 -81
  87. package/src/lib/components/Txt.ts +203 -203
  88. package/src/lib/components/TxtLeaf.ts +205 -205
  89. package/src/lib/components/Video.ts +461 -461
  90. package/src/lib/components/View2D.ts +98 -98
  91. package/src/lib/components/__tests__/children.test.tsx +142 -142
  92. package/src/lib/components/__tests__/clone.test.tsx +126 -126
  93. package/src/lib/components/__tests__/generatorTest.ts +28 -28
  94. package/src/lib/components/__tests__/mockScene2D.ts +45 -45
  95. package/src/lib/components/__tests__/query.test.tsx +122 -122
  96. package/src/lib/components/__tests__/state.test.tsx +60 -60
  97. package/src/lib/components/index.ts +28 -28
  98. package/src/lib/components/types.ts +35 -35
  99. package/src/lib/curves/ArcSegment.ts +159 -159
  100. package/src/lib/curves/CircleSegment.ts +77 -77
  101. package/src/lib/curves/CubicBezierSegment.ts +78 -78
  102. package/src/lib/curves/CurveDrawingInfo.ts +11 -11
  103. package/src/lib/curves/CurvePoint.ts +15 -15
  104. package/src/lib/curves/CurveProfile.ts +7 -7
  105. package/src/lib/curves/KnotInfo.ts +10 -10
  106. package/src/lib/curves/LineSegment.ts +62 -62
  107. package/src/lib/curves/Polynomial.ts +355 -355
  108. package/src/lib/curves/Polynomial2D.ts +62 -62
  109. package/src/lib/curves/PolynomialSegment.ts +124 -124
  110. package/src/lib/curves/QuadBezierSegment.ts +64 -64
  111. package/src/lib/curves/Segment.ts +17 -17
  112. package/src/lib/curves/UniformPolynomialCurveSampler.ts +94 -94
  113. package/src/lib/curves/createCurveProfileLerp.ts +471 -471
  114. package/src/lib/curves/getBezierSplineProfile.ts +223 -223
  115. package/src/lib/curves/getCircleProfile.ts +86 -86
  116. package/src/lib/curves/getPathProfile.ts +178 -178
  117. package/src/lib/curves/getPointAtDistance.ts +21 -21
  118. package/src/lib/curves/getPolylineProfile.test.ts +21 -21
  119. package/src/lib/curves/getPolylineProfile.ts +89 -89
  120. package/src/lib/curves/getRectProfile.ts +139 -139
  121. package/src/lib/curves/index.ts +16 -16
  122. package/src/lib/decorators/canvasStyleSignal.ts +16 -16
  123. package/src/lib/decorators/colorSignal.ts +9 -9
  124. package/src/lib/decorators/compound.ts +72 -72
  125. package/src/lib/decorators/computed.ts +18 -18
  126. package/src/lib/decorators/defaultStyle.ts +18 -18
  127. package/src/lib/decorators/filtersSignal.ts +136 -136
  128. package/src/lib/decorators/index.ts +10 -10
  129. package/src/lib/decorators/initializers.ts +32 -32
  130. package/src/lib/decorators/nodeName.ts +13 -13
  131. package/src/lib/decorators/signal.test.ts +90 -90
  132. package/src/lib/decorators/signal.ts +345 -345
  133. package/src/lib/decorators/spacingSignal.ts +15 -15
  134. package/src/lib/decorators/vector2Signal.ts +30 -30
  135. package/src/lib/globals.d.ts +2 -2
  136. package/src/lib/index.ts +8 -8
  137. package/src/lib/jsx-dev-runtime.ts +2 -2
  138. package/src/lib/jsx-runtime.ts +46 -46
  139. package/src/lib/parse-svg-path.d.ts +14 -14
  140. package/src/lib/partials/Filter.ts +180 -180
  141. package/src/lib/partials/Gradient.ts +102 -102
  142. package/src/lib/partials/Pattern.ts +34 -34
  143. package/src/lib/partials/ShaderConfig.ts +117 -117
  144. package/src/lib/partials/index.ts +4 -4
  145. package/src/lib/partials/types.ts +58 -58
  146. package/src/lib/scenes/Scene2D.ts +242 -242
  147. package/src/lib/scenes/index.ts +3 -3
  148. package/src/lib/scenes/makeScene2D.ts +16 -16
  149. package/src/lib/scenes/useScene2D.ts +6 -6
  150. package/src/lib/tsconfig.build.json +5 -5
  151. package/src/lib/tsconfig.json +10 -10
  152. package/src/lib/tsdoc.json +4 -4
  153. package/src/lib/utils/CanvasUtils.ts +306 -306
  154. package/src/lib/utils/diff.test.ts +453 -453
  155. package/src/lib/utils/diff.ts +148 -148
  156. package/src/lib/utils/index.ts +2 -2
  157. package/src/lib/utils/is.ts +11 -11
  158. package/src/lib/utils/makeSignalExtensions.ts +30 -30
  159. package/src/lib/utils/video/declarations.d.ts +1 -1
  160. package/src/lib/utils/video/ffmpeg-client.ts +50 -50
  161. package/src/lib/utils/video/mp4-parser-manager.ts +72 -72
  162. package/src/lib/utils/video/parser/index.ts +1 -1
  163. package/src/lib/utils/video/parser/parser.ts +257 -257
  164. package/src/lib/utils/video/parser/sampler.ts +72 -72
  165. package/src/lib/utils/video/parser/segment.ts +302 -302
  166. package/src/lib/utils/video/parser/sink.ts +29 -29
  167. package/src/lib/utils/video/parser/utils.ts +31 -31
  168. package/src/tsconfig.base.json +19 -19
  169. package/src/tsconfig.build.json +8 -8
  170. package/src/tsconfig.json +5 -5
  171. package/tsconfig.project.json +7 -7
@@ -1,461 +1,461 @@
1
- import type {SerializedVector2, SignalValue, SimpleSignal} from '@twick/core';
2
- import {BBox, DependencyContext, PlaybackState} from '@twick/core';
3
- import Hls from 'hls.js';
4
- import {computed, initial, nodeName, signal} from '../decorators';
5
- import type {DesiredLength} from '../partials';
6
- import {drawImage} from '../utils';
7
- import {ImageCommunication} from '../utils/video/ffmpeg-client';
8
- import {dropExtractor, getFrame} from '../utils/video/mp4-parser-manager';
9
- import type {MediaProps} from './Media';
10
- import {Media} from './Media';
11
-
12
- export interface VideoProps extends MediaProps {
13
- /**
14
- * {@inheritDoc Video.alpha}
15
- */
16
- alpha?: SignalValue<number>;
17
- /**
18
- * {@inheritDoc Video.smoothing}
19
- */
20
- smoothing?: SignalValue<boolean>;
21
- /**
22
- * {@inheritDoc Video.decoder}
23
- */
24
- decoder?: SignalValue<'web' | 'ffmpeg' | 'slow' | null>;
25
- }
26
-
27
- @nodeName('Video')
28
- export class Video extends Media {
29
- /**
30
- * The alpha value of this video.
31
- *
32
- * @remarks
33
- * Unlike opacity, the alpha value affects only the video itself, leaving the
34
- * fill, stroke, and children intact.
35
- */
36
- @initial(1)
37
- @signal()
38
- public declare readonly alpha: SimpleSignal<number, this>;
39
-
40
- /**
41
- * Whether the video should be smoothed.
42
- *
43
- * @remarks
44
- * When disabled, the video will be scaled using the nearest neighbor
45
- * interpolation with no smoothing. The resulting video will appear pixelated.
46
- *
47
- * @defaultValue true
48
- */
49
- @initial(true)
50
- @signal()
51
- public declare readonly smoothing: SimpleSignal<boolean, this>;
52
-
53
- /**
54
- * Which decoder to use during rendering. The `web` decoder is the fastest
55
- * but only supports MP4 files. The `ffmpeg` decoder is slower and more resource
56
- * intensive but supports more formats. The `slow` decoder is the slowest but
57
- * supports all formats.
58
- *
59
- * @defaultValue null
60
- */
61
- @initial(null)
62
- @signal()
63
- public declare readonly decoder: SimpleSignal<
64
- 'web' | 'ffmpeg' | 'slow' | null,
65
- this
66
- >;
67
-
68
- public detectedFileType: 'mp4' | 'webm' | 'hls' | 'mov' | 'unknown' =
69
- 'unknown';
70
- private fileTypeWasDetected: boolean = false;
71
-
72
- private static readonly pool: Record<string, HTMLVideoElement> = {};
73
-
74
- private static readonly imageCommunication = !import.meta.hot
75
- ? null
76
- : new ImageCommunication();
77
-
78
- public constructor(props: VideoProps) {
79
- super(props);
80
- }
81
-
82
- protected override desiredSize(): SerializedVector2<DesiredLength> {
83
- const custom = super.desiredSize();
84
- if (custom.x === null && custom.y === null) {
85
- const image = this.video();
86
- return {
87
- x: image.videoWidth,
88
- y: image.videoHeight,
89
- };
90
- }
91
-
92
- return custom;
93
- }
94
-
95
- protected mediaElement(): HTMLVideoElement {
96
- return this.video();
97
- }
98
-
99
- protected seekedMedia(): HTMLVideoElement {
100
- return this.seekedVideo();
101
- }
102
-
103
- protected fastSeekedMedia(): HTMLVideoElement {
104
- return this.fastSeekedVideo();
105
- }
106
-
107
- @computed()
108
- private video(): HTMLVideoElement {
109
- const src = this.src();
110
-
111
- // Use a temporary key for undefined src to avoid conflicts
112
- const key = `${this.key}/${src || 'pending'}`;
113
-
114
- let video = Video.pool[key];
115
- if (!video) {
116
- video = document.createElement('video');
117
- video.crossOrigin = 'anonymous';
118
-
119
- // Only set src if it's valid, otherwise leave it empty
120
- if (src && src !== 'undefined') {
121
- try {
122
- const parsedSrc = new URL(src, window.location.origin);
123
-
124
- if (parsedSrc.pathname.endsWith('.m3u8')) {
125
- const hls = new Hls();
126
- hls.loadSource(src);
127
- hls.attachMedia(video);
128
- } else {
129
- video.src = src;
130
- }
131
- } catch (error) {
132
- // Fallback to direct assignment
133
- video.src = src;
134
- }
135
- }
136
-
137
- Video.pool[key] = video;
138
- } else if (src && src !== 'undefined' && video.src !== src) {
139
- // Update existing video element if src has changed and is now valid
140
- try {
141
- const parsedSrc = new URL(src, window.location.origin);
142
-
143
- if (parsedSrc.pathname.endsWith('.m3u8')) {
144
- const hls = new Hls();
145
- hls.loadSource(src);
146
- hls.attachMedia(video);
147
- } else {
148
- video.src = src;
149
- }
150
- } catch (error) {
151
- // Fallback to direct assignment
152
- video.src = src;
153
- }
154
-
155
- // Move video to correct pool key
156
- delete Video.pool[key];
157
- const newKey = `${this.key}/${src}`;
158
- Video.pool[newKey] = video;
159
- }
160
-
161
- // If src is still undefined, wait for it to become available
162
- if (!src || src === 'undefined') {
163
- DependencyContext.collectPromise(
164
- new Promise<void>(resolve => {
165
- // Check periodically for valid src
166
- const checkSrc = () => {
167
- const currentSrc = this.src();
168
- if (currentSrc && currentSrc !== 'undefined') {
169
- resolve();
170
- } else {
171
- setTimeout(checkSrc, 10);
172
- }
173
- };
174
- checkSrc();
175
- }),
176
- );
177
- }
178
-
179
- const weNeedToWait = this.waitForCanPlayNecessary(video);
180
-
181
- if (!weNeedToWait) {
182
- return video;
183
- }
184
-
185
- DependencyContext.collectPromise(
186
- new Promise<void>(resolve => {
187
- this.waitForCanPlay(video, resolve);
188
- }),
189
- );
190
-
191
- return video;
192
- }
193
-
194
- @computed()
195
- protected seekedVideo(): HTMLVideoElement {
196
- const video = this.video();
197
- const time = this.clampTime(this.time());
198
-
199
- video.playbackRate = this.playbackRate();
200
-
201
- if (!video.paused) {
202
- video.pause();
203
- }
204
-
205
- if (this.lastTime === time) {
206
- return video;
207
- }
208
-
209
- this.setCurrentTime(time);
210
-
211
- return video;
212
- }
213
-
214
- @computed()
215
- protected fastSeekedVideo(): HTMLVideoElement {
216
- const video = this.video();
217
- const time = this.clampTime(this.time());
218
-
219
- video.playbackRate = this.playbackRate();
220
-
221
- if (this.lastTime === time) {
222
- return video;
223
- }
224
-
225
- const playing =
226
- this.playing() && time < video.duration && video.playbackRate > 0;
227
-
228
- if (playing) {
229
- if (video.paused) {
230
- DependencyContext.collectPromise(video.play());
231
- }
232
- } else {
233
- if (!video.paused) {
234
- video.pause();
235
- }
236
- }
237
-
238
- // reseek when video is out of sync by more than one second
239
- if (Math.abs(video.currentTime - time) > 1) {
240
- this.setCurrentTime(time);
241
- } else if (!playing) {
242
- video.currentTime = time;
243
- }
244
-
245
- return video;
246
- }
247
-
248
- protected lastFrame: ImageBitmap | null = null;
249
-
250
- protected async webcodecSeekedVideo(): Promise<CanvasImageSource> {
251
- const video = this.video();
252
- const time = this.clampTime(this.time());
253
-
254
- video.playbackRate = this.playbackRate();
255
-
256
- if (this.lastFrame && this.lastTime === time) {
257
- return this.lastFrame;
258
- }
259
-
260
- const fps = this.view().fps() / this.playbackRate();
261
- return getFrame(this.key, video.src, time, fps);
262
- }
263
-
264
- protected async ffmpegSeekedVideo(): Promise<ImageBitmap> {
265
- const video = this.video();
266
- const time = this.clampTime(this.time());
267
- const duration = this.getDuration();
268
-
269
- video.playbackRate = this.playbackRate();
270
-
271
- if (this.lastFrame && this.lastTime === time) {
272
- return this.lastFrame;
273
- }
274
-
275
- const fps = this.view().fps() / this.playbackRate();
276
-
277
- if (!Video.imageCommunication) {
278
- throw new Error('ServerSeekedVideo can only be used with HMR.');
279
- }
280
-
281
- const frame = await Video.imageCommunication.getFrame(
282
- this.key,
283
- video.src,
284
- time,
285
- duration,
286
- fps,
287
- );
288
- this.lastFrame = frame;
289
- this.lastTime = time;
290
-
291
- return frame;
292
- }
293
-
294
- protected async seekFunction() {
295
- const playbackState = this.view().playbackState();
296
-
297
- // During playback
298
- if (
299
- playbackState === PlaybackState.Playing ||
300
- playbackState === PlaybackState.Presenting
301
- ) {
302
- return this.fastSeekedVideo();
303
- }
304
-
305
- if (playbackState === PlaybackState.Paused) {
306
- return this.seekedVideo();
307
- }
308
-
309
- // During rendering, if set explicitly
310
- if (this.decoder() === 'slow') {
311
- return this.seekedVideo();
312
- }
313
-
314
- if (this.decoder() === 'ffmpeg') {
315
- return this.ffmpegSeekedVideo();
316
- }
317
-
318
- if (this.decoder() === 'web') {
319
- return this.webcodecSeekedVideo();
320
- }
321
-
322
- if (!this.fileTypeWasDetected) {
323
- this.detectFileType();
324
- }
325
-
326
- // If not set explicitly, use detected file type to determine decoder
327
- if (this.detectedFileType === 'webm') {
328
- return this.ffmpegSeekedVideo();
329
- }
330
-
331
- if (this.detectedFileType === 'hls') {
332
- return this.seekedVideo();
333
- }
334
-
335
- return this.webcodecSeekedVideo();
336
- }
337
-
338
- protected override async draw(context: CanvasRenderingContext2D) {
339
- // Auto-start playback if Revideo is playing but media isn't
340
- this.autoPlayBasedOnRevideo();
341
-
342
- this.drawShape(context);
343
- const alpha = this.alpha();
344
- if (alpha > 0) {
345
- const video = await this.seekFunction();
346
-
347
- const box = BBox.fromSizeCentered(this.computedSize());
348
- context.save();
349
- context.clip(this.getPath());
350
- if (alpha < 1) {
351
- context.globalAlpha *= alpha;
352
- }
353
- context.imageSmoothingEnabled = this.smoothing();
354
- drawImage(context, video, box);
355
- context.restore();
356
- }
357
-
358
- if (this.clip()) {
359
- context.clip(this.getPath());
360
- }
361
-
362
- await this.drawChildren(context);
363
- }
364
-
365
- protected override applyFlex() {
366
- super.applyFlex();
367
- try {
368
- const video = this.video();
369
- // Only set aspect ratio if video element is available and has valid dimensions
370
- if (video && video.videoWidth > 0 && video.videoHeight > 0) {
371
- this.element.style.aspectRatio = (
372
- this.ratio() ?? video.videoWidth / video.videoHeight
373
- ).toString();
374
- }
375
- } catch (error) {
376
- // If video element is not ready yet, skip setting aspect ratio
377
- // It will be set later when the video becomes available
378
- }
379
- }
380
-
381
- public override remove() {
382
- super.remove();
383
- dropExtractor(this.key, this.src());
384
- return this;
385
- }
386
-
387
- private handleUnknownFileType(src: string) {
388
- console.warn(
389
- `WARNING: Could not detect file type of video (${src}), will default to using mp4 decoder. If your video file is not an mp4 file, this will lead to an error - to fix this, reencode your video as an mp4 file (better performance) or specify a different decoder: https://docs.re.video/common-issues/slow-rendering#use-mp4-decoder`,
390
- );
391
- this.detectedFileType = 'unknown';
392
- this.fileTypeWasDetected = true;
393
- }
394
-
395
- private detectFileType() {
396
- return DependencyContext.collectPromise(
397
- (async () => {
398
- const src = this.src();
399
- const extension = src.split('?')[0].split('.').pop()?.toLowerCase();
400
-
401
- if (
402
- extension === 'mp4' ||
403
- extension === 'webm' ||
404
- extension === 'mov'
405
- ) {
406
- this.detectedFileType = extension;
407
- this.fileTypeWasDetected = true;
408
- return;
409
- }
410
-
411
- if (extension === 'm3u8') {
412
- this.detectedFileType = 'hls';
413
- this.fileTypeWasDetected = true;
414
- return;
415
- }
416
-
417
- if (!src.startsWith('http://') && !src.startsWith('https://')) {
418
- this.handleUnknownFileType(src);
419
- return;
420
- }
421
-
422
- const response = await fetch(src, {method: 'HEAD'});
423
- const contentType = response.headers.get('Content-Type');
424
-
425
- if (!contentType) {
426
- this.handleUnknownFileType(src);
427
- return;
428
- }
429
-
430
- if (contentType.includes('video/mp4')) {
431
- this.detectedFileType = 'mp4';
432
- this.fileTypeWasDetected = true;
433
- return;
434
- }
435
-
436
- if (contentType.includes('video/webm')) {
437
- this.detectedFileType = 'webm';
438
- this.fileTypeWasDetected = true;
439
- return;
440
- }
441
-
442
- if (contentType.includes('video/quicktime')) {
443
- this.detectedFileType = 'mov';
444
- this.fileTypeWasDetected = true;
445
- return;
446
- }
447
-
448
- if (
449
- contentType.includes('application/vnd.apple.mpegurl') ||
450
- contentType.includes('application/x-mpegURL')
451
- ) {
452
- this.detectedFileType = 'hls';
453
- this.fileTypeWasDetected = true;
454
- return;
455
- }
456
-
457
- this.handleUnknownFileType(src);
458
- })(),
459
- );
460
- }
461
- }
1
+ import type {SerializedVector2, SignalValue, SimpleSignal} from '@twick/core';
2
+ import {BBox, DependencyContext, PlaybackState} from '@twick/core';
3
+ import Hls from 'hls.js';
4
+ import {computed, initial, nodeName, signal} from '../decorators';
5
+ import type {DesiredLength} from '../partials';
6
+ import {drawImage} from '../utils';
7
+ import {ImageCommunication} from '../utils/video/ffmpeg-client';
8
+ import {dropExtractor, getFrame} from '../utils/video/mp4-parser-manager';
9
+ import type {MediaProps} from './Media';
10
+ import {Media} from './Media';
11
+
12
+ export interface VideoProps extends MediaProps {
13
+ /**
14
+ * {@inheritDoc Video.alpha}
15
+ */
16
+ alpha?: SignalValue<number>;
17
+ /**
18
+ * {@inheritDoc Video.smoothing}
19
+ */
20
+ smoothing?: SignalValue<boolean>;
21
+ /**
22
+ * {@inheritDoc Video.decoder}
23
+ */
24
+ decoder?: SignalValue<'web' | 'ffmpeg' | 'slow' | null>;
25
+ }
26
+
27
+ @nodeName('Video')
28
+ export class Video extends Media {
29
+ /**
30
+ * The alpha value of this video.
31
+ *
32
+ * @remarks
33
+ * Unlike opacity, the alpha value affects only the video itself, leaving the
34
+ * fill, stroke, and children intact.
35
+ */
36
+ @initial(1)
37
+ @signal()
38
+ public declare readonly alpha: SimpleSignal<number, this>;
39
+
40
+ /**
41
+ * Whether the video should be smoothed.
42
+ *
43
+ * @remarks
44
+ * When disabled, the video will be scaled using the nearest neighbor
45
+ * interpolation with no smoothing. The resulting video will appear pixelated.
46
+ *
47
+ * @defaultValue true
48
+ */
49
+ @initial(true)
50
+ @signal()
51
+ public declare readonly smoothing: SimpleSignal<boolean, this>;
52
+
53
+ /**
54
+ * Which decoder to use during rendering. The `web` decoder is the fastest
55
+ * but only supports MP4 files. The `ffmpeg` decoder is slower and more resource
56
+ * intensive but supports more formats. The `slow` decoder is the slowest but
57
+ * supports all formats.
58
+ *
59
+ * @defaultValue null
60
+ */
61
+ @initial(null)
62
+ @signal()
63
+ public declare readonly decoder: SimpleSignal<
64
+ 'web' | 'ffmpeg' | 'slow' | null,
65
+ this
66
+ >;
67
+
68
+ public detectedFileType: 'mp4' | 'webm' | 'hls' | 'mov' | 'unknown' =
69
+ 'unknown';
70
+ private fileTypeWasDetected: boolean = false;
71
+
72
+ private static readonly pool: Record<string, HTMLVideoElement> = {};
73
+
74
+ private static readonly imageCommunication = !import.meta.hot
75
+ ? null
76
+ : new ImageCommunication();
77
+
78
+ public constructor(props: VideoProps) {
79
+ super(props);
80
+ }
81
+
82
+ protected override desiredSize(): SerializedVector2<DesiredLength> {
83
+ const custom = super.desiredSize();
84
+ if (custom.x === null && custom.y === null) {
85
+ const image = this.video();
86
+ return {
87
+ x: image.videoWidth,
88
+ y: image.videoHeight,
89
+ };
90
+ }
91
+
92
+ return custom;
93
+ }
94
+
95
+ protected mediaElement(): HTMLVideoElement {
96
+ return this.video();
97
+ }
98
+
99
+ protected seekedMedia(): HTMLVideoElement {
100
+ return this.seekedVideo();
101
+ }
102
+
103
+ protected fastSeekedMedia(): HTMLVideoElement {
104
+ return this.fastSeekedVideo();
105
+ }
106
+
107
+ @computed()
108
+ private video(): HTMLVideoElement {
109
+ const src = this.src();
110
+
111
+ // Use a temporary key for undefined src to avoid conflicts
112
+ const key = `${this.key}/${src || 'pending'}`;
113
+
114
+ let video = Video.pool[key];
115
+ if (!video) {
116
+ video = document.createElement('video');
117
+ video.crossOrigin = 'anonymous';
118
+
119
+ // Only set src if it's valid, otherwise leave it empty
120
+ if (src && src !== 'undefined') {
121
+ try {
122
+ const parsedSrc = new URL(src, window.location.origin);
123
+
124
+ if (parsedSrc.pathname.endsWith('.m3u8')) {
125
+ const hls = new Hls();
126
+ hls.loadSource(src);
127
+ hls.attachMedia(video);
128
+ } else {
129
+ video.src = src;
130
+ }
131
+ } catch (error) {
132
+ // Fallback to direct assignment
133
+ video.src = src;
134
+ }
135
+ }
136
+
137
+ Video.pool[key] = video;
138
+ } else if (src && src !== 'undefined' && video.src !== src) {
139
+ // Update existing video element if src has changed and is now valid
140
+ try {
141
+ const parsedSrc = new URL(src, window.location.origin);
142
+
143
+ if (parsedSrc.pathname.endsWith('.m3u8')) {
144
+ const hls = new Hls();
145
+ hls.loadSource(src);
146
+ hls.attachMedia(video);
147
+ } else {
148
+ video.src = src;
149
+ }
150
+ } catch (error) {
151
+ // Fallback to direct assignment
152
+ video.src = src;
153
+ }
154
+
155
+ // Move video to correct pool key
156
+ delete Video.pool[key];
157
+ const newKey = `${this.key}/${src}`;
158
+ Video.pool[newKey] = video;
159
+ }
160
+
161
+ // If src is still undefined, wait for it to become available
162
+ if (!src || src === 'undefined') {
163
+ DependencyContext.collectPromise(
164
+ new Promise<void>(resolve => {
165
+ // Check periodically for valid src
166
+ const checkSrc = () => {
167
+ const currentSrc = this.src();
168
+ if (currentSrc && currentSrc !== 'undefined') {
169
+ resolve();
170
+ } else {
171
+ setTimeout(checkSrc, 10);
172
+ }
173
+ };
174
+ checkSrc();
175
+ }),
176
+ );
177
+ }
178
+
179
+ const weNeedToWait = this.waitForCanPlayNecessary(video);
180
+
181
+ if (!weNeedToWait) {
182
+ return video;
183
+ }
184
+
185
+ DependencyContext.collectPromise(
186
+ new Promise<void>(resolve => {
187
+ this.waitForCanPlay(video, resolve);
188
+ }),
189
+ );
190
+
191
+ return video;
192
+ }
193
+
194
+ @computed()
195
+ protected seekedVideo(): HTMLVideoElement {
196
+ const video = this.video();
197
+ const time = this.clampTime(this.time());
198
+
199
+ video.playbackRate = this.playbackRate();
200
+
201
+ if (!video.paused) {
202
+ video.pause();
203
+ }
204
+
205
+ if (this.lastTime === time) {
206
+ return video;
207
+ }
208
+
209
+ this.setCurrentTime(time);
210
+
211
+ return video;
212
+ }
213
+
214
+ @computed()
215
+ protected fastSeekedVideo(): HTMLVideoElement {
216
+ const video = this.video();
217
+ const time = this.clampTime(this.time());
218
+
219
+ video.playbackRate = this.playbackRate();
220
+
221
+ if (this.lastTime === time) {
222
+ return video;
223
+ }
224
+
225
+ const playing =
226
+ this.playing() && time < video.duration && video.playbackRate > 0;
227
+
228
+ if (playing) {
229
+ if (video.paused) {
230
+ DependencyContext.collectPromise(video.play());
231
+ }
232
+ } else {
233
+ if (!video.paused) {
234
+ video.pause();
235
+ }
236
+ }
237
+
238
+ // reseek when video is out of sync by more than one second
239
+ if (Math.abs(video.currentTime - time) > 1) {
240
+ this.setCurrentTime(time);
241
+ } else if (!playing) {
242
+ video.currentTime = time;
243
+ }
244
+
245
+ return video;
246
+ }
247
+
248
+ protected lastFrame: ImageBitmap | null = null;
249
+
250
+ protected async webcodecSeekedVideo(): Promise<CanvasImageSource> {
251
+ const video = this.video();
252
+ const time = this.clampTime(this.time());
253
+
254
+ video.playbackRate = this.playbackRate();
255
+
256
+ if (this.lastFrame && this.lastTime === time) {
257
+ return this.lastFrame;
258
+ }
259
+
260
+ const fps = this.view().fps() / this.playbackRate();
261
+ return getFrame(this.key, video.src, time, fps);
262
+ }
263
+
264
+ protected async ffmpegSeekedVideo(): Promise<ImageBitmap> {
265
+ const video = this.video();
266
+ const time = this.clampTime(this.time());
267
+ const duration = this.getDuration();
268
+
269
+ video.playbackRate = this.playbackRate();
270
+
271
+ if (this.lastFrame && this.lastTime === time) {
272
+ return this.lastFrame;
273
+ }
274
+
275
+ const fps = this.view().fps() / this.playbackRate();
276
+
277
+ if (!Video.imageCommunication) {
278
+ throw new Error('ServerSeekedVideo can only be used with HMR.');
279
+ }
280
+
281
+ const frame = await Video.imageCommunication.getFrame(
282
+ this.key,
283
+ video.src,
284
+ time,
285
+ duration,
286
+ fps,
287
+ );
288
+ this.lastFrame = frame;
289
+ this.lastTime = time;
290
+
291
+ return frame;
292
+ }
293
+
294
+ protected async seekFunction() {
295
+ const playbackState = this.view().playbackState();
296
+
297
+ // During playback
298
+ if (
299
+ playbackState === PlaybackState.Playing ||
300
+ playbackState === PlaybackState.Presenting
301
+ ) {
302
+ return this.fastSeekedVideo();
303
+ }
304
+
305
+ if (playbackState === PlaybackState.Paused) {
306
+ return this.seekedVideo();
307
+ }
308
+
309
+ // During rendering, if set explicitly
310
+ if (this.decoder() === 'slow') {
311
+ return this.seekedVideo();
312
+ }
313
+
314
+ if (this.decoder() === 'ffmpeg') {
315
+ return this.ffmpegSeekedVideo();
316
+ }
317
+
318
+ if (this.decoder() === 'web') {
319
+ return this.webcodecSeekedVideo();
320
+ }
321
+
322
+ if (!this.fileTypeWasDetected) {
323
+ this.detectFileType();
324
+ }
325
+
326
+ // If not set explicitly, use detected file type to determine decoder
327
+ if (this.detectedFileType === 'webm') {
328
+ return this.ffmpegSeekedVideo();
329
+ }
330
+
331
+ if (this.detectedFileType === 'hls') {
332
+ return this.seekedVideo();
333
+ }
334
+
335
+ return this.webcodecSeekedVideo();
336
+ }
337
+
338
+ protected override async draw(context: CanvasRenderingContext2D) {
339
+ // Auto-start playback if Twick is playing but media isn't
340
+ this.autoPlayBasedOnTwick();
341
+
342
+ this.drawShape(context);
343
+ const alpha = this.alpha();
344
+ if (alpha > 0) {
345
+ const video = await this.seekFunction();
346
+
347
+ const box = BBox.fromSizeCentered(this.computedSize());
348
+ context.save();
349
+ context.clip(this.getPath());
350
+ if (alpha < 1) {
351
+ context.globalAlpha *= alpha;
352
+ }
353
+ context.imageSmoothingEnabled = this.smoothing();
354
+ drawImage(context, video, box);
355
+ context.restore();
356
+ }
357
+
358
+ if (this.clip()) {
359
+ context.clip(this.getPath());
360
+ }
361
+
362
+ await this.drawChildren(context);
363
+ }
364
+
365
+ protected override applyFlex() {
366
+ super.applyFlex();
367
+ try {
368
+ const video = this.video();
369
+ // Only set aspect ratio if video element is available and has valid dimensions
370
+ if (video && video.videoWidth > 0 && video.videoHeight > 0) {
371
+ this.element.style.aspectRatio = (
372
+ this.ratio() ?? video.videoWidth / video.videoHeight
373
+ ).toString();
374
+ }
375
+ } catch (error) {
376
+ // If video element is not ready yet, skip setting aspect ratio
377
+ // It will be set later when the video becomes available
378
+ }
379
+ }
380
+
381
+ public override remove() {
382
+ super.remove();
383
+ dropExtractor(this.key, this.src());
384
+ return this;
385
+ }
386
+
387
+ private handleUnknownFileType(src: string) {
388
+ console.warn(
389
+ `WARNING: Could not detect file type of video (${src}), will default to using mp4 decoder. If your video file is not an mp4 file, this will lead to an error - to fix this, reencode your video as an mp4 file (better performance) or specify a different decoder: https://docs.re.video/common-issues/slow-rendering#use-mp4-decoder`,
390
+ );
391
+ this.detectedFileType = 'unknown';
392
+ this.fileTypeWasDetected = true;
393
+ }
394
+
395
+ private detectFileType() {
396
+ return DependencyContext.collectPromise(
397
+ (async () => {
398
+ const src = this.src();
399
+ const extension = src.split('?')[0].split('.').pop()?.toLowerCase();
400
+
401
+ if (
402
+ extension === 'mp4' ||
403
+ extension === 'webm' ||
404
+ extension === 'mov'
405
+ ) {
406
+ this.detectedFileType = extension;
407
+ this.fileTypeWasDetected = true;
408
+ return;
409
+ }
410
+
411
+ if (extension === 'm3u8') {
412
+ this.detectedFileType = 'hls';
413
+ this.fileTypeWasDetected = true;
414
+ return;
415
+ }
416
+
417
+ if (!src.startsWith('http://') && !src.startsWith('https://')) {
418
+ this.handleUnknownFileType(src);
419
+ return;
420
+ }
421
+
422
+ const response = await fetch(src, {method: 'HEAD'});
423
+ const contentType = response.headers.get('Content-Type');
424
+
425
+ if (!contentType) {
426
+ this.handleUnknownFileType(src);
427
+ return;
428
+ }
429
+
430
+ if (contentType.includes('video/mp4')) {
431
+ this.detectedFileType = 'mp4';
432
+ this.fileTypeWasDetected = true;
433
+ return;
434
+ }
435
+
436
+ if (contentType.includes('video/webm')) {
437
+ this.detectedFileType = 'webm';
438
+ this.fileTypeWasDetected = true;
439
+ return;
440
+ }
441
+
442
+ if (contentType.includes('video/quicktime')) {
443
+ this.detectedFileType = 'mov';
444
+ this.fileTypeWasDetected = true;
445
+ return;
446
+ }
447
+
448
+ if (
449
+ contentType.includes('application/vnd.apple.mpegurl') ||
450
+ contentType.includes('application/x-mpegURL')
451
+ ) {
452
+ this.detectedFileType = 'hls';
453
+ this.fileTypeWasDetected = true;
454
+ return;
455
+ }
456
+
457
+ this.handleUnknownFileType(src);
458
+ })(),
459
+ );
460
+ }
461
+ }