@tscaps/engine 0.3.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +35 -8
- package/dist/modules/document/CssVariable.d.ts +6 -0
- package/dist/modules/document/CssVariable.d.ts.map +1 -1
- package/dist/modules/document/CssVariable.js +6 -0
- package/dist/modules/document/CssVariable.js.map +1 -1
- package/dist/modules/document/Document.d.ts +3 -0
- package/dist/modules/document/Document.d.ts.map +1 -1
- package/dist/modules/document/Document.js +12 -1
- package/dist/modules/document/Document.js.map +1 -1
- package/dist/modules/document/DocumentEditor.d.ts +16 -4
- package/dist/modules/document/DocumentEditor.d.ts.map +1 -1
- package/dist/modules/document/DocumentEditor.js +14 -10
- package/dist/modules/document/DocumentEditor.js.map +1 -1
- package/dist/modules/document/Line.d.ts +6 -0
- package/dist/modules/document/Line.d.ts.map +1 -1
- package/dist/modules/document/Line.js +10 -0
- package/dist/modules/document/Line.js.map +1 -1
- package/dist/modules/document/Word.d.ts +8 -0
- package/dist/modules/document/Word.d.ts.map +1 -1
- package/dist/modules/document/Word.js +11 -1
- package/dist/modules/document/Word.js.map +1 -1
- package/dist/modules/pipeline/RenderPipelineBuilder.d.ts +1 -1
- package/dist/modules/pipeline/RenderPipelineBuilder.d.ts.map +1 -1
- package/dist/modules/pipeline/RenderPipelineBuilder.js +15 -5
- package/dist/modules/pipeline/RenderPipelineBuilder.js.map +1 -1
- package/dist/modules/rendering/BrowserSubtitleFrameRenderer.d.ts +18 -8
- package/dist/modules/rendering/BrowserSubtitleFrameRenderer.d.ts.map +1 -1
- package/dist/modules/rendering/BrowserSubtitleFrameRenderer.js +27 -21
- package/dist/modules/rendering/BrowserSubtitleFrameRenderer.js.map +1 -1
- package/dist/modules/rendering/SubtitleFrameRenderer.d.ts +41 -33
- package/dist/modules/rendering/SubtitleFrameRenderer.d.ts.map +1 -1
- package/dist/modules/rendering/index.d.ts +4 -0
- package/dist/modules/rendering/index.d.ts.map +1 -1
- package/dist/modules/rendering/index.js +2 -0
- package/dist/modules/rendering/index.js.map +1 -1
- package/dist/modules/rendering/subtitle/ActiveRenderSession.d.ts +13 -6
- package/dist/modules/rendering/subtitle/ActiveRenderSession.d.ts.map +1 -1
- package/dist/modules/rendering/subtitle/ActiveRenderSession.js +16 -9
- package/dist/modules/rendering/subtitle/ActiveRenderSession.js.map +1 -1
- package/dist/modules/rendering/subtitle/ActiveRenderSessionFactory.d.ts +5 -2
- package/dist/modules/rendering/subtitle/ActiveRenderSessionFactory.d.ts.map +1 -1
- package/dist/modules/rendering/subtitle/ActiveRenderSessionFactory.js +26 -8
- package/dist/modules/rendering/subtitle/ActiveRenderSessionFactory.js.map +1 -1
- package/dist/modules/rendering/subtitle/AnimationStateFingerprint.d.ts +20 -0
- package/dist/modules/rendering/subtitle/AnimationStateFingerprint.d.ts.map +1 -0
- package/dist/modules/rendering/subtitle/AnimationStateFingerprint.js +2 -0
- package/dist/modules/rendering/subtitle/AnimationStateFingerprint.js.map +1 -0
- package/dist/modules/rendering/subtitle/AssetGroupBuilder.d.ts +4 -0
- package/dist/modules/rendering/subtitle/AssetGroupBuilder.d.ts.map +1 -1
- package/dist/modules/rendering/subtitle/AssetGroupBuilder.js +8 -0
- package/dist/modules/rendering/subtitle/AssetGroupBuilder.js.map +1 -1
- package/dist/modules/rendering/subtitle/BatchPlanner.d.ts +50 -5
- package/dist/modules/rendering/subtitle/BatchPlanner.d.ts.map +1 -1
- package/dist/modules/rendering/subtitle/BatchPlanner.js +97 -15
- package/dist/modules/rendering/subtitle/BatchPlanner.js.map +1 -1
- package/dist/modules/rendering/subtitle/ElementWidthMeasurer.d.ts +56 -0
- package/dist/modules/rendering/subtitle/ElementWidthMeasurer.d.ts.map +1 -0
- package/dist/modules/rendering/subtitle/ElementWidthMeasurer.js +119 -0
- package/dist/modules/rendering/subtitle/ElementWidthMeasurer.js.map +1 -0
- package/dist/modules/rendering/subtitle/ElementWidths.d.ts +23 -0
- package/dist/modules/rendering/subtitle/ElementWidths.d.ts.map +1 -0
- package/dist/modules/rendering/subtitle/ElementWidths.js +29 -0
- package/dist/modules/rendering/subtitle/ElementWidths.js.map +1 -0
- package/dist/modules/rendering/subtitle/ImageDecodeSpriteSheetRasterProbe.d.ts +18 -0
- package/dist/modules/rendering/subtitle/ImageDecodeSpriteSheetRasterProbe.d.ts.map +1 -0
- package/dist/modules/rendering/subtitle/ImageDecodeSpriteSheetRasterProbe.js +42 -0
- package/dist/modules/rendering/subtitle/ImageDecodeSpriteSheetRasterProbe.js.map +1 -0
- package/dist/modules/rendering/subtitle/KeyframeScanAnimationStateFingerprint.d.ts +35 -0
- package/dist/modules/rendering/subtitle/KeyframeScanAnimationStateFingerprint.d.ts.map +1 -0
- package/dist/modules/rendering/subtitle/KeyframeScanAnimationStateFingerprint.js +64 -0
- package/dist/modules/rendering/subtitle/KeyframeScanAnimationStateFingerprint.js.map +1 -0
- package/dist/modules/rendering/subtitle/PreparedStyle.d.ts +2 -0
- package/dist/modules/rendering/subtitle/PreparedStyle.d.ts.map +1 -1
- package/dist/modules/rendering/subtitle/PreparedStyleFactory.d.ts.map +1 -1
- package/dist/modules/rendering/subtitle/PreparedStyleFactory.js +12 -1
- package/dist/modules/rendering/subtitle/PreparedStyleFactory.js.map +1 -1
- package/dist/modules/rendering/subtitle/SegmentAnchorVarsBuilder.d.ts +28 -0
- package/dist/modules/rendering/subtitle/SegmentAnchorVarsBuilder.d.ts.map +1 -0
- package/dist/modules/rendering/subtitle/SegmentAnchorVarsBuilder.js +23 -0
- package/dist/modules/rendering/subtitle/SegmentAnchorVarsBuilder.js.map +1 -0
- package/dist/modules/rendering/subtitle/SegmentSubtreeHtmlBuilder.d.ts +3 -0
- package/dist/modules/rendering/subtitle/SegmentSubtreeHtmlBuilder.d.ts.map +1 -1
- package/dist/modules/rendering/subtitle/SegmentSubtreeHtmlBuilder.js +12 -3
- package/dist/modules/rendering/subtitle/SegmentSubtreeHtmlBuilder.js.map +1 -1
- package/dist/modules/rendering/subtitle/SegmentWrapperRenderer.d.ts +7 -1
- package/dist/modules/rendering/subtitle/SegmentWrapperRenderer.d.ts.map +1 -1
- package/dist/modules/rendering/subtitle/SegmentWrapperRenderer.js +23 -5
- package/dist/modules/rendering/subtitle/SegmentWrapperRenderer.js.map +1 -1
- package/dist/modules/rendering/subtitle/SpriteSheetProbeObserver.d.ts +29 -0
- package/dist/modules/rendering/subtitle/SpriteSheetProbeObserver.d.ts.map +1 -0
- package/dist/modules/rendering/subtitle/SpriteSheetProbeObserver.js +2 -0
- package/dist/modules/rendering/subtitle/SpriteSheetProbeObserver.js.map +1 -0
- package/dist/modules/rendering/subtitle/SpriteSheetRasterProbe.d.ts +19 -0
- package/dist/modules/rendering/subtitle/SpriteSheetRasterProbe.d.ts.map +1 -0
- package/dist/modules/rendering/subtitle/SpriteSheetRasterProbe.js +2 -0
- package/dist/modules/rendering/subtitle/SpriteSheetRasterProbe.js.map +1 -0
- package/dist/modules/rendering/subtitle/SpriteSheetSizeProbe.d.ts +31 -0
- package/dist/modules/rendering/subtitle/SpriteSheetSizeProbe.d.ts.map +1 -0
- package/dist/modules/rendering/subtitle/SpriteSheetSizeProbe.js +89 -0
- package/dist/modules/rendering/subtitle/SpriteSheetSizeProbe.js.map +1 -0
- package/dist/modules/rendering/subtitle/SubtreeAnimationReading.d.ts +49 -0
- package/dist/modules/rendering/subtitle/SubtreeAnimationReading.d.ts.map +1 -0
- package/dist/modules/rendering/subtitle/SubtreeAnimationReading.js +87 -0
- package/dist/modules/rendering/subtitle/SubtreeAnimationReading.js.map +1 -0
- package/dist/modules/rendering/subtitle/SubtreeAnimationSupport.d.ts +17 -0
- package/dist/modules/rendering/subtitle/SubtreeAnimationSupport.d.ts.map +1 -0
- package/dist/modules/rendering/subtitle/SubtreeAnimationSupport.js +53 -0
- package/dist/modules/rendering/subtitle/SubtreeAnimationSupport.js.map +1 -0
- package/dist/modules/rendering/subtitle/SubtreeMountAnimationStateFingerprint.d.ts +62 -0
- package/dist/modules/rendering/subtitle/SubtreeMountAnimationStateFingerprint.d.ts.map +1 -0
- package/dist/modules/rendering/subtitle/SubtreeMountAnimationStateFingerprint.js +153 -0
- package/dist/modules/rendering/subtitle/SubtreeMountAnimationStateFingerprint.js.map +1 -0
- package/dist/modules/rendering/subtitle/SvgFilterStateFingerprint.d.ts +47 -0
- package/dist/modules/rendering/subtitle/SvgFilterStateFingerprint.d.ts.map +1 -0
- package/dist/modules/rendering/subtitle/SvgFilterStateFingerprint.js +69 -0
- package/dist/modules/rendering/subtitle/SvgFilterStateFingerprint.js.map +1 -0
- package/dist/modules/rendering/subtitle/UnknownAnimationStateFingerprint.d.ts +9 -0
- package/dist/modules/rendering/subtitle/UnknownAnimationStateFingerprint.d.ts.map +1 -0
- package/dist/modules/rendering/subtitle/UnknownAnimationStateFingerprint.js +10 -0
- package/dist/modules/rendering/subtitle/UnknownAnimationStateFingerprint.js.map +1 -0
- package/dist/modules/rendering/subtitle/VideoFrameVarsBuilder.d.ts.map +1 -1
- package/dist/modules/rendering/subtitle/VideoFrameVarsBuilder.js +2 -0
- package/dist/modules/rendering/subtitle/VideoFrameVarsBuilder.js.map +1 -1
- package/dist/modules/rendering/types/ElementRenderOverrides.d.ts +2 -0
- package/dist/modules/rendering/types/ElementRenderOverrides.d.ts.map +1 -1
- package/dist/modules/rendering/types/ElementRenderOverrides.js +4 -0
- package/dist/modules/rendering/types/ElementRenderOverrides.js.map +1 -1
- package/dist/modules/splitting/BalancedLineSplitter.d.ts +9 -0
- package/dist/modules/splitting/BalancedLineSplitter.d.ts.map +1 -1
- package/dist/modules/splitting/BalancedLineSplitter.js +15 -3
- package/dist/modules/splitting/BalancedLineSplitter.js.map +1 -1
- package/dist/modules/splitting/BalancedPixelWidthLineSplitter.d.ts +1 -4
- package/dist/modules/splitting/BalancedPixelWidthLineSplitter.d.ts.map +1 -1
- package/dist/modules/splitting/BalancedPixelWidthLineSplitter.js +5 -7
- package/dist/modules/splitting/BalancedPixelWidthLineSplitter.js.map +1 -1
- package/dist/modules/splitting/DomProbeCanvasTextMeasurer.d.ts +16 -7
- package/dist/modules/splitting/DomProbeCanvasTextMeasurer.d.ts.map +1 -1
- package/dist/modules/splitting/DomProbeCanvasTextMeasurer.js +44 -33
- package/dist/modules/splitting/DomProbeCanvasTextMeasurer.js.map +1 -1
- package/dist/modules/splitting/TextMeasurer.d.ts +9 -4
- package/dist/modules/splitting/TextMeasurer.d.ts.map +1 -1
- package/dist/modules/svg-filter/SvgFilter.d.ts +9 -0
- package/dist/modules/svg-filter/SvgFilter.d.ts.map +1 -1
- package/dist/modules/svg-filter/SvgFilter.js +17 -0
- package/dist/modules/svg-filter/SvgFilter.js.map +1 -1
- package/dist/modules/svg-filter/SvgFilterDefinitions.d.ts +7 -0
- package/dist/modules/svg-filter/SvgFilterDefinitions.d.ts.map +1 -1
- package/dist/modules/svg-filter/SvgFilterDefinitions.js +17 -0
- package/dist/modules/svg-filter/SvgFilterDefinitions.js.map +1 -1
- package/dist/modules/tagging/RegexTagger.js +1 -2
- package/dist/modules/tagging/RegexTagger.js.map +1 -1
- package/dist/modules/tagging/SpanTagger.js +1 -2
- package/dist/modules/tagging/SpanTagger.js.map +1 -1
- package/dist/modules/tagging/StructureTagger.js +1 -2
- package/dist/modules/tagging/StructureTagger.js.map +1 -1
- package/dist/modules/tagging/WordlistTagger.js +1 -2
- package/dist/modules/tagging/WordlistTagger.js.map +1 -1
- package/dist/modules/transcription/AudioSegmentLayout.d.ts +23 -0
- package/dist/modules/transcription/AudioSegmentLayout.d.ts.map +1 -0
- package/dist/modules/transcription/AudioSegmentLayout.js +40 -0
- package/dist/modules/transcription/AudioSegmentLayout.js.map +1 -0
- package/dist/modules/transcription/FallbackAudioDecoder.d.ts +25 -0
- package/dist/modules/transcription/FallbackAudioDecoder.d.ts.map +1 -0
- package/dist/modules/transcription/FallbackAudioDecoder.js +46 -0
- package/dist/modules/transcription/FallbackAudioDecoder.js.map +1 -0
- package/dist/modules/transcription/MediaBunnyAudioOnlyRemuxer.d.ts +71 -0
- package/dist/modules/transcription/MediaBunnyAudioOnlyRemuxer.d.ts.map +1 -0
- package/dist/modules/transcription/MediaBunnyAudioOnlyRemuxer.js +145 -0
- package/dist/modules/transcription/MediaBunnyAudioOnlyRemuxer.js.map +1 -0
- package/dist/modules/transcription/WebAudioAudioDecoder.d.ts +46 -0
- package/dist/modules/transcription/WebAudioAudioDecoder.d.ts.map +1 -0
- package/dist/modules/transcription/WebAudioAudioDecoder.js +123 -0
- package/dist/modules/transcription/WebAudioAudioDecoder.js.map +1 -0
- package/dist/modules/transcription/index.d.ts +4 -0
- package/dist/modules/transcription/index.d.ts.map +1 -1
- package/dist/modules/transcription/index.js +3 -0
- package/dist/modules/transcription/index.js.map +1 -1
- package/dist/modules/video/RenderJob.d.ts +23 -0
- package/dist/modules/video/RenderJob.d.ts.map +1 -1
- package/dist/modules/video/index.d.ts +5 -3
- package/dist/modules/video/index.d.ts.map +1 -1
- package/dist/modules/video/index.js +2 -0
- package/dist/modules/video/index.js.map +1 -1
- package/dist/modules/video/mediabunny/MediaBunnyTranscodeCoordinator.d.ts +33 -1
- package/dist/modules/video/mediabunny/MediaBunnyTranscodeCoordinator.d.ts.map +1 -1
- package/dist/modules/video/mediabunny/MediaBunnyTranscodeCoordinator.js +97 -20
- package/dist/modules/video/mediabunny/MediaBunnyTranscodeCoordinator.js.map +1 -1
- package/dist/modules/video/mediabunny/MediaBunnyVideoRenderer.d.ts.map +1 -1
- package/dist/modules/video/mediabunny/MediaBunnyVideoRenderer.js +1 -0
- package/dist/modules/video/mediabunny/MediaBunnyVideoRenderer.js.map +1 -1
- package/dist/modules/video/mediabunny/caption/BatchedSubtitleLayerSource.d.ts +20 -9
- package/dist/modules/video/mediabunny/caption/BatchedSubtitleLayerSource.d.ts.map +1 -1
- package/dist/modules/video/mediabunny/caption/BatchedSubtitleLayerSource.js +56 -14
- package/dist/modules/video/mediabunny/caption/BatchedSubtitleLayerSource.js.map +1 -1
- package/dist/modules/video/mediabunny/caption/ComposedSubtitleLayerSource.d.ts +29 -10
- package/dist/modules/video/mediabunny/caption/ComposedSubtitleLayerSource.d.ts.map +1 -1
- package/dist/modules/video/mediabunny/caption/ComposedSubtitleLayerSource.js +48 -17
- package/dist/modules/video/mediabunny/caption/ComposedSubtitleLayerSource.js.map +1 -1
- package/dist/modules/video/mediabunny/caption/SubtitleLayerSource.d.ts +28 -11
- package/dist/modules/video/mediabunny/caption/SubtitleLayerSource.d.ts.map +1 -1
- package/dist/modules/video/mediabunny/caption/VideoBoundSubtitleLayerSource.d.ts +33 -12
- package/dist/modules/video/mediabunny/caption/VideoBoundSubtitleLayerSource.d.ts.map +1 -1
- package/dist/modules/video/mediabunny/caption/VideoBoundSubtitleLayerSource.js +53 -18
- package/dist/modules/video/mediabunny/caption/VideoBoundSubtitleLayerSource.js.map +1 -1
- package/dist/modules/video/mediabunny/encoder/MediaBunnyCanvasVideoTrackEncoder.d.ts.map +1 -1
- package/dist/modules/video/mediabunny/encoder/MediaBunnyCanvasVideoTrackEncoder.js +7 -3
- package/dist/modules/video/mediabunny/encoder/MediaBunnyCanvasVideoTrackEncoder.js.map +1 -1
- package/dist/modules/video/mediabunny/frame/BufferedVideoFrameSource.d.ts +45 -0
- package/dist/modules/video/mediabunny/frame/BufferedVideoFrameSource.d.ts.map +1 -0
- package/dist/modules/video/mediabunny/frame/BufferedVideoFrameSource.js +93 -0
- package/dist/modules/video/mediabunny/frame/BufferedVideoFrameSource.js.map +1 -0
- package/dist/modules/video/mediabunny/frame/DefaultVideoFrameDecoderFactory.d.ts.map +1 -1
- package/dist/modules/video/mediabunny/frame/DefaultVideoFrameDecoderFactory.js +3 -1
- package/dist/modules/video/mediabunny/frame/DefaultVideoFrameDecoderFactory.js.map +1 -1
- package/dist/modules/video/mediabunny/frame/RetainedVideoFrame.d.ts +26 -0
- package/dist/modules/video/mediabunny/frame/RetainedVideoFrame.d.ts.map +1 -0
- package/dist/modules/video/mediabunny/frame/RetainedVideoFrame.js +55 -0
- package/dist/modules/video/mediabunny/frame/RetainedVideoFrame.js.map +1 -0
- package/dist/modules/video/mediabunny/frame/VideoFrameDecoderFactory.d.ts +6 -1
- package/dist/modules/video/mediabunny/frame/VideoFrameDecoderFactory.d.ts.map +1 -1
- package/dist/modules/video/mediabunny/frame/VideoFrameDecoderSelectionFailedError.d.ts +14 -0
- package/dist/modules/video/mediabunny/frame/VideoFrameDecoderSelectionFailedError.d.ts.map +1 -0
- package/dist/modules/video/mediabunny/frame/VideoFrameDecoderSelectionFailedError.js +14 -0
- package/dist/modules/video/mediabunny/frame/VideoFrameDecoderSelectionFailedError.js.map +1 -0
- package/dist/modules/video/mediabunny/frame/WebCodecsOnlyVideoFrameDecoderFactory.d.ts +16 -0
- package/dist/modules/video/mediabunny/frame/WebCodecsOnlyVideoFrameDecoderFactory.d.ts.map +1 -0
- package/dist/modules/video/mediabunny/frame/WebCodecsOnlyVideoFrameDecoderFactory.js +23 -0
- package/dist/modules/video/mediabunny/frame/WebCodecsOnlyVideoFrameDecoderFactory.js.map +1 -0
- package/dist/modules/video/mediabunny/painter/CaptionsOverlayFramePainter.d.ts +9 -3
- package/dist/modules/video/mediabunny/painter/CaptionsOverlayFramePainter.d.ts.map +1 -1
- package/dist/modules/video/mediabunny/painter/CaptionsOverlayFramePainter.js +24 -7
- package/dist/modules/video/mediabunny/painter/CaptionsOverlayFramePainter.js.map +1 -1
- package/dist/modules/video/mediabunny/painter/FramePainter.d.ts +35 -14
- package/dist/modules/video/mediabunny/painter/FramePainter.d.ts.map +1 -1
- package/package.json +4 -1
- package/dist/modules/rendering/subtitle/AbstractAnim.d.ts +0 -13
- package/dist/modules/rendering/subtitle/AbstractAnim.d.ts.map +0 -1
- package/dist/modules/rendering/subtitle/AbstractAnim.js +0 -2
- package/dist/modules/rendering/subtitle/AbstractAnim.js.map +0 -1
- package/dist/modules/rendering/subtitle/AnimationProbe.d.ts +0 -39
- package/dist/modules/rendering/subtitle/AnimationProbe.d.ts.map +0 -1
- package/dist/modules/rendering/subtitle/AnimationProbe.js +0 -203
- package/dist/modules/rendering/subtitle/AnimationProbe.js.map +0 -1
- package/dist/modules/rendering/subtitle/SpriteSheetSizeProber.d.ts +0 -17
- package/dist/modules/rendering/subtitle/SpriteSheetSizeProber.d.ts.map +0 -1
- package/dist/modules/rendering/subtitle/SpriteSheetSizeProber.js +0 -58
- package/dist/modules/rendering/subtitle/SpriteSheetSizeProber.js.map +0 -1
- package/dist/modules/video/mediabunny/frame/CurrentFrameVideoSource.d.ts +0 -27
- package/dist/modules/video/mediabunny/frame/CurrentFrameVideoSource.d.ts.map +0 -1
- package/dist/modules/video/mediabunny/frame/CurrentFrameVideoSource.js +0 -84
- package/dist/modules/video/mediabunny/frame/CurrentFrameVideoSource.js.map +0 -1
|
@@ -5,20 +5,33 @@ import { LayeredSubtitleFrame } from '../../../rendering/LayeredSubtitleFrame';
|
|
|
5
5
|
// sub-frame difference.
|
|
6
6
|
const CAPTION_BOUNDARY_EPSILON_S = 1e-6;
|
|
7
7
|
/**
|
|
8
|
-
* Composes two `SubtitleLayerSource`s, routing each Section to one
|
|
9
|
-
*
|
|
10
|
-
*
|
|
11
|
-
* `frameAt` call to the nearest caption tick and merges the rasters
|
|
12
|
-
* produced by sub-sources with active sections into a single frame.
|
|
8
|
+
* Composes two `SubtitleLayerSource`s, routing each Section to one of
|
|
9
|
+
* them by whether its style declares `rendering.videoFrame.required`,
|
|
10
|
+
* and merging what they paint into one raster per request.
|
|
13
11
|
*
|
|
14
|
-
*
|
|
12
|
+
* The two halves are handed **different times for the same request**,
|
|
13
|
+
* and that difference is the reason this class exists:
|
|
14
|
+
*
|
|
15
|
+
* - the batched half is given the request snapped to a caption tick,
|
|
16
|
+
* which is what lets the frames that resolve to one picture share a
|
|
17
|
+
* tile;
|
|
18
|
+
* - the video-bound half is given the frame's own time, because it
|
|
19
|
+
* bakes the pixels under the caption into its picture, and those
|
|
20
|
+
* differ at every frame the decoder hands over.
|
|
21
|
+
*
|
|
22
|
+
* Snapping both would tie a whole run of frames to one backdrop
|
|
23
|
+
* wherever the video runs faster than the tick rate, leaving the
|
|
24
|
+
* caption sitting on pixels the video has already moved past.
|
|
25
|
+
*
|
|
26
|
+
* `framesFor` expects monotonically advancing times.
|
|
15
27
|
*/
|
|
16
28
|
export class ComposedSubtitleLayerSource {
|
|
17
29
|
batchedSource;
|
|
18
30
|
videoBoundSource;
|
|
19
31
|
captionInterval = 0;
|
|
20
32
|
captionIdx = 0;
|
|
21
|
-
|
|
33
|
+
openedBatched = null;
|
|
34
|
+
openedVideoBound = null;
|
|
22
35
|
constructor(batchedSource, videoBoundSource) {
|
|
23
36
|
this.batchedSource = batchedSource;
|
|
24
37
|
this.videoBoundSource = videoBoundSource;
|
|
@@ -29,23 +42,41 @@ export class ComposedSubtitleLayerSource {
|
|
|
29
42
|
const { batched, videoBound } = this.partitionStyles(styles);
|
|
30
43
|
if (Object.keys(batched).length > 0) {
|
|
31
44
|
await this.batchedSource.open(doc, batched, width, height, captionInterval);
|
|
32
|
-
this.
|
|
45
|
+
this.openedBatched = this.batchedSource;
|
|
33
46
|
}
|
|
34
47
|
if (Object.keys(videoBound).length > 0) {
|
|
35
48
|
await this.videoBoundSource.open(doc, videoBound, width, height, captionInterval);
|
|
36
|
-
this.
|
|
49
|
+
this.openedVideoBound = this.videoBoundSource;
|
|
37
50
|
}
|
|
38
51
|
}
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
52
|
+
/**
|
|
53
|
+
* The largest either open half asks for, so a run gathered for this
|
|
54
|
+
* composite is long enough to serve the hungriest of them. The other
|
|
55
|
+
* is handed the same run and is free to ignore its length.
|
|
56
|
+
*/
|
|
57
|
+
lookAhead() {
|
|
58
|
+
return Math.max(1, this.openedBatched?.lookAhead() ?? 1, this.openedVideoBound?.lookAhead() ?? 1);
|
|
59
|
+
}
|
|
60
|
+
async framesFor(requests) {
|
|
61
|
+
// Walked here and nowhere else: the tick index carries across
|
|
62
|
+
// calls, so a request snapped twice would advance it twice.
|
|
63
|
+
const onTicks = requests.map((request) => this.snapToCaptionTick(request));
|
|
64
|
+
const [batchedLayers, videoBoundLayers] = await Promise.all([
|
|
65
|
+
this.openedBatched?.framesFor(onTicks) ?? [],
|
|
66
|
+
this.openedVideoBound?.framesFor(requests) ?? [],
|
|
67
|
+
]);
|
|
68
|
+
// Batched first: the video-bound layer paints over it.
|
|
69
|
+
return requests.map((_, i) => LayeredSubtitleFrame.from(batchedLayers[i] ?? null, videoBoundLayers[i] ?? null));
|
|
70
|
+
}
|
|
71
|
+
snapToCaptionTick(request) {
|
|
72
|
+
this.captionIdx = this.advanceCaptionIdxToTime(this.captionIdx, request.time);
|
|
73
|
+
return { time: this.captionIdx * this.captionInterval, videoFrame: request.videoFrame };
|
|
44
74
|
}
|
|
45
75
|
close() {
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
this.
|
|
76
|
+
this.openedBatched?.close();
|
|
77
|
+
this.openedVideoBound?.close();
|
|
78
|
+
this.openedBatched = null;
|
|
79
|
+
this.openedVideoBound = null;
|
|
49
80
|
}
|
|
50
81
|
partitionStyles(styles) {
|
|
51
82
|
const batched = {};
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"ComposedSubtitleLayerSource.js","sourceRoot":"","sources":["../../../../../src/modules/video/mediabunny/caption/ComposedSubtitleLayerSource.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,oBAAoB,EAAE,MAAM,yCAAyC,CAAC;
|
|
1
|
+
{"version":3,"file":"ComposedSubtitleLayerSource.js","sourceRoot":"","sources":["../../../../../src/modules/video/mediabunny/caption/ComposedSubtitleLayerSource.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,oBAAoB,EAAE,MAAM,yCAAyC,CAAC;AAM/E,8DAA8D;AAC9D,8DAA8D;AAC9D,4DAA4D;AAC5D,wBAAwB;AACxB,MAAM,0BAA0B,GAAG,IAAI,CAAC;AAOxC;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,MAAM,OAAO,2BAA2B;IAQnB;IACA;IAPX,eAAe,GAAG,CAAC,CAAC;IACpB,UAAU,GAAG,CAAC,CAAC;IACf,aAAa,GAA+B,IAAI,CAAC;IACjD,gBAAgB,GAA+B,IAAI,CAAC;IAE5D,YACmB,aAAkC,EAClC,gBAAqC;QADrC,kBAAa,GAAb,aAAa,CAAqB;QAClC,qBAAgB,GAAhB,gBAAgB,CAAqB;IACrD,CAAC;IAEJ,KAAK,CAAC,IAAI,CACR,GAAa,EACb,MAA+C,EAC/C,KAAa,EACb,MAAc,EACd,eAAuB;QAEvB,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;QACvC,IAAI,CAAC,UAAU,GAAG,CAAC,CAAC;QACpB,MAAM,EAAE,OAAO,EAAE,UAAU,EAAE,GAAG,IAAI,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC;QAC7D,IAAI,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACpC,MAAM,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,GAAG,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,eAAe,CAAC,CAAC;YAC5E,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC,aAAa,CAAC;QAC1C,CAAC;QACD,IAAI,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACvC,MAAM,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,GAAG,EAAE,UAAU,EAAE,KAAK,EAAE,MAAM,EAAE,eAAe,CAAC,CAAC;YAClF,IAAI,CAAC,gBAAgB,GAAG,IAAI,CAAC,gBAAgB,CAAC;QAChD,CAAC;IACH,CAAC;IAED;;;;OAIG;IACH,SAAS;QACP,OAAO,IAAI,CAAC,GAAG,CACb,CAAC,EACD,IAAI,CAAC,aAAa,EAAE,SAAS,EAAE,IAAI,CAAC,EACpC,IAAI,CAAC,gBAAgB,EAAE,SAAS,EAAE,IAAI,CAAC,CACxC,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,SAAS,CAAC,QAA6C;QAC3D,8DAA8D;QAC9D,4DAA4D;QAC5D,MAAM,OAAO,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,IAAI,CAAC,iBAAiB,CAAC,OAAO,CAAC,CAAC,CAAC;QAC3E,MAAM,CAAC,aAAa,EAAE,gBAAgB,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;YAC1D,IAAI,CAAC,aAAa,EAAE,SAAS,CAAC,OAAO,CAAC,IAAI,EAAE;YAC5C,IAAI,CAAC,gBAAgB,EAAE,SAAS,CAAC,QAAQ,CAAC,IAAI,EAAE;SACjD,CAAC,CAAC;QACH,uDAAuD;QACvD,OAAO,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAC3B,oBAAoB,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,IAAI,IAAI,EAAE,gBAAgB,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,CACjF,CAAC;IACJ,CAAC;IAEO,iBAAiB,CAAC,OAA6B;QACrD,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,uBAAuB,CAAC,IAAI,CAAC,UAAU,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC;QAC9E,OAAO,EAAE,IAAI,EAAE,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,eAAe,EAAE,UAAU,EAAE,OAAO,CAAC,UAAU,EAAE,CAAC;IAC1F,CAAC;IAED,KAAK;QACH,IAAI,CAAC,aAAa,EAAE,KAAK,EAAE,CAAC;QAC5B,IAAI,CAAC,gBAAgB,EAAE,KAAK,EAAE,CAAC;QAC/B,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC;QAC1B,IAAI,CAAC,gBAAgB,GAAG,IAAI,CAAC;IAC/B,CAAC;IAEO,eAAe,CAAC,MAA+C;QACrE,MAAM,OAAO,GAAkC,EAAE,CAAC;QAClD,MAAM,UAAU,GAAkC,EAAE,CAAC;QACrD,KAAK,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;YACnD,IAAI,KAAK,CAAC,SAAS,CAAC,UAAU,CAAC,QAAQ;gBAAE,UAAU,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC;;gBAC7D,OAAO,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC;QAC7B,CAAC;QACD,OAAO,EAAE,OAAO,EAAE,UAAU,EAAE,CAAC;IACjC,CAAC;IAED,gEAAgE;IAChE,+DAA+D;IAC/D,iEAAiE;IACzD,uBAAuB,CAAC,UAAkB,EAAE,IAAY;QAC9D,IAAI,GAAG,GAAG,UAAU,CAAC;QACrB,OAAO,CAAC,GAAG,GAAG,GAAG,CAAC,GAAG,IAAI,CAAC,eAAe,IAAI,IAAI,GAAG,0BAA0B,EAAE,CAAC;YAC/E,GAAG,EAAE,CAAC;QACR,CAAC;QACD,OAAO,GAAG,CAAC;IACb,CAAC;CACF"}
|
|
@@ -1,15 +1,21 @@
|
|
|
1
1
|
import type { Document } from '../../../document/Document';
|
|
2
2
|
import type { SubtitleFrame, SubtitleStyle } from '../../../rendering/SubtitleFrameRenderer';
|
|
3
3
|
import type { DecodedVideoFrame } from '../../../video/mediabunny/frame/VideoFrameDecoder';
|
|
4
|
+
/** One output frame's caption: the time it lands on and the video frame under it. */
|
|
5
|
+
export interface SubtitleLayerRequest {
|
|
6
|
+
/** Presentation timestamp of `videoFrame` in seconds. */
|
|
7
|
+
readonly time: number;
|
|
8
|
+
readonly videoFrame: DecodedVideoFrame;
|
|
9
|
+
}
|
|
4
10
|
/**
|
|
5
|
-
* Produces
|
|
11
|
+
* Produces caption rasters for a render, a run of frames at a time.
|
|
6
12
|
*
|
|
7
13
|
* Lifecycle is per-render: {@link open} prepares the source for a
|
|
8
|
-
* specific document and style set, {@link
|
|
9
|
-
*
|
|
10
|
-
*
|
|
11
|
-
*
|
|
12
|
-
*
|
|
14
|
+
* specific document and style set, {@link framesFor} services runs of
|
|
15
|
+
* frames in monotonically advancing time order, and {@link close}
|
|
16
|
+
* releases the underlying resources. Open/close pairs may repeat
|
|
17
|
+
* against the same instance, but a source cannot serve two renders
|
|
18
|
+
* concurrently.
|
|
13
19
|
*/
|
|
14
20
|
export interface SubtitleLayerSource {
|
|
15
21
|
/**
|
|
@@ -20,13 +26,24 @@ export interface SubtitleLayerSource {
|
|
|
20
26
|
*/
|
|
21
27
|
open(doc: Document, styles: Readonly<Record<string, SubtitleStyle>>, width: number, height: number, captionInterval: number): Promise<void>;
|
|
22
28
|
/**
|
|
23
|
-
*
|
|
24
|
-
*
|
|
29
|
+
* How many consecutive requests the source wants in one call to work
|
|
30
|
+
* at its best. `1` means it can answer frame by frame; a larger
|
|
31
|
+
* number means it amortizes work across a run and the caller should
|
|
32
|
+
* gather that many before asking. It is a preference, not a
|
|
33
|
+
* requirement: a call with fewer is always valid, and the last run of
|
|
34
|
+
* a render is short by nature.
|
|
35
|
+
*
|
|
36
|
+
* Meaningful only after {@link open}.
|
|
37
|
+
*/
|
|
38
|
+
lookAhead(): number;
|
|
39
|
+
/**
|
|
40
|
+
* Resolves to one raster per request, in the order given, with
|
|
41
|
+
* `null` wherever no Section served by this source is active.
|
|
25
42
|
*
|
|
26
|
-
*
|
|
27
|
-
* @
|
|
43
|
+
* The rasters are owned by the source and stay valid only until the
|
|
44
|
+
* next {@link framesFor} call or until {@link close}.
|
|
28
45
|
*/
|
|
29
|
-
|
|
46
|
+
framesFor(requests: ReadonlyArray<SubtitleLayerRequest>): Promise<Array<SubtitleFrame | null>>;
|
|
30
47
|
close(): void;
|
|
31
48
|
}
|
|
32
49
|
//# sourceMappingURL=SubtitleLayerSource.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"SubtitleLayerSource.d.ts","sourceRoot":"","sources":["../../../../../src/modules/video/mediabunny/caption/SubtitleLayerSource.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,4BAA4B,CAAC;AAC3D,OAAO,KAAK,EACV,aAAa,EACb,aAAa,EACd,MAAM,0CAA0C,CAAC;AAClD,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,mDAAmD,CAAC;AAE3F;;;;;;;;;GASG;AACH,MAAM,WAAW,mBAAmB;IAClC;;;;;OAKG;IACH,IAAI,CACF,GAAG,EAAE,QAAQ,EACb,MAAM,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,aAAa,CAAC,CAAC,EAC/C,KAAK,EAAE,MAAM,EACb,MAAM,EAAE,MAAM,EACd,eAAe,EAAE,MAAM,GACtB,OAAO,CAAC,IAAI,CAAC,CAAC;IAEjB;;;;;;OAMG;IACH,
|
|
1
|
+
{"version":3,"file":"SubtitleLayerSource.d.ts","sourceRoot":"","sources":["../../../../../src/modules/video/mediabunny/caption/SubtitleLayerSource.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,4BAA4B,CAAC;AAC3D,OAAO,KAAK,EACV,aAAa,EACb,aAAa,EACd,MAAM,0CAA0C,CAAC;AAClD,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,mDAAmD,CAAC;AAE3F,qFAAqF;AACrF,MAAM,WAAW,oBAAoB;IACnC,yDAAyD;IACzD,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,UAAU,EAAE,iBAAiB,CAAC;CACxC;AAED;;;;;;;;;GASG;AACH,MAAM,WAAW,mBAAmB;IAClC;;;;;OAKG;IACH,IAAI,CACF,GAAG,EAAE,QAAQ,EACb,MAAM,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,aAAa,CAAC,CAAC,EAC/C,KAAK,EAAE,MAAM,EACb,MAAM,EAAE,MAAM,EACd,eAAe,EAAE,MAAM,GACtB,OAAO,CAAC,IAAI,CAAC,CAAC;IAEjB;;;;;;;;;OASG;IACH,SAAS,IAAI,MAAM,CAAC;IAEpB;;;;;;OAMG;IACH,SAAS,CAAC,QAAQ,EAAE,aAAa,CAAC,oBAAoB,CAAC,GAAG,OAAO,CAAC,KAAK,CAAC,aAAa,GAAG,IAAI,CAAC,CAAC,CAAC;IAE/F,KAAK,IAAI,IAAI,CAAC;CACf"}
|
|
@@ -1,25 +1,46 @@
|
|
|
1
1
|
import type { Document } from '../../../document/Document';
|
|
2
2
|
import type { SubtitleFrame, SubtitleFrameRenderer, SubtitleStyle } from '../../../rendering/SubtitleFrameRenderer';
|
|
3
|
-
import type {
|
|
4
|
-
import type { SubtitleLayerSource } from '../../../video/mediabunny/caption/SubtitleLayerSource';
|
|
3
|
+
import type { SubtitleLayerRequest, SubtitleLayerSource } from '../../../video/mediabunny/caption/SubtitleLayerSource';
|
|
5
4
|
/**
|
|
6
|
-
* `SubtitleLayerSource`
|
|
7
|
-
* a
|
|
8
|
-
*
|
|
9
|
-
* `captionInterval` argument is unused; the cadence is whatever the
|
|
10
|
-
* caller drives `frameAt` at.
|
|
5
|
+
* `SubtitleLayerSource` for sections whose stylesheets sample the
|
|
6
|
+
* underlying video pixels — a backdrop blur, a frame-fed filter, text
|
|
7
|
+
* behind the actor.
|
|
11
8
|
*
|
|
12
|
-
*
|
|
13
|
-
*
|
|
14
|
-
*
|
|
15
|
-
*
|
|
9
|
+
* Such a caption cannot be rendered ahead of the video the way a
|
|
10
|
+
* document-driven one can: its picture is only knowable once the frame
|
|
11
|
+
* beneath it exists. So the source batches the other way round, over
|
|
12
|
+
* frames the caller has already gathered: it publishes the whole run
|
|
13
|
+
* into a `BufferedVideoFrameSource` and asks the renderer for all of
|
|
14
|
+
* their tiles in one go, so a run of frames costs one rasterization
|
|
15
|
+
* instead of one each.
|
|
16
|
+
*
|
|
17
|
+
* The caller decides how large a run to gather, and {@link lookAhead}
|
|
18
|
+
* tells it what this source can use — which is however many tiles the
|
|
19
|
+
* renderer's sheet holds, since a run longer than that would spill into
|
|
20
|
+
* a second rasterization and buy nothing.
|
|
16
21
|
*/
|
|
17
22
|
export declare class VideoBoundSubtitleLayerSource implements SubtitleLayerSource {
|
|
18
23
|
private readonly subtitleRenderer;
|
|
19
24
|
private videoFrameSource;
|
|
25
|
+
private maxTilesPerBatch;
|
|
20
26
|
constructor(subtitleRenderer: SubtitleFrameRenderer);
|
|
21
27
|
open(doc: Document, styles: Readonly<Record<string, SubtitleStyle>>, width: number, height: number, _captionInterval: number): Promise<void>;
|
|
22
|
-
|
|
28
|
+
/**
|
|
29
|
+
* However many tiles the sheet holds, since a run longer than that
|
|
30
|
+
* would spill into a second rasterization and buy nothing.
|
|
31
|
+
*
|
|
32
|
+
* It is an exact fit rather than an estimate: every timestamp served
|
|
33
|
+
* here takes a tile of its own — the planner forces that for a style
|
|
34
|
+
* that reads the frame — so a run of N costs exactly N tiles and one
|
|
35
|
+
* of `maxTilesPerBatch` fills the sheet without ever overflowing it.
|
|
36
|
+
*
|
|
37
|
+
* The frames the run holds are a second buffer beside the sheet, but
|
|
38
|
+
* not one this number should be shrunk for: what caps the sheet is
|
|
39
|
+
* partly the raster length a host will decode without blanking its
|
|
40
|
+
* far edge, and a run of separate canvases has no such edge.
|
|
41
|
+
*/
|
|
42
|
+
lookAhead(): number;
|
|
43
|
+
framesFor(requests: ReadonlyArray<SubtitleLayerRequest>): Promise<Array<SubtitleFrame | null>>;
|
|
23
44
|
close(): void;
|
|
24
45
|
}
|
|
25
46
|
//# sourceMappingURL=VideoBoundSubtitleLayerSource.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"VideoBoundSubtitleLayerSource.d.ts","sourceRoot":"","sources":["../../../../../src/modules/video/mediabunny/caption/VideoBoundSubtitleLayerSource.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,4BAA4B,CAAC;AAC3D,OAAO,KAAK,EACV,aAAa,EACb,qBAAqB,EACrB,aAAa,EACd,MAAM,0CAA0C,CAAC;
|
|
1
|
+
{"version":3,"file":"VideoBoundSubtitleLayerSource.d.ts","sourceRoot":"","sources":["../../../../../src/modules/video/mediabunny/caption/VideoBoundSubtitleLayerSource.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,4BAA4B,CAAC;AAC3D,OAAO,KAAK,EACV,aAAa,EACb,qBAAqB,EACrB,aAAa,EACd,MAAM,0CAA0C,CAAC;AAElD,OAAO,KAAK,EACV,oBAAoB,EACpB,mBAAmB,EACpB,MAAM,uDAAuD,CAAC;AAE/D;;;;;;;;;;;;;;;;;GAiBG;AACH,qBAAa,6BAA8B,YAAW,mBAAmB;IAK3D,OAAO,CAAC,QAAQ,CAAC,gBAAgB;IAH7C,OAAO,CAAC,gBAAgB,CAAyC;IACjE,OAAO,CAAC,gBAAgB,CAAK;gBAEA,gBAAgB,EAAE,qBAAqB;IAE9D,IAAI,CACR,GAAG,EAAE,QAAQ,EACb,MAAM,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,aAAa,CAAC,CAAC,EAC/C,KAAK,EAAE,MAAM,EACb,MAAM,EAAE,MAAM,EACd,gBAAgB,EAAE,MAAM,GACvB,OAAO,CAAC,IAAI,CAAC;IAMhB;;;;;;;;;;;;;OAaG;IACH,SAAS,IAAI,MAAM;IAIb,SAAS,CAAC,QAAQ,EAAE,aAAa,CAAC,oBAAoB,CAAC,GAAG,OAAO,CAAC,KAAK,CAAC,aAAa,GAAG,IAAI,CAAC,CAAC;IAsBpG,KAAK,IAAI,IAAI;CAId"}
|
|
@@ -1,33 +1,68 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { BufferedVideoFrameSource } from '../../../video/mediabunny/frame/BufferedVideoFrameSource';
|
|
2
2
|
/**
|
|
3
|
-
* `SubtitleLayerSource`
|
|
4
|
-
* a
|
|
5
|
-
*
|
|
6
|
-
* `captionInterval` argument is unused; the cadence is whatever the
|
|
7
|
-
* caller drives `frameAt` at.
|
|
3
|
+
* `SubtitleLayerSource` for sections whose stylesheets sample the
|
|
4
|
+
* underlying video pixels — a backdrop blur, a frame-fed filter, text
|
|
5
|
+
* behind the actor.
|
|
8
6
|
*
|
|
9
|
-
*
|
|
10
|
-
*
|
|
11
|
-
*
|
|
12
|
-
*
|
|
7
|
+
* Such a caption cannot be rendered ahead of the video the way a
|
|
8
|
+
* document-driven one can: its picture is only knowable once the frame
|
|
9
|
+
* beneath it exists. So the source batches the other way round, over
|
|
10
|
+
* frames the caller has already gathered: it publishes the whole run
|
|
11
|
+
* into a `BufferedVideoFrameSource` and asks the renderer for all of
|
|
12
|
+
* their tiles in one go, so a run of frames costs one rasterization
|
|
13
|
+
* instead of one each.
|
|
14
|
+
*
|
|
15
|
+
* The caller decides how large a run to gather, and {@link lookAhead}
|
|
16
|
+
* tells it what this source can use — which is however many tiles the
|
|
17
|
+
* renderer's sheet holds, since a run longer than that would spill into
|
|
18
|
+
* a second rasterization and buy nothing.
|
|
13
19
|
*/
|
|
14
20
|
export class VideoBoundSubtitleLayerSource {
|
|
15
21
|
subtitleRenderer;
|
|
16
22
|
videoFrameSource = null;
|
|
23
|
+
maxTilesPerBatch = 1;
|
|
17
24
|
constructor(subtitleRenderer) {
|
|
18
25
|
this.subtitleRenderer = subtitleRenderer;
|
|
19
26
|
}
|
|
20
27
|
async open(doc, styles, width, height, _captionInterval) {
|
|
21
|
-
this.videoFrameSource = new
|
|
28
|
+
this.videoFrameSource = new BufferedVideoFrameSource(width, height);
|
|
22
29
|
await this.subtitleRenderer.open(doc, styles, width, height, this.videoFrameSource);
|
|
30
|
+
this.maxTilesPerBatch = await this.subtitleRenderer.getMaxTilesPerBatch();
|
|
31
|
+
}
|
|
32
|
+
/**
|
|
33
|
+
* However many tiles the sheet holds, since a run longer than that
|
|
34
|
+
* would spill into a second rasterization and buy nothing.
|
|
35
|
+
*
|
|
36
|
+
* It is an exact fit rather than an estimate: every timestamp served
|
|
37
|
+
* here takes a tile of its own — the planner forces that for a style
|
|
38
|
+
* that reads the frame — so a run of N costs exactly N tiles and one
|
|
39
|
+
* of `maxTilesPerBatch` fills the sheet without ever overflowing it.
|
|
40
|
+
*
|
|
41
|
+
* The frames the run holds are a second buffer beside the sheet, but
|
|
42
|
+
* not one this number should be shrunk for: what caps the sheet is
|
|
43
|
+
* partly the raster length a host will decode without blanking its
|
|
44
|
+
* far edge, and a run of separate canvases has no such edge.
|
|
45
|
+
*/
|
|
46
|
+
lookAhead() {
|
|
47
|
+
return this.maxTilesPerBatch;
|
|
23
48
|
}
|
|
24
|
-
async
|
|
25
|
-
if (!this.videoFrameSource)
|
|
26
|
-
return null;
|
|
27
|
-
//
|
|
28
|
-
// frame back through
|
|
29
|
-
this.videoFrameSource.
|
|
30
|
-
|
|
49
|
+
async framesFor(requests) {
|
|
50
|
+
if (!this.videoFrameSource || requests.length === 0)
|
|
51
|
+
return requests.map(() => null);
|
|
52
|
+
// Published before the fetch: the renderer's stylesheets pull each
|
|
53
|
+
// frame back through the source while its tile rasterizes.
|
|
54
|
+
this.videoFrameSource.hold(requests.map((request) => ({ time: request.time, frame: request.videoFrame })));
|
|
55
|
+
const times = requests.map((request) => request.time);
|
|
56
|
+
const frames = await this.subtitleRenderer.getFrames(times);
|
|
57
|
+
// A batch normally covers a prefix, but not one asked for here: a
|
|
58
|
+
// run never exceeds the sheet's tile capacity and every timestamp
|
|
59
|
+
// costs exactly one tile, so the whole run fits by construction.
|
|
60
|
+
// Serving the tail as `null` instead would silently ship frames
|
|
61
|
+
// with no caption on them.
|
|
62
|
+
if (frames.length !== times.length) {
|
|
63
|
+
throw new Error(`Subtitle renderer covered ${frames.length} of ${times.length} video-bound timestamps in one batch.`);
|
|
64
|
+
}
|
|
65
|
+
return frames;
|
|
31
66
|
}
|
|
32
67
|
close() {
|
|
33
68
|
this.subtitleRenderer.close();
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"VideoBoundSubtitleLayerSource.js","sourceRoot":"","sources":["../../../../../src/modules/video/mediabunny/caption/VideoBoundSubtitleLayerSource.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"VideoBoundSubtitleLayerSource.js","sourceRoot":"","sources":["../../../../../src/modules/video/mediabunny/caption/VideoBoundSubtitleLayerSource.ts"],"names":[],"mappings":"AAMA,OAAO,EAAE,wBAAwB,EAAE,MAAM,0DAA0D,CAAC;AAMpG;;;;;;;;;;;;;;;;;GAiBG;AACH,MAAM,OAAO,6BAA6B;IAKX;IAHrB,gBAAgB,GAAoC,IAAI,CAAC;IACzD,gBAAgB,GAAG,CAAC,CAAC;IAE7B,YAA6B,gBAAuC;QAAvC,qBAAgB,GAAhB,gBAAgB,CAAuB;IAAG,CAAC;IAExE,KAAK,CAAC,IAAI,CACR,GAAa,EACb,MAA+C,EAC/C,KAAa,EACb,MAAc,EACd,gBAAwB;QAExB,IAAI,CAAC,gBAAgB,GAAG,IAAI,wBAAwB,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;QACpE,MAAM,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,CAAC,gBAAgB,CAAC,CAAC;QACpF,IAAI,CAAC,gBAAgB,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,mBAAmB,EAAE,CAAC;IAC5E,CAAC;IAED;;;;;;;;;;;;;OAaG;IACH,SAAS;QACP,OAAO,IAAI,CAAC,gBAAgB,CAAC;IAC/B,CAAC;IAED,KAAK,CAAC,SAAS,CAAC,QAA6C;QAC3D,IAAI,CAAC,IAAI,CAAC,gBAAgB,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,QAAQ,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,CAAC;QACrF,mEAAmE;QACnE,2DAA2D;QAC3D,IAAI,CAAC,gBAAgB,CAAC,IAAI,CACxB,QAAQ,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,OAAO,CAAC,IAAI,EAAE,KAAK,EAAE,OAAO,CAAC,UAAU,EAAE,CAAC,CAAC,CAC/E,CAAC;QACF,MAAM,KAAK,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;QACtD,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;QAC5D,kEAAkE;QAClE,kEAAkE;QAClE,iEAAiE;QACjE,gEAAgE;QAChE,2BAA2B;QAC3B,IAAI,MAAM,CAAC,MAAM,KAAK,KAAK,CAAC,MAAM,EAAE,CAAC;YACnC,MAAM,IAAI,KAAK,CACb,6BAA6B,MAAM,CAAC,MAAM,OAAO,KAAK,CAAC,MAAM,uCAAuC,CACrG,CAAC;QACJ,CAAC;QACD,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,KAAK;QACH,IAAI,CAAC,gBAAgB,CAAC,KAAK,EAAE,CAAC;QAC9B,IAAI,CAAC,gBAAgB,GAAG,IAAI,CAAC;IAC/B,CAAC;CACF"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"MediaBunnyCanvasVideoTrackEncoder.d.ts","sourceRoot":"","sources":["../../../../../src/modules/video/mediabunny/encoder/MediaBunnyCanvasVideoTrackEncoder.ts"],"names":[],"mappings":"AAAA,OAAO,EAAgB,KAAK,MAAM,EAAE,KAAK,mBAAmB,EAAE,MAAM,YAAY,CAAC;AACjF,OAAO,KAAK,EACV,UAAU,EACV,iBAAiB,EAClB,MAAM,qDAAqD,CAAC;AAE7D,MAAM,WAAW,uCAAuC;IACtD,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,aAAa,EAAE,mBAAmB,CAAC;CACpC;AAED;;;;;;;;GAQG;AACH,qBAAa,iCAAkC,YAAW,iBAAiB;IAEzE,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IAExB,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAkB;IACzC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAoC;IAC5D,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAe;gBAEhC,MAAM,EAAE,uCAAuC;
|
|
1
|
+
{"version":3,"file":"MediaBunnyCanvasVideoTrackEncoder.d.ts","sourceRoot":"","sources":["../../../../../src/modules/video/mediabunny/encoder/MediaBunnyCanvasVideoTrackEncoder.ts"],"names":[],"mappings":"AAAA,OAAO,EAAgB,KAAK,MAAM,EAAE,KAAK,mBAAmB,EAAE,MAAM,YAAY,CAAC;AACjF,OAAO,KAAK,EACV,UAAU,EACV,iBAAiB,EAClB,MAAM,qDAAqD,CAAC;AAE7D,MAAM,WAAW,uCAAuC;IACtD,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,aAAa,EAAE,mBAAmB,CAAC;CACpC;AAED;;;;;;;;GAQG;AACH,qBAAa,iCAAkC,YAAW,iBAAiB;IAEzE,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IAExB,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAkB;IACzC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAoC;IAC5D,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAe;gBAEhC,MAAM,EAAE,uCAAuC;IAwB3D,QAAQ,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI;IAIxB,MAAM,CAAC,SAAS,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC;CAIpF"}
|
|
@@ -18,9 +18,13 @@ export class MediaBunnyCanvasVideoTrackEncoder {
|
|
|
18
18
|
this.width = config.width;
|
|
19
19
|
this.height = config.height;
|
|
20
20
|
this.canvas = new OffscreenCanvas(config.width, config.height);
|
|
21
|
-
// alpha:false — the compositor paints an opaque source frame first
|
|
22
|
-
// imageSmoothingEnabled:false —
|
|
23
|
-
//
|
|
21
|
+
// alpha:false — the compositor paints an opaque source frame first.
|
|
22
|
+
// imageSmoothingEnabled:false — free whenever the output keeps the
|
|
23
|
+
// source's dimensions, which is the common path. It is not free when
|
|
24
|
+
// an export asks for smaller ones: the source frame is then scaled
|
|
25
|
+
// down unsmoothed, straight into the encoder. Whether that shows
|
|
26
|
+
// enough to be worth the smoothing is unmeasured, so the flag stays
|
|
27
|
+
// as it was rather than moving on an argument alone.
|
|
24
28
|
const ctx = this.canvas.getContext('2d', {
|
|
25
29
|
alpha: false,
|
|
26
30
|
desynchronized: false,
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"MediaBunnyCanvasVideoTrackEncoder.js","sourceRoot":"","sources":["../../../../../src/modules/video/mediabunny/encoder/MediaBunnyCanvasVideoTrackEncoder.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAyC,MAAM,YAAY,CAAC;AAYjF;;;;;;;;GAQG;AACH,MAAM,OAAO,iCAAiC;IAEnC,KAAK,CAAS;IACd,MAAM,CAAS;IAEP,MAAM,CAAkB;IACxB,OAAO,CAAoC;IAC3C,YAAY,CAAe;IAE5C,YAAY,MAA+C;QACzD,IAAI,CAAC,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC;QAC1B,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC;QAC5B,IAAI,CAAC,MAAM,GAAG,IAAI,eAAe,CAAC,MAAM,CAAC,KAAK,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC;QAC/D,oEAAoE;QACpE,
|
|
1
|
+
{"version":3,"file":"MediaBunnyCanvasVideoTrackEncoder.js","sourceRoot":"","sources":["../../../../../src/modules/video/mediabunny/encoder/MediaBunnyCanvasVideoTrackEncoder.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAyC,MAAM,YAAY,CAAC;AAYjF;;;;;;;;GAQG;AACH,MAAM,OAAO,iCAAiC;IAEnC,KAAK,CAAS;IACd,MAAM,CAAS;IAEP,MAAM,CAAkB;IACxB,OAAO,CAAoC;IAC3C,YAAY,CAAe;IAE5C,YAAY,MAA+C;QACzD,IAAI,CAAC,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC;QAC1B,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC;QAC5B,IAAI,CAAC,MAAM,GAAG,IAAI,eAAe,CAAC,MAAM,CAAC,KAAK,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC;QAC/D,oEAAoE;QACpE,mEAAmE;QACnE,qEAAqE;QACrE,mEAAmE;QACnE,iEAAiE;QACjE,oEAAoE;QACpE,qDAAqD;QACrD,MAAM,GAAG,GAAG,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,IAAI,EAAE;YACvC,KAAK,EAAE,KAAK;YACZ,cAAc,EAAE,KAAK;YACrB,kBAAkB,EAAE,KAAK;SAC1B,CAAC,CAAC;QACH,IAAI,CAAC,GAAG,EAAE,CAAC;YACT,MAAM,IAAI,KAAK,CAAC,wDAAwD,CAAC,CAAC;QAC5E,CAAC;QACD,GAAG,CAAC,qBAAqB,GAAG,KAAK,CAAC;QAClC,IAAI,CAAC,OAAO,GAAG,GAAG,CAAC;QACnB,IAAI,CAAC,YAAY,GAAG,IAAI,YAAY,CAAC,IAAI,CAAC,MAAM,EAAE,MAAM,CAAC,aAAa,CAAC,CAAC;IAC1E,CAAC;IAED,QAAQ,CAAC,MAAc;QACrB,MAAM,CAAC,aAAa,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;IAC1C,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,SAAiB,EAAE,QAAgB,EAAE,KAAiB;QACjE,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QACpB,MAAM,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC;IACnD,CAAC;CACF"}
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
import type { VideoFrameSource, VideoFrameRegion } from '../../../rendering/types/VideoFrameSource';
|
|
2
|
+
import type { DecodedVideoFrame } from '../../../video/mediabunny/frame/VideoFrameDecoder';
|
|
3
|
+
/** One frame of the resident group, under the time it answers to. */
|
|
4
|
+
export interface HeldFrame {
|
|
5
|
+
readonly time: number;
|
|
6
|
+
readonly frame: DecodedVideoFrame;
|
|
7
|
+
}
|
|
8
|
+
/**
|
|
9
|
+
* `VideoFrameSource` over a resident group of frames. {@link hold}
|
|
10
|
+
* publishes the group and the times it answers to; {@link getFrameAt}
|
|
11
|
+
* serves any of them and nothing else. Querying a time outside the
|
|
12
|
+
* resident group throws.
|
|
13
|
+
*
|
|
14
|
+
* Holding several at once is what lets a batch of caption tiles cover
|
|
15
|
+
* several frames: every tile in the batch pulls its own frame while
|
|
16
|
+
* the batch renders.
|
|
17
|
+
*/
|
|
18
|
+
export declare class BufferedVideoFrameSource implements VideoFrameSource {
|
|
19
|
+
private readonly width;
|
|
20
|
+
private readonly height;
|
|
21
|
+
private static readonly TIMESTAMP_TOLERANCE_S;
|
|
22
|
+
private held;
|
|
23
|
+
constructor(width: number, height: number);
|
|
24
|
+
/**
|
|
25
|
+
* Makes `frames` the only ones {@link getFrameAt} will serve, each
|
|
26
|
+
* under its paired time, until the next call. Does not take ownership:
|
|
27
|
+
* the frames must stay usable until then and are closed by whoever
|
|
28
|
+
* produced them.
|
|
29
|
+
*/
|
|
30
|
+
hold(frames: ReadonlyArray<HeldFrame>): void;
|
|
31
|
+
getFrameAt(timeSeconds: number, jpegQuality: number, region?: VideoFrameRegion): Promise<string>;
|
|
32
|
+
private frameAt;
|
|
33
|
+
private encode;
|
|
34
|
+
/**
|
|
35
|
+
* Draws `region` of the frame onto a canvas sized to that region,
|
|
36
|
+
* which is what the JPEG is then encoded from. Drawing straight from
|
|
37
|
+
* the frame rather than from a full-size copy of it is one fewer trip
|
|
38
|
+
* through the pixels per tile.
|
|
39
|
+
*/
|
|
40
|
+
private drawInto;
|
|
41
|
+
private encodeJpeg;
|
|
42
|
+
private createCanvas;
|
|
43
|
+
private clipToFrame;
|
|
44
|
+
}
|
|
45
|
+
//# sourceMappingURL=BufferedVideoFrameSource.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"BufferedVideoFrameSource.d.ts","sourceRoot":"","sources":["../../../../../src/modules/video/mediabunny/frame/BufferedVideoFrameSource.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,gBAAgB,EAAE,gBAAgB,EAAE,MAAM,2CAA2C,CAAC;AACpG,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,mDAAmD,CAAC;AAG3F,qEAAqE;AACrE,MAAM,WAAW,SAAS;IACxB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,KAAK,EAAE,iBAAiB,CAAC;CACnC;AAED;;;;;;;;;GASG;AACH,qBAAa,wBAAyB,YAAW,gBAAgB;IAQnD,OAAO,CAAC,QAAQ,CAAC,KAAK;IAAU,OAAO,CAAC,QAAQ,CAAC,MAAM;IAJnE,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,qBAAqB,CAAQ;IAErD,OAAO,CAAC,IAAI,CAAgC;gBAEf,KAAK,EAAE,MAAM,EAAmB,MAAM,EAAE,MAAM;IAE3E;;;;;OAKG;IACH,IAAI,CAAC,MAAM,EAAE,aAAa,CAAC,SAAS,CAAC,GAAG,IAAI;IAItC,UAAU,CAAC,WAAW,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,gBAAgB,GAAG,OAAO,CAAC,MAAM,CAAC;IAOtG,OAAO,CAAC,OAAO;IAaf,OAAO,CAAC,MAAM;IAYd;;;;;OAKG;IACH,OAAO,CAAC,QAAQ;IAwBhB,OAAO,CAAC,UAAU;IAMlB,OAAO,CAAC,YAAY;IAOpB,OAAO,CAAC,WAAW;CAOpB"}
|
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
import { profiler } from '../../../profiling/Profiler';
|
|
2
|
+
/**
|
|
3
|
+
* `VideoFrameSource` over a resident group of frames. {@link hold}
|
|
4
|
+
* publishes the group and the times it answers to; {@link getFrameAt}
|
|
5
|
+
* serves any of them and nothing else. Querying a time outside the
|
|
6
|
+
* resident group throws.
|
|
7
|
+
*
|
|
8
|
+
* Holding several at once is what lets a batch of caption tiles cover
|
|
9
|
+
* several frames: every tile in the batch pulls its own frame while
|
|
10
|
+
* the batch renders.
|
|
11
|
+
*/
|
|
12
|
+
export class BufferedVideoFrameSource {
|
|
13
|
+
width;
|
|
14
|
+
height;
|
|
15
|
+
// Any mismatch beyond IEEE-754 noise is a coordination bug between
|
|
16
|
+
// the producer and the subtitle renderer.
|
|
17
|
+
static TIMESTAMP_TOLERANCE_S = 1e-6;
|
|
18
|
+
held = [];
|
|
19
|
+
constructor(width, height) {
|
|
20
|
+
this.width = width;
|
|
21
|
+
this.height = height;
|
|
22
|
+
}
|
|
23
|
+
/**
|
|
24
|
+
* Makes `frames` the only ones {@link getFrameAt} will serve, each
|
|
25
|
+
* under its paired time, until the next call. Does not take ownership:
|
|
26
|
+
* the frames must stay usable until then and are closed by whoever
|
|
27
|
+
* produced them.
|
|
28
|
+
*/
|
|
29
|
+
hold(frames) {
|
|
30
|
+
this.held = frames;
|
|
31
|
+
}
|
|
32
|
+
async getFrameAt(timeSeconds, jpegQuality, region) {
|
|
33
|
+
const frame = this.frameAt(timeSeconds);
|
|
34
|
+
return profiler.time('BufferedVideoFrameSource.encode', () => this.encode(frame, region, jpegQuality));
|
|
35
|
+
}
|
|
36
|
+
frameAt(timeSeconds) {
|
|
37
|
+
const match = this.held.find((candidate) => Math.abs(candidate.time - timeSeconds) <= BufferedVideoFrameSource.TIMESTAMP_TOLERANCE_S);
|
|
38
|
+
if (!match) {
|
|
39
|
+
const times = this.held.map((candidate) => candidate.time).join(', ');
|
|
40
|
+
throw new Error(`BufferedVideoFrameSource: requested time ${timeSeconds}, but the resident group holds [${times}].`);
|
|
41
|
+
}
|
|
42
|
+
return match.frame;
|
|
43
|
+
}
|
|
44
|
+
encode(frame, region, jpegQuality) {
|
|
45
|
+
const clipped = region ? this.clipToFrame(region) : null;
|
|
46
|
+
if (!clipped || clipped.width === 0 || clipped.height === 0) {
|
|
47
|
+
return this.encodeJpeg(this.drawInto(frame, { x: 0, y: 0, width: this.width, height: this.height }), jpegQuality);
|
|
48
|
+
}
|
|
49
|
+
return this.encodeJpeg(this.drawInto(frame, clipped), jpegQuality);
|
|
50
|
+
}
|
|
51
|
+
/**
|
|
52
|
+
* Draws `region` of the frame onto a canvas sized to that region,
|
|
53
|
+
* which is what the JPEG is then encoded from. Drawing straight from
|
|
54
|
+
* the frame rather than from a full-size copy of it is one fewer trip
|
|
55
|
+
* through the pixels per tile.
|
|
56
|
+
*/
|
|
57
|
+
drawInto(frame, region) {
|
|
58
|
+
const canvas = this.createCanvas(region.width, region.height);
|
|
59
|
+
const context = profiler.time('BufferedVideoFrameSource.cropCanvasContext', () => canvas.getContext('2d', { alpha: false }));
|
|
60
|
+
if (!context) {
|
|
61
|
+
throw new Error('Canvas 2D context is not available for cropping.');
|
|
62
|
+
}
|
|
63
|
+
profiler.time('BufferedVideoFrameSource.cropDraw', () => frame.draw(context, -region.x, -region.y, this.width, this.height));
|
|
64
|
+
return canvas;
|
|
65
|
+
}
|
|
66
|
+
// Chosen for predictability over raw speed. `toDataURL` encodes inline;
|
|
67
|
+
// `OffscreenCanvas.convertToBlob` defers to the browser's async image
|
|
68
|
+
// scheduler, which was measured taking ~1s per call instead of the usual
|
|
69
|
+
// few ms, under a load that could not be pinned down or reproduced.
|
|
70
|
+
// Output is byte-identical between the two on Chrome and Firefox, and the
|
|
71
|
+
// async overhead is per-call rather than per-pixel, so the two only differ
|
|
72
|
+
// meaningfully at small crop sizes. The tradeoff accepted here is that
|
|
73
|
+
// encoding inline blocks the main thread for the whole encode, and that a
|
|
74
|
+
// DOM canvas rules out ever running this step in a Worker.
|
|
75
|
+
// Benchmarked in experiments/jpeg-encode-apis/.
|
|
76
|
+
encodeJpeg(canvas, jpegQuality) {
|
|
77
|
+
return profiler.time('BufferedVideoFrameSource.encodeJpeg', () => canvas.toDataURL('image/jpeg', jpegQuality));
|
|
78
|
+
}
|
|
79
|
+
createCanvas(width, height) {
|
|
80
|
+
const canvas = document.createElement('canvas');
|
|
81
|
+
canvas.width = width;
|
|
82
|
+
canvas.height = height;
|
|
83
|
+
return canvas;
|
|
84
|
+
}
|
|
85
|
+
clipToFrame(region) {
|
|
86
|
+
const x = Math.max(0, Math.floor(region.x));
|
|
87
|
+
const y = Math.max(0, Math.floor(region.y));
|
|
88
|
+
const right = Math.min(this.width, Math.ceil(region.x + region.width));
|
|
89
|
+
const bottom = Math.min(this.height, Math.ceil(region.y + region.height));
|
|
90
|
+
return { x, y, width: Math.max(0, right - x), height: Math.max(0, bottom - y) };
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
//# sourceMappingURL=BufferedVideoFrameSource.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"BufferedVideoFrameSource.js","sourceRoot":"","sources":["../../../../../src/modules/video/mediabunny/frame/BufferedVideoFrameSource.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,QAAQ,EAAE,MAAM,6BAA6B,CAAC;AAQvD;;;;;;;;;GASG;AACH,MAAM,OAAO,wBAAwB;IAQN;IAAgC;IAN7D,mEAAmE;IACnE,0CAA0C;IAClC,MAAM,CAAU,qBAAqB,GAAG,IAAI,CAAC;IAE7C,IAAI,GAA6B,EAAE,CAAC;IAE5C,YAA6B,KAAa,EAAmB,MAAc;QAA9C,UAAK,GAAL,KAAK,CAAQ;QAAmB,WAAM,GAAN,MAAM,CAAQ;IAAG,CAAC;IAE/E;;;;;OAKG;IACH,IAAI,CAAC,MAAgC;QACnC,IAAI,CAAC,IAAI,GAAG,MAAM,CAAC;IACrB,CAAC;IAED,KAAK,CAAC,UAAU,CAAC,WAAmB,EAAE,WAAmB,EAAE,MAAyB;QAClF,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC;QACxC,OAAO,QAAQ,CAAC,IAAI,CAAC,iCAAiC,EAAE,GAAG,EAAE,CAC3D,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,WAAW,CAAC,CACxC,CAAC;IACJ,CAAC;IAEO,OAAO,CAAC,WAAmB;QACjC,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAC1B,CAAC,SAAS,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,IAAI,GAAG,WAAW,CAAC,IAAI,wBAAwB,CAAC,qBAAqB,CACxG,CAAC;QACF,IAAI,CAAC,KAAK,EAAE,CAAC;YACX,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACtE,MAAM,IAAI,KAAK,CACb,4CAA4C,WAAW,mCAAmC,KAAK,IAAI,CACpG,CAAC;QACJ,CAAC;QACD,OAAO,KAAK,CAAC,KAAK,CAAC;IACrB,CAAC;IAEO,MAAM,CACZ,KAAwB,EACxB,MAAoC,EACpC,WAAmB;QAEnB,MAAM,OAAO,GAAG,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QACzD,IAAI,CAAC,OAAO,IAAI,OAAO,CAAC,KAAK,KAAK,CAAC,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC5D,OAAO,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,EAAE,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,WAAW,CAAC,CAAC;QACpH,CAAC;QACD,OAAO,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,EAAE,OAAO,CAAC,EAAE,WAAW,CAAC,CAAC;IACrE,CAAC;IAED;;;;;OAKG;IACK,QAAQ,CAAC,KAAwB,EAAE,MAAwB;QACjE,MAAM,MAAM,GAAG,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,KAAK,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC;QAC9D,MAAM,OAAO,GAAG,QAAQ,CAAC,IAAI,CAAC,4CAA4C,EAAE,GAAG,EAAE,CAC/E,MAAM,CAAC,UAAU,CAAC,IAAI,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,CAC1C,CAAC;QACF,IAAI,CAAC,OAAO,EAAE,CAAC;YACb,MAAM,IAAI,KAAK,CAAC,kDAAkD,CAAC,CAAC;QACtE,CAAC;QACD,QAAQ,CAAC,IAAI,CAAC,mCAAmC,EAAE,GAAG,EAAE,CACtD,KAAK,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC,MAAM,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,CACnE,CAAC;QACF,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,wEAAwE;IACxE,sEAAsE;IACtE,yEAAyE;IACzE,oEAAoE;IACpE,0EAA0E;IAC1E,2EAA2E;IAC3E,uEAAuE;IACvE,0EAA0E;IAC1E,2DAA2D;IAC3D,gDAAgD;IACxC,UAAU,CAAC,MAAyB,EAAE,WAAmB;QAC/D,OAAO,QAAQ,CAAC,IAAI,CAAC,qCAAqC,EAAE,GAAG,EAAE,CAC/D,MAAM,CAAC,SAAS,CAAC,YAAY,EAAE,WAAW,CAAC,CAC5C,CAAC;IACJ,CAAC;IAEO,YAAY,CAAC,KAAa,EAAE,MAAc;QAChD,MAAM,MAAM,GAAG,QAAQ,CAAC,aAAa,CAAC,QAAQ,CAAC,CAAC;QAChD,MAAM,CAAC,KAAK,GAAG,KAAK,CAAC;QACrB,MAAM,CAAC,MAAM,GAAG,MAAM,CAAC;QACvB,OAAO,MAAM,CAAC;IAChB,CAAC;IAEO,WAAW,CAAC,MAAwB;QAC1C,MAAM,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;QAC5C,MAAM,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;QAC5C,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;QACvE,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC;QAC1E,OAAO,EAAE,CAAC,EAAE,CAAC,EAAE,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,GAAG,CAAC,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,GAAG,CAAC,CAAC,EAAE,CAAC;IAClF,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"DefaultVideoFrameDecoderFactory.d.ts","sourceRoot":"","sources":["../../../../../src/modules/video/mediabunny/frame/DefaultVideoFrameDecoderFactory.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,mDAAmD,CAAC;AAC3F,OAAO,KAAK,EACV,wBAAwB,EACxB,wBAAwB,EACzB,MAAM,0DAA0D,CAAC;AAIlE;;;;;GAKG;AACH,qBAAa,+BAAgC,YAAW,wBAAwB;IAExE,MAAM,CAAC,OAAO,EAAE,wBAAwB,GAAG,OAAO,CAAC,iBAAiB,CAAC;
|
|
1
|
+
{"version":3,"file":"DefaultVideoFrameDecoderFactory.d.ts","sourceRoot":"","sources":["../../../../../src/modules/video/mediabunny/frame/DefaultVideoFrameDecoderFactory.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,mDAAmD,CAAC;AAC3F,OAAO,KAAK,EACV,wBAAwB,EACxB,wBAAwB,EACzB,MAAM,0DAA0D,CAAC;AAIlE;;;;;GAKG;AACH,qBAAa,+BAAgC,YAAW,wBAAwB;IAExE,MAAM,CAAC,OAAO,EAAE,wBAAwB,GAAG,OAAO,CAAC,iBAAiB,CAAC;CAgB5E"}
|
|
@@ -9,16 +9,18 @@ import { HtmlVideoElementVideoFrameDecoder } from '../../../video/mediabunny/fra
|
|
|
9
9
|
export class DefaultVideoFrameDecoderFactory {
|
|
10
10
|
async create(request) {
|
|
11
11
|
const webCodecsCapable = await request.track.canDecode();
|
|
12
|
+
const inputCodec = (await request.track.getCodec()) ?? 'unknown';
|
|
12
13
|
if (webCodecsCapable) {
|
|
14
|
+
request.onDecoderSelected?.({ kind: 'web-codecs', inputCodec });
|
|
13
15
|
return new WebCodecsVideoFrameDecoder(request.track);
|
|
14
16
|
}
|
|
15
|
-
const inputCodec = (await request.track.getCodec()) ?? 'unknown';
|
|
16
17
|
if (request.confirmFallback) {
|
|
17
18
|
const accepted = await request.confirmFallback({ inputCodec });
|
|
18
19
|
if (!accepted) {
|
|
19
20
|
throw new DOMException('Fallback decoder declined by the caller.', 'AbortError');
|
|
20
21
|
}
|
|
21
22
|
}
|
|
23
|
+
request.onDecoderSelected?.({ kind: 'html-video-element', inputCodec });
|
|
22
24
|
return new HtmlVideoElementVideoFrameDecoder(request.source, inputCodec);
|
|
23
25
|
}
|
|
24
26
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"DefaultVideoFrameDecoderFactory.js","sourceRoot":"","sources":["../../../../../src/modules/video/mediabunny/frame/DefaultVideoFrameDecoderFactory.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,0BAA0B,EAAE,MAAM,4DAA4D,CAAC;AACxG,OAAO,EAAE,iCAAiC,EAAE,MAAM,mEAAmE,CAAC;AAEtH;;;;;GAKG;AACH,MAAM,OAAO,+BAA+B;IAE1C,KAAK,CAAC,MAAM,CAAC,OAAiC;QAC5C,MAAM,gBAAgB,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,SAAS,EAAE,CAAC;QACzD,
|
|
1
|
+
{"version":3,"file":"DefaultVideoFrameDecoderFactory.js","sourceRoot":"","sources":["../../../../../src/modules/video/mediabunny/frame/DefaultVideoFrameDecoderFactory.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,0BAA0B,EAAE,MAAM,4DAA4D,CAAC;AACxG,OAAO,EAAE,iCAAiC,EAAE,MAAM,mEAAmE,CAAC;AAEtH;;;;;GAKG;AACH,MAAM,OAAO,+BAA+B;IAE1C,KAAK,CAAC,MAAM,CAAC,OAAiC;QAC5C,MAAM,gBAAgB,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,SAAS,EAAE,CAAC;QACzD,MAAM,UAAU,GAAG,CAAC,MAAM,OAAO,CAAC,KAAK,CAAC,QAAQ,EAAE,CAAC,IAAI,SAAS,CAAC;QACjE,IAAI,gBAAgB,EAAE,CAAC;YACrB,OAAO,CAAC,iBAAiB,EAAE,CAAC,EAAE,IAAI,EAAE,YAAY,EAAE,UAAU,EAAE,CAAC,CAAC;YAChE,OAAO,IAAI,0BAA0B,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;QACvD,CAAC;QACD,IAAI,OAAO,CAAC,eAAe,EAAE,CAAC;YAC5B,MAAM,QAAQ,GAAG,MAAM,OAAO,CAAC,eAAe,CAAC,EAAE,UAAU,EAAE,CAAC,CAAC;YAC/D,IAAI,CAAC,QAAQ,EAAE,CAAC;gBACd,MAAM,IAAI,YAAY,CAAC,0CAA0C,EAAE,YAAY,CAAC,CAAC;YACnF,CAAC;QACH,CAAC;QACD,OAAO,CAAC,iBAAiB,EAAE,CAAC,EAAE,IAAI,EAAE,oBAAoB,EAAE,UAAU,EAAE,CAAC,CAAC;QACxE,OAAO,IAAI,iCAAiC,CAAC,OAAO,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;IAC3E,CAAC;CACF"}
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
import type { DecodedVideoFrame } from '../../../video/mediabunny/frame/VideoFrameDecoder';
|
|
2
|
+
/**
|
|
3
|
+
* A decoded frame's pixels copied onto a canvas of the caller's own, so
|
|
4
|
+
* they outlive the decoder's frame.
|
|
5
|
+
*
|
|
6
|
+
* A decoder hands out frames it expects back promptly: holding several
|
|
7
|
+
* at once back-pressures the frame pool into a stall. Copying breaks
|
|
8
|
+
* that tie — the decoder's frame closes immediately and what stays
|
|
9
|
+
* alive is an ordinary canvas, at the cost of one full-frame draw and
|
|
10
|
+
* `width × height × 4` bytes per frame held.
|
|
11
|
+
*/
|
|
12
|
+
export declare class RetainedVideoFrame implements DecodedVideoFrame {
|
|
13
|
+
readonly timestamp: number;
|
|
14
|
+
readonly duration: number;
|
|
15
|
+
/**
|
|
16
|
+
* Copies `frame` at the given dimensions. Does not close `frame`;
|
|
17
|
+
* the caller keeps that responsibility and may do so as soon as this
|
|
18
|
+
* returns.
|
|
19
|
+
*/
|
|
20
|
+
static copyOf(frame: DecodedVideoFrame, width: number, height: number): RetainedVideoFrame;
|
|
21
|
+
private canvas;
|
|
22
|
+
private constructor();
|
|
23
|
+
draw(context: OffscreenCanvasRenderingContext2D | CanvasRenderingContext2D, dx: number, dy: number, dWidth: number, dHeight: number): void;
|
|
24
|
+
close(): void;
|
|
25
|
+
}
|
|
26
|
+
//# sourceMappingURL=RetainedVideoFrame.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"RetainedVideoFrame.d.ts","sourceRoot":"","sources":["../../../../../src/modules/video/mediabunny/frame/RetainedVideoFrame.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,mDAAmD,CAAC;AAG3F;;;;;;;;;GASG;AACH,qBAAa,kBAAmB,YAAW,iBAAiB;IAwBxD,QAAQ,CAAC,SAAS,EAAE,MAAM;IAC1B,QAAQ,CAAC,QAAQ,EAAE,MAAM;IAvB3B;;;;OAIG;IACH,MAAM,CAAC,MAAM,CAAC,KAAK,EAAE,iBAAiB,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,kBAAkB;IAa1F,OAAO,CAAC,MAAM,CAAyB;IAEvC,OAAO;IAQP,IAAI,CACF,OAAO,EAAE,iCAAiC,GAAG,wBAAwB,EACrE,EAAE,EAAE,MAAM,EACV,EAAE,EAAE,MAAM,EACV,MAAM,EAAE,MAAM,EACd,OAAO,EAAE,MAAM,GACd,IAAI;IAOP,KAAK,IAAI,IAAI;CAUd"}
|