straight_to_video 0.0.13 → 0.0.14

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: db199c55c9da4fd2927332a51b0bacc98c8ee0c4422218f9bdbbf2f280dd5648
-  data.tar.gz: 67c67d5170d3d7be451f2d050d2f6b2063f225f1c7dbbd8b3fc82462ddccee2f
+  metadata.gz: 0cb2a8345491c6b11662c50c5d7604a9ac78024303e066da972ece9b8c967881
+  data.tar.gz: 3ec93954a73ad30c65804b77ebdd5bfb7d8ebb10004913de7be5e955416e5f71
 SHA512:
-  metadata.gz: 7a85c866bf129896f1c49d5ae9e323f7e7e13cd52f8e92c8aa64d80bc716c1f5545f7f9376f3c4c9ca5a3fcfa188cd247f04cc90650c0b1a66f09eb23dd7cf22
-  data.tar.gz: ec227f671fd6e10196b91cbd9ccc46abed79a317ea0d822ebccf5aa36488098d2a52efe7770081e225c95d4c1e2c91b225642632e3c8e837b616121ab75fe135
+  metadata.gz: 8f38b3c21dfa452d47113a6f77ce066387b8b2eca3587ffcf5ca4d2908926fc8d70f87fc486841188bef73695bf5979945c30cb1f38a70bd24d4b1100eb98506
+  data.tar.gz: d59d9a1c2b0c4bfea1e68bfefa6d57c69eedd43aa7ce54d691c571e463d3833322d0e4a9c8f4a88af6a3910022f1700e96734e025d37b61d41eb1a708b60b0fd

data/CHANGELOG.md

@@ -1,5 +1,10 @@
 # Changelog
 
+## 0.0.14
+
+* Preserve B-frame presentation timestamps (`ctts`) when normalizing the MP4 container, fixing juddery playback of compliant uploads that were re-muxed on the passthrough path.
+* Request an encoder keyframe every 2 seconds so re-encoded videos can recover from seeks, dropped frames, and downstream transcoding (previously the entire video had a single keyframe).
+
 ## 0.0.13
 
 * Fast-start already-compliant MP4 and MOV uploads without re-encoding their media packets.

data/app/assets/javascripts/straight-to-video.js

@@ -1,4 +1,4 @@
-// straight-to-video@0.0.13 vendored by the straight_to_video gem
+// straight-to-video@0.0.14 vendored by the straight_to_video gem
 // straight-to-video - https://github.com/searlsco/straight-to-video
 
 // ----- External imports -----
@@ -16,6 +16,7 @@ const TARGET_AUDIO_BITRATE = 96_000
 const TARGET_AUDIO_SR = 48_000
 const TARGET_AUDIO_CHANNELS = 2
 const MAX_VIDEO_ENCODER_QUEUE_SIZE = 4
+const KEY_FRAME_INTERVAL_SECONDS = 2
 
 // ----- Video metadata probe -----
 async function probeVideo (file) {
@@ -206,6 +207,14 @@ function shouldDecodeViaVideoElement () {
   return (navigator?.vendor || '').includes('Apple')
 }
 
+// Encoders only emit keyframes when asked (WebKit's VideoToolbox never adds
+// its own), so request one every KEY_FRAME_INTERVAL_SECONDS or players get a
+// single sync sample for the whole video and cannot recover from seeks or
+// dropped frames.
+function keyFramesEveryNthFrame (step) {
+  return Math.max(1, Math.round(KEY_FRAME_INTERVAL_SECONDS / step))
+}
+
 async function applyVideoEncoderBackpressure (encoder) {
   while (encoder.encodeQueueSize > MAX_VIDEO_ENCODER_QUEUE_SIZE) {
     await new Promise(resolve => setTimeout(resolve, 0))
@@ -260,6 +269,7 @@ async function encodeFramesViaVideoElement ({ file, durationCfr, step, frames, c
     }
   })
 
+  const keyFrameEvery = keyFramesEveryNthFrame(step)
   for (let i = 0; i < frames; i++) {
     const t = i * step
     const drawTime = Math.min(Math.max(0, t + (step * 0.5)), Math.max(0.000001, durationCfr - 0.000001))
@@ -274,7 +284,7 @@ async function encodeFramesViaVideoElement ({ file, durationCfr, step, frames, c
 
     ctx.drawImage(v, 0, 0, canvas.width, canvas.height)
     const vf = new VideoFrame(canvas, { timestamp: Math.round(t * 1e6), duration: Math.round(step * 1e6) })
-    ve.encode(vf, { keyFrame: i === 0 })
+    ve.encode(vf, { keyFrame: i % keyFrameEvery === 0 })
     vf.close()
     await applyVideoEncoderBackpressure(ve)
 
@@ -304,6 +314,7 @@ async function encodeFramesViaVideoSampleSink ({ file, durationCfr, step, frames
     sample.drawWithFit(ctx, { fit: 'fill' })
   }
 
+  const keyFrameEvery = keyFramesEveryNthFrame(step)
   let i = 0
   let prev = null
   let prevStart = 0
@@ -321,7 +332,7 @@ async function encodeFramesViaVideoSampleSink ({ file, durationCfr, step, frames
         if (displayTime < prevStart || displayTime >= end) break
         const t = i * step
         const vf = new VideoFrame(canvas, { timestamp: Math.round(t * 1e6), duration: Math.round(step * 1e6) })
-        ve.encode(vf, { keyFrame: i === 0 })
+        ve.encode(vf, { keyFrame: i % keyFrameEvery === 0 })
         vf.close()
         await applyVideoEncoderBackpressure(ve)
 
@@ -348,7 +359,7 @@ async function encodeFramesViaVideoSampleSink ({ file, durationCfr, step, frames
     while (i < frames) {
       const t = i * step
       const vf = new VideoFrame(canvas, { timestamp: Math.round(t * 1e6), duration: Math.round(step * 1e6) })
-      ve.encode(vf, { keyFrame: i === 0 })
+      ve.encode(vf, { keyFrame: i % keyFrameEvery === 0 })
       vf.close()
       await applyVideoEncoderBackpressure(ve)
 
@@ -498,6 +509,7 @@ function _extractTrack (u8, dv, trakBox) {
   const stszBox = stblKids.find(b => b.type === 'stsz')
   const stcoBox = stblKids.find(b => b.type === 'stco')
   const stssBox = stblKids.find(b => b.type === 'stss')
+  const cttsBox = stblKids.find(b => b.type === 'ctts')
   if (!stsdBox || !sttsBox || !stscBox || !stszBox || !stcoBox) return null
 
   const sampleCount = dv.getUint32(stszBox.offset + 16)
@@ -509,6 +521,9 @@ function _extractTrack (u8, dv, trakBox) {
   const stscBody = u8.slice(stscBox.offset + 12, stscBox.offset + stscBox.size)
   const stszBody = u8.slice(stszBox.offset + 12, stszBox.offset + stszBox.size)
   const stssBody = stssBox ? u8.slice(stssBox.offset + 12, stssBox.offset + stssBox.size) : null
+  // Copied whole (header included) to keep its version: mediabunny writes
+  // version 1 (signed offsets), which _full would misdeclare as version 0
+  const cttsRaw = cttsBox ? u8.slice(cttsBox.offset, cttsBox.offset + cttsBox.size) : null
 
   const entryOff = stsdBox.offset + 16
   const entrySize = dv.getUint32(entryOff)
@@ -546,7 +561,7 @@ function _extractTrack (u8, dv, trakBox) {
     tkDur, tkW, tkH, mdTs, mdDur, mdLang,
     videoCodecConfig, audioSpecificConfig,
     audioChannels, audioSampleSize, audioSampleRate,
-    sttsBody, stscBody, stszBody, stssBody,
+    sttsBody, stscBody, stszBody, stssBody, cttsRaw,
     stcoEntries, sampleCount, bitrate
   }
 }
@@ -615,6 +630,7 @@ function _buildTrak (t) {
   const stss = t.stssBody ? _full('stss', 0, 0, t.stssBody) : null
 
   const stblParts = [stsd, stts]
+  if (t.cttsRaw) stblParts.push(t.cttsRaw)
   if (stss) stblParts.push(stss)
   stblParts.push(stsc, stsz, stco)
   if (t.isAudio) {
@@ -793,11 +809,14 @@ async function encodeVideo ({ file, srcMeta, plan, onProgress }) {
 
     const muxCount = Math.min(frames, pendingPackets.length)
     videoDuration = muxCount * step
+    const keyFrameEvery = keyFramesEveryNthFrame(step)
     for (let i = 0; i < muxCount; i++) {
       const { chunk } = pendingPackets[i]
       const data = new Uint8Array(chunk.byteLength); chunk.copyTo(data)
       const ts = i * step; const dur = step
-      const pkt = new EncodedPacket(data, i === 0 || chunk.type === 'key' ? 'key' : 'delta', ts, dur)
+      // WebKit labels requested keyframes as delta chunks, so trust the
+      // request cadence over chunk.type when marking sync samples
+      const pkt = new EncodedPacket(data, i % keyFrameEvery === 0 || chunk.type === 'key' ? 'key' : 'delta', ts, dur)
       await videoTrack.add(pkt, { decoderConfig: { codec: encoder.config.codec, codedWidth: targetWidth, codedHeight: targetHeight, description: codecDesc } })
     }
   }

data/index.js

@@ -15,6 +15,7 @@ const TARGET_AUDIO_BITRATE = 96_000
 const TARGET_AUDIO_SR = 48_000
 const TARGET_AUDIO_CHANNELS = 2
 const MAX_VIDEO_ENCODER_QUEUE_SIZE = 4
+const KEY_FRAME_INTERVAL_SECONDS = 2
 
 // ----- Video metadata probe -----
 async function probeVideo (file) {
@@ -205,6 +206,14 @@ function shouldDecodeViaVideoElement () {
   return (navigator?.vendor || '').includes('Apple')
 }
 
+// Encoders only emit keyframes when asked (WebKit's VideoToolbox never adds
+// its own), so request one every KEY_FRAME_INTERVAL_SECONDS or players get a
+// single sync sample for the whole video and cannot recover from seeks or
+// dropped frames.
+function keyFramesEveryNthFrame (step) {
+  return Math.max(1, Math.round(KEY_FRAME_INTERVAL_SECONDS / step))
+}
+
 async function applyVideoEncoderBackpressure (encoder) {
   while (encoder.encodeQueueSize > MAX_VIDEO_ENCODER_QUEUE_SIZE) {
     await new Promise(resolve => setTimeout(resolve, 0))
@@ -259,6 +268,7 @@ async function encodeFramesViaVideoElement ({ file, durationCfr, step, frames, c
     }
   })
 
+  const keyFrameEvery = keyFramesEveryNthFrame(step)
   for (let i = 0; i < frames; i++) {
     const t = i * step
     const drawTime = Math.min(Math.max(0, t + (step * 0.5)), Math.max(0.000001, durationCfr - 0.000001))
@@ -273,7 +283,7 @@ async function encodeFramesViaVideoElement ({ file, durationCfr, step, frames, c
 
     ctx.drawImage(v, 0, 0, canvas.width, canvas.height)
     const vf = new VideoFrame(canvas, { timestamp: Math.round(t * 1e6), duration: Math.round(step * 1e6) })
-    ve.encode(vf, { keyFrame: i === 0 })
+    ve.encode(vf, { keyFrame: i % keyFrameEvery === 0 })
     vf.close()
     await applyVideoEncoderBackpressure(ve)
 
@@ -303,6 +313,7 @@ async function encodeFramesViaVideoSampleSink ({ file, durationCfr, step, frames
     sample.drawWithFit(ctx, { fit: 'fill' })
   }
 
+  const keyFrameEvery = keyFramesEveryNthFrame(step)
   let i = 0
   let prev = null
   let prevStart = 0
@@ -320,7 +331,7 @@ async function encodeFramesViaVideoSampleSink ({ file, durationCfr, step, frames
         if (displayTime < prevStart || displayTime >= end) break
         const t = i * step
         const vf = new VideoFrame(canvas, { timestamp: Math.round(t * 1e6), duration: Math.round(step * 1e6) })
-        ve.encode(vf, { keyFrame: i === 0 })
+        ve.encode(vf, { keyFrame: i % keyFrameEvery === 0 })
         vf.close()
         await applyVideoEncoderBackpressure(ve)
 
@@ -347,7 +358,7 @@ async function encodeFramesViaVideoSampleSink ({ file, durationCfr, step, frames
     while (i < frames) {
       const t = i * step
       const vf = new VideoFrame(canvas, { timestamp: Math.round(t * 1e6), duration: Math.round(step * 1e6) })
-      ve.encode(vf, { keyFrame: i === 0 })
+      ve.encode(vf, { keyFrame: i % keyFrameEvery === 0 })
       vf.close()
       await applyVideoEncoderBackpressure(ve)
 
@@ -497,6 +508,7 @@ function _extractTrack (u8, dv, trakBox) {
   const stszBox = stblKids.find(b => b.type === 'stsz')
   const stcoBox = stblKids.find(b => b.type === 'stco')
   const stssBox = stblKids.find(b => b.type === 'stss')
+  const cttsBox = stblKids.find(b => b.type === 'ctts')
   if (!stsdBox || !sttsBox || !stscBox || !stszBox || !stcoBox) return null
 
   const sampleCount = dv.getUint32(stszBox.offset + 16)
@@ -508,6 +520,9 @@ function _extractTrack (u8, dv, trakBox) {
   const stscBody = u8.slice(stscBox.offset + 12, stscBox.offset + stscBox.size)
   const stszBody = u8.slice(stszBox.offset + 12, stszBox.offset + stszBox.size)
   const stssBody = stssBox ? u8.slice(stssBox.offset + 12, stssBox.offset + stssBox.size) : null
+  // Copied whole (header included) to keep its version: mediabunny writes
+  // version 1 (signed offsets), which _full would misdeclare as version 0
+  const cttsRaw = cttsBox ? u8.slice(cttsBox.offset, cttsBox.offset + cttsBox.size) : null
 
   const entryOff = stsdBox.offset + 16
   const entrySize = dv.getUint32(entryOff)
@@ -545,7 +560,7 @@ function _extractTrack (u8, dv, trakBox) {
     tkDur, tkW, tkH, mdTs, mdDur, mdLang,
     videoCodecConfig, audioSpecificConfig,
     audioChannels, audioSampleSize, audioSampleRate,
-    sttsBody, stscBody, stszBody, stssBody,
+    sttsBody, stscBody, stszBody, stssBody, cttsRaw,
     stcoEntries, sampleCount, bitrate
   }
 }
@@ -614,6 +629,7 @@ function _buildTrak (t) {
   const stss = t.stssBody ? _full('stss', 0, 0, t.stssBody) : null
 
   const stblParts = [stsd, stts]
+  if (t.cttsRaw) stblParts.push(t.cttsRaw)
   if (stss) stblParts.push(stss)
   stblParts.push(stsc, stsz, stco)
   if (t.isAudio) {
@@ -792,11 +808,14 @@ async function encodeVideo ({ file, srcMeta, plan, onProgress }) {
 
     const muxCount = Math.min(frames, pendingPackets.length)
     videoDuration = muxCount * step
+    const keyFrameEvery = keyFramesEveryNthFrame(step)
     for (let i = 0; i < muxCount; i++) {
       const { chunk } = pendingPackets[i]
       const data = new Uint8Array(chunk.byteLength); chunk.copyTo(data)
       const ts = i * step; const dur = step
-      const pkt = new EncodedPacket(data, i === 0 || chunk.type === 'key' ? 'key' : 'delta', ts, dur)
+      // WebKit labels requested keyframes as delta chunks, so trust the
+      // request cadence over chunk.type when marking sync samples
+      const pkt = new EncodedPacket(data, i % keyFrameEvery === 0 || chunk.type === 'key' ? 'key' : 'delta', ts, dur)
       await videoTrack.add(pkt, { decoderConfig: { codec: encoder.config.codec, codedWidth: targetWidth, codedHeight: targetHeight, description: codecDesc } })
     }
   }

data/lib/straight_to_video/version.rb

@@ -1,3 +1,3 @@
 module StraightToVideo
-  VERSION = "0.0.13"
+  VERSION = "0.0.14"
 end

data/package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "straight-to-video",
-  "version": "0.0.13",
+  "version": "0.0.14",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "straight-to-video",
-      "version": "0.0.13",
+      "version": "0.0.14",
       "license": "MIT",
       "dependencies": {
         "mediabunny": "^1.27.3"

data/package.json

@@ -1,6 +1,6 @@
 {
   "name": "straight-to-video",
-  "version": "0.0.13",
+  "version": "0.0.14",
   "description": "Browser-based, hardware-accelerated video upload optimization",
   "type": "module",
   "exports": {

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: straight_to_video
 version: !ruby/object:Gem::Version
-  version: 0.0.13
+  version: 0.0.14
 platform: ruby
 authors:
 - Justin Searls