@kromatv/sdk 0.1.39-canary.3579809

package/rust/kroma-media/src/hls/progress.rs

@@ -0,0 +1,157 @@
+//! What a live remux says about itself.
+//!
+//! ffmpeg writes a key=value block to `-progress` once a second. `speed` is the
+//! number the operator has no other way to get: below 1.0x the encoder is
+//! falling behind the film and the player will run out of segments before the
+//! scene ends, which from the outside looks like a network fault. Everything
+//! else here is context for that one figure.
+//!
+//! The file is append-only and every block repeats every key, so the newest
+//! block is the last occurrence of each key before the final `progress=` line.
+//! Only the tail is read: a two-hour session writes about a megabyte of these.
+
+use std::io::{Read, Seek, SeekFrom};
+use std::path::Path;
+
+/// Where `spawn_stream` points `-progress`, relative to the session directory.
+pub(super) const FILE: &str = "progress";
+
+// Comfortably more than one block, which runs to about 150 bytes.
+const TAIL: u64 = 4096;
+
+#[derive(Clone, Copy, Default, Debug, PartialEq)]
+pub(super) struct Progress {
+    pub frames: u64,
+    pub fps: f32,
+    // Multiple of realtime. `-readrate` caps it, so a session that is keeping up
+    // sits at the read rate rather than climbing past it.
+    pub speed: f32,
+    pub out_time_ms: u64,
+    pub dropped: u64,
+    pub ended: bool,
+}
+
+/// The newest block in `dir`'s progress file, or None before ffmpeg has written one.
+pub(super) fn read(dir: &Path) -> Option<Progress> {
+    parse(&tail(&dir.join(FILE))?)
+}
+
+fn tail(path: &Path) -> Option<Vec<u8>> {
+    let mut file = std::fs::File::open(path).ok()?;
+    let len = file.metadata().ok()?.len();
+    file.seek(SeekFrom::Start(len.saturating_sub(TAIL))).ok()?;
+    let mut buf = Vec::new();
+    file.read_to_end(&mut buf).ok()?;
+    Some(buf)
+}
+
+fn parse(tail: &[u8]) -> Option<Progress> {
+    let text = String::from_utf8_lossy(tail);
+    let end = text.rfind("progress=")?;
+    let mut out = Progress {
+        ended: text[end..].starts_with("progress=end"),
+        ..Progress::default()
+    };
+    for line in text[..end].lines() {
+        let Some((key, value)) = line.split_once('=') else {
+            continue;
+        };
+        let value = value.trim();
+        match key.trim() {
+            "frame" => out.frames = value.parse().unwrap_or(out.frames),
+            "fps" => out.fps = value.parse().unwrap_or(out.fps),
+            "speed" => {
+                out.speed = value.trim_end_matches('x').parse().unwrap_or(out.speed);
+            }
+            // Under `-copyts` this starts at the anchor, and ffmpeg spells an
+            // unset clock as a large negative rather than as `N/A`.
+            "out_time_us" => {
+                if let Ok(us) = value.parse::<i64>() {
+                    out.out_time_ms = (us.max(0) / 1000) as u64;
+                }
+            }
+            "drop_frames" => out.dropped = value.parse().unwrap_or(out.dropped),
+            _ => {}
+        }
+    }
+    Some(out)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    const BLOCK: &str = "frame=120\nfps=24.50\nstream_0_0_q=-0.0\nbitrate=N/A\n\
+                         total_size=N/A\nout_time_us=5000000\nout_time_ms=5000000\n\
+                         out_time=00:00:05.000000\ndup_frames=0\ndrop_frames=3\n\
+                         speed=1.98x\nprogress=continue\n";
+
+    #[test]
+    fn reads_the_rate_the_encoder_is_actually_holding() {
+        let p = parse(BLOCK.as_bytes()).expect("a written block");
+
+        assert_eq!(p.frames, 120);
+        assert!((p.fps - 24.5).abs() < 1e-3);
+        assert!((p.speed - 1.98).abs() < 1e-3);
+        assert_eq!(p.out_time_ms, 5_000);
+        assert_eq!(p.dropped, 3);
+        assert!(!p.ended);
+    }
+
+    #[test]
+    fn the_newest_block_wins_over_every_earlier_one() {
+        let older = BLOCK
+            .replace("frame=120", "frame=10")
+            .replace("speed=1.98x", "speed=0.30x");
+
+        let p = parse(format!("{older}{BLOCK}").as_bytes()).expect("two blocks");
+
+        assert_eq!(p.frames, 120);
+        assert!((p.speed - 1.98).abs() < 1e-3);
+    }
+
+    // The tail read starts at a byte offset, so the first block it sees is
+    // usually cut in half.
+    #[test]
+    fn a_block_the_tail_read_cut_in_half_does_not_corrupt_the_answer() {
+        let cut = format!("_us=999\nspeed=0.10x\nprogress=continue\n{BLOCK}");
+
+        let p = parse(cut.as_bytes()).expect("a whole block after a fragment");
+
+        assert_eq!(p.frames, 120);
+        assert!((p.speed - 1.98).abs() < 1e-3);
+    }
+
+    #[test]
+    fn the_last_block_of_a_finished_run_says_so() {
+        let done = BLOCK.replace("progress=continue", "progress=end");
+
+        assert!(parse(done.as_bytes()).expect("a final block").ended);
+    }
+
+    #[test]
+    fn values_ffmpeg_has_not_computed_yet_leave_the_field_alone() {
+        let early = "frame=0\nfps=0.00\nbitrate=N/A\ntotal_size=N/A\n\
+                     out_time_us=-577014400000\nspeed=N/A\nprogress=continue\n";
+
+        let p = parse(early.as_bytes()).expect("an opening block");
+
+        assert_eq!(p.speed, 0.0);
+        assert_eq!(p.out_time_ms, 0, "a clock ffmpeg has not set reads as zero");
+    }
+
+    #[test]
+    fn nothing_is_reported_before_the_first_block_lands() {
+        assert_eq!(parse(b"frame=1\nfps=2\n"), None);
+        assert_eq!(parse(b""), None);
+    }
+
+    #[test]
+    fn reads_from_the_session_directory() {
+        let dir = kroma_testing::temp_dir("hls-progress");
+        std::fs::write(dir.path().join(FILE), BLOCK).expect("progress file");
+
+        assert_eq!(read(dir.path()).expect("a block").frames, 120);
+        assert_eq!(read(&dir.path().join("nope")), None);
+    }
+}

package/rust/kroma-media/src/hls/reclaim/tests.rs

@@ -0,0 +1,235 @@
+use super::super::naming::session_dir;
+use super::super::session::testing::{fill, keys, registry, registry_with_budget, LIVE};
+use super::*;
+
+const QUIET: Duration = Duration::from_secs(BUDGET_GRACE.as_secs() + 5);
+const DEAD: Duration = Duration::from_secs(IDLE_TIMEOUT.as_secs() + 30);
+
+#[tokio::test]
+async fn the_cap_takes_the_oldest_session_that_has_gone_quiet() {
+    let (s, _dir) = registry(
+        "quiet",
+        2,
+        &[("itA:copy:0:a0", QUIET), ("itB:aac:0:a0", LIVE)],
+    )
+    .await;
+
+    {
+        let mut map = s.inner.lock().await;
+        s.make_room(&mut map).await;
+    }
+
+    assert_eq!(keys(&s).await, ["itB:aac:0:a0"]);
+    assert!(!s.root.join(session_dir("itA:copy:0:a0")).exists());
+}
+
+#[tokio::test]
+async fn the_cap_never_takes_a_session_someone_is_reading() {
+    let (s, _dir) = registry(
+        "cap-live",
+        2,
+        &[
+            ("itA:copy:0:a0", Duration::from_secs(3)),
+            ("itB:aac:0:a0", LIVE),
+        ],
+    )
+    .await;
+
+    {
+        let mut map = s.inner.lock().await;
+        s.make_room(&mut map).await;
+    }
+
+    assert_eq!(keys(&s).await, ["itA:copy:0:a0", "itB:aac:0:a0"]);
+}
+
+#[tokio::test]
+async fn superseded_siblings_are_reclaimed_only_once_quiet() {
+    let (s, _dir) = registry(
+        "supersede",
+        8,
+        &[
+            ("itA:copy:0:a0", QUIET), // the same program, gone quiet: superseded
+            ("itA:aac:600:a0", LIVE), // the same program but still being read
+            ("itA:copy:0:a1", QUIET), // another language track = another program
+            ("itB:copy:0:a0", QUIET), // another title
+        ],
+    )
+    .await;
+    {
+        let mut map = s.inner.lock().await;
+        s.reap_superseded(&mut map, "itA:aac-night:900:a0").await;
+    }
+    assert_eq!(
+        keys(&s).await,
+        ["itA:aac:600:a0", "itA:copy:0:a1", "itB:copy:0:a0"]
+    );
+    assert!(!s.root.join(session_dir("itA:copy:0:a0")).exists());
+}
+
+#[tokio::test]
+async fn a_sibling_another_viewer_is_reading_survives_a_seek_that_supersedes_it() {
+    let (s, _dir) = registry("arriving", 8, &[("itA:copy:0:a0", LIVE)]).await;
+
+    {
+        let mut map = s.inner.lock().await;
+        s.reap_superseded(&mut map, "itA:copy:600:a0").await;
+    }
+
+    assert_eq!(keys(&s).await, ["itA:copy:0:a0"]);
+}
+
+#[tokio::test]
+async fn a_superseded_session_is_taken_a_grace_later_rather_than_a_whole_idle_timeout() {
+    let (s, _dir) = registry("superseded-grace", 8, &[("itA:copy:0:a0", LIVE)]).await;
+    {
+        let mut map = s.inner.lock().await;
+        s.reap_superseded(&mut map, "itA:copy:600:a0").await;
+        *map.get("itA:copy:0:a0")
+            .expect("seeded")
+            .last_access
+            .lock()
+            .await = Instant::now() - QUIET;
+    }
+
+    s.reap_once().await;
+
+    assert!(keys(&s).await.is_empty());
+}
+
+#[tokio::test]
+async fn a_superseded_session_read_again_keeps_the_ordinary_idle_life() {
+    let (s, _dir) = registry("superseded-revived", 8, &[("itA:copy:0:a0", LIVE)]).await;
+    {
+        let mut map = s.inner.lock().await;
+        s.reap_superseded(&mut map, "itA:copy:600:a0").await;
+        let session = map.get("itA:copy:0:a0").expect("seeded");
+        session.keep();
+        *session.last_access.lock().await = Instant::now() - QUIET;
+    }
+
+    s.reap_once().await;
+
+    assert_eq!(keys(&s).await, ["itA:copy:0:a0"]);
+}
+
+#[tokio::test]
+async fn the_budget_is_enforced_against_what_survives_the_reap() {
+    let (s, _dir) = registry_with_budget(
+        "budget-order",
+        8,
+        500,
+        &[
+            ("itA:copy:0:a0", DEAD),
+            ("itB:copy:0:a0", QUIET),
+            ("itC:copy:0:a0", QUIET),
+        ],
+    )
+    .await;
+    fill(&s, "itA:copy:0:a0", 1000);
+    fill(&s, "itB:copy:0:a0", 100);
+    fill(&s, "itC:copy:0:a0", 100);
+
+    s.reap_once().await;
+
+    assert_eq!(keys(&s).await, ["itB:copy:0:a0", "itC:copy:0:a0"]);
+}
+
+#[tokio::test]
+async fn the_budget_still_trims_when_the_survivors_alone_exceed_it() {
+    let (s, _dir) = registry_with_budget(
+        "budget-trim",
+        8,
+        500,
+        &[("itA:copy:0:a0", QUIET), ("itB:copy:0:a0", LIVE)],
+    )
+    .await;
+    fill(&s, "itA:copy:0:a0", 400);
+    fill(&s, "itB:copy:0:a0", 400);
+
+    s.reap_once().await;
+
+    assert_eq!(
+        keys(&s).await,
+        ["itB:copy:0:a0"],
+        "the quiet one goes, the live one stays"
+    );
+}
+
+// The idle list is drawn up, the lock is released for the pruning, and the
+// viewer presses play in that window.
+#[tokio::test]
+async fn a_session_revived_before_the_eviction_lands_is_spared() {
+    let (s, _dir) = registry("revived", 8, &[("itA:copy:0:a0", DEAD)]).await;
+    {
+        let mut map = s.inner.lock().await;
+        *map.get("itA:copy:0:a0")
+            .expect("seeded")
+            .last_access
+            .lock()
+            .await = Instant::now();
+        s.evict_if_still_idle(&mut map, "itA:copy:0:a0").await;
+    }
+    assert_eq!(keys(&s).await, ["itA:copy:0:a0"]);
+    assert!(
+        s.root.join(session_dir("itA:copy:0:a0")).exists(),
+        "its segments survive too"
+    );
+}
+
+#[tokio::test]
+async fn a_session_still_idle_when_the_eviction_lands_is_taken() {
+    let (s, _dir) = registry("still-idle", 8, &[("itA:copy:0:a0", DEAD)]).await;
+    {
+        let mut map = s.inner.lock().await;
+        s.evict_if_still_idle(&mut map, "itA:copy:0:a0").await;
+    }
+    assert!(keys(&s).await.is_empty());
+}
+
+#[tokio::test]
+async fn discarding_a_directory_frees_its_name_before_it_returns() {
+    let data = kroma_testing::temp_dir("hls-test-discard");
+    let dir = data.path().join("s");
+    std::fs::create_dir_all(&dir).expect("session dir");
+    std::fs::write(dir.join("seg_00001.m4s"), b"x").expect("segment");
+
+    discard_dir(&dir);
+
+    assert!(!dir.exists());
+}
+
+#[tokio::test]
+async fn eviction_reports_the_bytes_it_frees() {
+    let (s, _dir) = registry_with_budget("freed", 8, 500, &[("itA:copy:0:a0", QUIET)]).await;
+    fill(&s, "itA:copy:0:a0", 300);
+
+    let freed = {
+        let mut map = s.inner.lock().await;
+        s.evict(&mut map, "itA:copy:0:a0").await
+    };
+
+    assert_eq!(freed, 300);
+}
+
+#[tokio::test]
+async fn prune_dir_removes_only_what_is_behind_the_cutoff() {
+    let data = kroma_testing::temp_dir("hls-test-prune");
+    let dir = data.path().join("s");
+    std::fs::create_dir_all(&dir).expect("session dir");
+    for name in [
+        "seg_00001.m4s",
+        "seg_00009.m4s",
+        "seg_00010.m4s",
+        "init.mp4",
+        "index.m3u8",
+    ] {
+        std::fs::write(dir.join(name), b"x").expect("segment");
+    }
+    prune_dir(&dir, 10);
+    assert!(!dir.join("seg_00001.m4s").exists());
+    assert!(!dir.join("seg_00009.m4s").exists());
+    assert!(dir.join("seg_00010.m4s").exists());
+    assert!(dir.join("init.mp4").exists());
+    assert!(dir.join("index.m3u8").exists());
+}

package/rust/kroma-media/src/hls/reclaim.rs

@@ -0,0 +1,258 @@
+//! Taking sessions away: the concurrency cap, the superseded-sibling sweep, the
+//! on-disk byte budget, and the periodic reaper that also advances each live
+//! session's retention window.
+
+use std::collections::HashMap;
+use std::ffi::OsStr;
+use std::path::Path;
+use std::sync::atomic::Ordering;
+use std::sync::Arc;
+use std::time::{Duration, Instant};
+
+use tokio::time::sleep;
+
+use super::key::same_program;
+use super::naming::seg_index;
+use super::session::{Session, Sessions};
+
+const IDLE_TIMEOUT: Duration = Duration::from_secs(180);
+const REAP_INTERVAL: Duration = Duration::from_secs(30);
+// Under this age a session counts as being read: never evicted, whatever the
+// cap or the byte budget says.
+pub(super) const BUDGET_GRACE: Duration = Duration::from_secs(45);
+
+impl Sessions {
+    // The cap takes the oldest session that has gone quiet, and nothing else. The
+    // HLS routes are anonymous, so a session someone is reading cannot be told
+    // from one they have left, and a cap that guessed wrong would end one
+    // viewer's film to start another's.
+    pub(super) async fn make_room(&self, map: &mut HashMap<String, Arc<Session>>) {
+        let mut freed = 0;
+        while map.len() >= self.cap {
+            let Some((oldest, la)) = lru(map.iter()).await else {
+                break;
+            };
+            if Instant::now().duration_since(la) < BUDGET_GRACE {
+                break; // the oldest is being read, so every one of them is
+            }
+            freed += self.evict(map, &oldest).await;
+        }
+        // `bytes()` walks the whole cache and this runs with the sessions lock
+        // held, so an unlimited budget must not pay for a figure nothing reads.
+        if self.budget_bytes() != 0 {
+            let total = self.bytes().saturating_sub(freed);
+            self.enforce_budget(map, total).await;
+        }
+    }
+
+    pub(super) fn budget_bytes(&self) -> u64 {
+        self.budget.load(Ordering::Relaxed)
+    }
+
+    // A sibling of `key` is either the same viewer's superseded stream or another
+    // viewer still on the title, and nothing here can tell them apart. So one
+    // that has gone quiet goes now, and one still being read is only marked: the
+    // reaper takes it a grace later if nothing reads it again, and the next
+    // request for it clears the mark.
+    pub(super) async fn reap_superseded(&self, map: &mut HashMap<String, Arc<Session>>, key: &str) {
+        let now = Instant::now();
+        let mut stale = Vec::new();
+        for (k, s) in map.iter() {
+            if k == key || !same_program(k, key) {
+                continue;
+            }
+            if now.duration_since(*s.last_access.lock().await) >= BUDGET_GRACE {
+                stale.push(k.clone());
+            } else {
+                s.supersede();
+            }
+        }
+        for k in stale {
+            self.evict(map, &k).await;
+        }
+    }
+
+    // Evict idle sessions oldest-first until the cache is under budget (0
+    // disables trimming). A session younger than BUDGET_GRACE is left alone
+    // even if that briefly exceeds the budget: dropping its segments would stall it.
+    async fn enforce_budget(&self, map: &mut HashMap<String, Arc<Session>>, mut total: u64) {
+        let budget = self.budget.load(Ordering::Relaxed);
+        if budget == 0 {
+            return;
+        }
+        while total > budget && map.len() > 1 {
+            let Some((k, la)) = lru(map.iter()).await else {
+                break;
+            };
+            if Instant::now().duration_since(la) < BUDGET_GRACE {
+                break; // the oldest is live, so the rest are too
+            }
+            total = total.saturating_sub(self.evict(map, &k).await);
+        }
+    }
+
+    // Returns the bytes the session was holding, so a caller measuring the cache
+    // afterwards can discount a directory the discard has not unlinked yet.
+    async fn evict(&self, map: &mut HashMap<String, Arc<Session>>, key: &str) -> u64 {
+        let Some(s) = map.remove(key) else { return 0 };
+        let _ = s.child.lock().await.start_kill();
+        let held = if self.budget_bytes() == 0 {
+            0
+        } else {
+            dir_bytes(&s.dir)
+        };
+        discard_dir(&s.dir);
+        held
+    }
+
+    // The idle list is drawn up, the lock is released for the slow work, and by
+    // the time it is taken back a viewer may have pressed play again. Killing
+    // them then is worse than a 404: a `master()` racing this would poll a
+    // directory that no longer exists for the whole of FILE_WAIT.
+    async fn evict_if_still_idle(&self, map: &mut HashMap<String, Arc<Session>>, key: &str) -> u64 {
+        let idle = match map.get(key) {
+            Some(s) => s.idle_past_its_limit(Instant::now()).await,
+            None => return 0,
+        };
+        if idle {
+            self.evict(map, key).await
+        } else {
+            0
+        }
+    }
+
+    // Deleting segments and walking the cache are the two slow things here, and
+    // every in-flight segment request needs the same lock: neither runs under it,
+    // which on a network mount is the difference between a pause and a stall for
+    // everyone on the box.
+    async fn reap_once(&self) {
+        let now = Instant::now();
+        let (dead, plans) = {
+            let map = self.inner.lock().await;
+            let mut dead = Vec::new();
+            let mut plans = Vec::new();
+            for (id, s) in map.iter() {
+                if s.idle_past_its_limit(now).await {
+                    dead.push(id.clone());
+                } else if let Some(cutoff) = s.window.cutoff() {
+                    plans.push((s.dir.clone(), cutoff));
+                }
+            }
+            (dead, plans)
+        };
+        if !plans.is_empty() {
+            let _ = tokio::task::spawn_blocking(move || {
+                for (dir, cutoff) in plans {
+                    prune_dir(&dir, cutoff);
+                }
+            })
+            .await;
+        }
+        let mut freed = 0;
+        {
+            let mut map = self.inner.lock().await;
+            for id in dead {
+                freed += self.evict_if_still_idle(&mut map, &id).await;
+            }
+            self.serving.keep_only(&map).await;
+        }
+        // Measured AFTER the idle evictions, or the budget would be enforced
+        // against directories that no longer exist and take a live viewer with
+        // them. Outside the lock: it is a full walk of the cache.
+        if self.budget_bytes() != 0 {
+            let total = self.bytes().saturating_sub(freed);
+            let mut map = self.inner.lock().await;
+            self.enforce_budget(&mut map, total).await;
+        }
+    }
+
+    pub fn spawn_reaper(self: &Arc<Self>) {
+        let this = self.clone();
+        tokio::spawn(async move {
+            loop {
+                sleep(REAP_INTERVAL).await;
+                this.reap_once().await;
+            }
+        });
+    }
+}
+
+impl Session {
+    pub(super) fn supersede(&self) {
+        self.superseded.store(true, Ordering::Relaxed);
+    }
+
+    pub(super) fn keep(&self) {
+        self.superseded.store(false, Ordering::Relaxed);
+    }
+
+    // A superseded session has minutes of an ordinary idle life taken off it: a
+    // remux nobody re-anchored into keeps pulling its input at up to twice
+    // realtime, for a playlist nobody is reading.
+    async fn idle_past_its_limit(&self, now: Instant) -> bool {
+        let limit = match self.superseded.load(Ordering::Relaxed) {
+            true => BUDGET_GRACE,
+            false => IDLE_TIMEOUT,
+        };
+        now.duration_since(*self.last_access.lock().await) > limit
+    }
+}
+
+async fn lru<'a>(
+    sessions: impl Iterator<Item = (&'a String, &'a Arc<Session>)>,
+) -> Option<(String, Instant)> {
+    let mut victim: Option<(String, Instant)> = None;
+    for (k, s) in sessions {
+        let la = *s.last_access.lock().await;
+        match &victim {
+            Some((_, t)) if *t <= la => {}
+            _ => victim = Some((k.clone(), la)),
+        }
+    }
+    victim
+}
+
+// Renaming is one metadata operation; the recursive delete behind it is a walk,
+// so that runs on a blocking thread. The session's own name is free the moment
+// this returns, so a key played again gets a clean directory rather than one the
+// pending delete is about to empty.
+pub(super) fn discard_dir(dir: &Path) {
+    let Some(name) = dir.file_name().and_then(OsStr::to_str) else {
+        return;
+    };
+    let doomed = dir.with_file_name(format!("{name}.gone-{}", kroma_primitives::random_token()));
+    if std::fs::rename(dir, &doomed).is_err() {
+        return;
+    }
+    tokio::task::spawn_blocking(move || {
+        let _ = std::fs::remove_dir_all(&doomed);
+    });
+}
+
+// The playlist keeps listing pruned entries; a request for one 404s rather than
+// waiting, and the client re-anchors.
+fn prune_dir(dir: &Path, cutoff: u64) {
+    let Ok(entries) = std::fs::read_dir(dir) else {
+        return;
+    };
+    for entry in entries.flatten() {
+        if let Some(idx) = entry.file_name().to_str().and_then(seg_index) {
+            if idx < cutoff {
+                let _ = std::fs::remove_file(entry.path());
+            }
+        }
+    }
+}
+
+pub(super) fn dir_bytes(dir: &Path) -> u64 {
+    walkdir::WalkDir::new(dir)
+        .into_iter()
+        .filter_map(Result::ok)
+        .filter(|e| e.file_type().is_file())
+        .filter_map(|e| e.metadata().ok())
+        .map(|m| m.len())
+        .sum()
+}
+
+#[cfg(test)]
+mod tests;