@kromatv/sdk 0.1.39-canary.3579809

package/rust/kroma-media/src/stream.rs

@@ -0,0 +1,262 @@
+//! HTTP range streaming of original media files. The server never transcodes
+//! it just serves bytes, honouring the `Range` header so clients can seek.
+
+use std::io::SeekFrom;
+use std::path::Path;
+use std::pin::Pin;
+use std::task::{Context, Poll};
+
+use axum::body::Body;
+use axum::http::{header, HeaderMap, HeaderValue, StatusCode};
+use axum::response::Response;
+use tokio::fs::File;
+use tokio::io::{AsyncRead, AsyncReadExt, AsyncSeek, AsyncSeekExt, ReadBuf};
+use tokio_util::io::ReaderStream;
+
+use crate::json_error::json_error;
+
+use crate::ByteMeter;
+
+const CHUNK_BYTES: usize = 64 * 1024;
+
+/// Meters every byte handed to the response body into a [`ByteMeter`]. Socket
+/// backpressure drives `poll_read`, so it counts bytes delivered, not read ahead.
+pub struct CountingReader<R, M> {
+    inner: R,
+    meter: M,
+}
+
+impl<R, M> CountingReader<R, M> {
+    pub fn new(inner: R, meter: M) -> Self {
+        Self { inner, meter }
+    }
+}
+
+impl<R: AsyncRead + Unpin, M: ByteMeter> AsyncRead for CountingReader<R, M> {
+    fn poll_read(
+        mut self: Pin<&mut Self>,
+        cx: &mut Context<'_>,
+        buf: &mut ReadBuf<'_>,
+    ) -> Poll<std::io::Result<()>> {
+        let before = buf.filled().len();
+        let poll = Pin::new(&mut self.inner).poll_read(cx, buf);
+        if let Poll::Ready(Ok(())) = &poll {
+            let n = buf.filled().len() - before;
+            if n > 0 {
+                self.meter.add(n as u64);
+            }
+        }
+        poll
+    }
+}
+
+/// A response body that meters into `meter` through a [`CountingReader`], so a
+/// body the client abandons counts only what it took.
+pub fn metered_body<R, M>(reader: R, meter: M) -> Body
+where
+    R: AsyncRead + Unpin + Send + 'static,
+    M: ByteMeter,
+{
+    Body::from_stream(ReaderStream::with_capacity(
+        CountingReader::new(reader, meter),
+        CHUNK_BYTES,
+    ))
+}
+
+/// Honours `Range: bytes=start-end`: `206 Partial Content` with a
+/// `Content-Range` when satisfiable, else a full `200 OK`. Never buffers the
+/// whole file. Pass [`crate::Unmetered`] to opt out of bandwidth metering.
+pub async fn stream_file<M: ByteMeter>(path: &Path, req_headers: &HeaderMap, meter: M) -> Response {
+    let Ok(file) = File::open(path).await else {
+        return json_error(StatusCode::NOT_FOUND, "media file not found on disk");
+    };
+    let Ok(metadata) = file.metadata().await else {
+        return json_error(
+            StatusCode::INTERNAL_SERVER_ERROR,
+            "could not stat media file",
+        );
+    };
+    stream_reader(
+        file,
+        metadata.len(),
+        content_type_for(path),
+        req_headers,
+        meter,
+    )
+    .await
+}
+
+/// [`stream_file`] over any seekable reader of `len` bytes.
+async fn stream_reader<R, M>(
+    mut reader: R,
+    len: u64,
+    content_type: &str,
+    req_headers: &HeaderMap,
+    meter: M,
+) -> Response
+where
+    R: AsyncRead + AsyncSeek + Unpin + Send + 'static,
+    M: ByteMeter,
+{
+    match parse_range(req_headers, len) {
+        RangeOutcome::Full => full_response(reader, len, content_type, meter),
+        RangeOutcome::Partial { start, end } => {
+            if reader.seek(SeekFrom::Start(start)).await.is_err() {
+                return json_error(
+                    StatusCode::INTERNAL_SERVER_ERROR,
+                    "could not seek media file",
+                );
+            }
+            partial_response(reader, start, end, len, content_type, meter)
+        }
+        RangeOutcome::Unsatisfiable => {
+            let mut resp = json_error(
+                StatusCode::RANGE_NOT_SATISFIABLE,
+                "requested range not satisfiable",
+            );
+            resp.headers_mut().insert(
+                header::CONTENT_RANGE,
+                HeaderValue::from_str(&format!("bytes */{}", len))
+                    .unwrap_or(HeaderValue::from_static("bytes */0")),
+            );
+            resp
+        }
+    }
+}
+
+fn full_response<R: AsyncRead + Unpin + Send + 'static, M: ByteMeter>(
+    reader: R,
+    total_size: u64,
+    content_type: &str,
+    meter: M,
+) -> Response {
+    let mut resp = Response::new(metered_body(reader, meter));
+    let headers = resp.headers_mut();
+    set_common_headers(headers, content_type);
+    headers.insert(header::CONTENT_LENGTH, HeaderValue::from(total_size));
+    resp
+}
+
+// `[start, end]` inclusive, per RFC 7233.
+fn partial_response<R: AsyncRead + Unpin + Send + 'static, M: ByteMeter>(
+    reader: R,
+    start: u64,
+    end: u64,
+    total_size: u64,
+    content_type: &str,
+    meter: M,
+) -> Response {
+    let length = end - start + 1;
+    let mut resp = Response::new(metered_body(reader.take(length), meter));
+    *resp.status_mut() = StatusCode::PARTIAL_CONTENT;
+    let headers = resp.headers_mut();
+    set_common_headers(headers, content_type);
+    headers.insert(header::CONTENT_LENGTH, HeaderValue::from(length));
+    headers.insert(
+        header::CONTENT_RANGE,
+        HeaderValue::from_str(&format!("bytes {}-{}/{}", start, end, total_size))
+            .unwrap_or(HeaderValue::from_static("bytes 0-0/0")),
+    );
+    resp
+}
+
+fn set_common_headers(headers: &mut HeaderMap, content_type: &str) {
+    headers.insert(
+        header::CONTENT_TYPE,
+        HeaderValue::from_str(content_type)
+            .unwrap_or(HeaderValue::from_static("application/octet-stream")),
+    );
+    headers.insert(header::ACCEPT_RANGES, HeaderValue::from_static("bytes"));
+}
+
+enum RangeOutcome {
+    Full,
+    Partial { start: u64, end: u64 },
+    Unsatisfiable,
+}
+
+fn parse_range(headers: &HeaderMap, total_size: u64) -> RangeOutcome {
+    let raw = match headers.get(header::RANGE).and_then(|v| v.to_str().ok()) {
+        Some(r) => r,
+        None => return RangeOutcome::Full,
+    };
+
+    let spec = match raw.strip_prefix("bytes=") {
+        Some(s) => s.trim(),
+        None => return RangeOutcome::Full,
+    };
+
+    // Reject multi-range; serve the whole file instead of erroring.
+    if spec.contains(',') {
+        return RangeOutcome::Full;
+    }
+
+    if total_size == 0 {
+        return RangeOutcome::Unsatisfiable;
+    }
+
+    let (start_str, end_str) = match spec.split_once('-') {
+        Some(parts) => parts,
+        None => return RangeOutcome::Full,
+    };
+
+    let last = total_size - 1;
+
+    if start_str.is_empty() {
+        // Suffix range: bytes=-N → last N bytes.
+        let n: u64 = match end_str.parse() {
+            Ok(n) if n > 0 => n,
+            _ => return RangeOutcome::Unsatisfiable,
+        };
+        let start = total_size.saturating_sub(n);
+        return RangeOutcome::Partial { start, end: last };
+    }
+
+    let start: u64 = match start_str.parse() {
+        Ok(s) => s,
+        Err(_) => return RangeOutcome::Unsatisfiable,
+    };
+
+    if start > last {
+        return RangeOutcome::Unsatisfiable;
+    }
+
+    let end: u64 = if end_str.is_empty() {
+        last
+    } else {
+        match end_str.parse::<u64>() {
+            Ok(e) => e.min(last),
+            Err(_) => return RangeOutcome::Unsatisfiable,
+        }
+    };
+
+    if end < start {
+        return RangeOutcome::Unsatisfiable;
+    }
+
+    RangeOutcome::Partial { start, end }
+}
+
+fn content_type_for(path: &Path) -> &'static str {
+    let ext = path
+        .extension()
+        .and_then(std::ffi::OsStr::to_str)
+        .unwrap_or("")
+        .to_ascii_lowercase();
+
+    match ext.as_str() {
+        "mp4" | "m4v" => "video/mp4",
+        "mkv" => "video/x-matroska",
+        "webm" => "video/webm",
+        "mov" => "video/quicktime",
+        "avi" => "video/x-msvideo",
+        "ts" => "video/mp2t",
+        "mp3" => "audio/mpeg",
+        "m4a" => "audio/mp4",
+        "ogg" | "oga" => "audio/ogg",
+        _ => "application/octet-stream",
+    }
+}
+
+#[cfg(test)]
+mod tests;

package/rust/kroma-media/src/subtitles.rs

@@ -0,0 +1,140 @@
+//! Text subtitle streams of one input, written out as WebVTT.
+
+use std::path::PathBuf;
+use std::process::Command;
+use std::time::Duration;
+
+use crate::tmp_file::{finalize, unique_tmp};
+
+const READ_BYTES_PER_SEC: u64 = 20 * 1024 * 1024;
+const SHORTEST_BUDGET: u64 = 150;
+const LONGEST_BUDGET: u64 = 7200;
+
+/// Whether a normalised codec can become WebVTT. A picture subtitle (PGS,
+/// VobSub, DVB) is a bitmap and never can.
+pub fn is_text_codec(codec: &str) -> bool {
+    matches!(
+        codec,
+        "subrip" | "srt" | "ass" | "ssa" | "mov_text" | "webvtt" | "vtt"
+    )
+}
+
+/// The wall-clock budget one whole-file pass over `input` gets: its size at
+/// 20 MB/s, held between 150 s and two hours. A text-subtitle demux is
+/// bandwidth-bound, and a fixed ceiling times a 4K remux out and throws away
+/// the partial output, so the next attempt reads it all again.
+pub fn timeout_for(input: &str) -> Duration {
+    let size = std::fs::metadata(input).map(|m| m.len()).unwrap_or(0);
+    let secs = (size / READ_BYTES_PER_SEC).clamp(SHORTEST_BUDGET, LONGEST_BUDGET);
+    Duration::from_secs(secs)
+}
+
+/// One ffmpeg pass writing each of `tracks` (a subtitle-relative index and
+/// where its WebVTT goes) out of `input`, so N tracks cost one read and not N.
+/// Each output lands atomically, and the count is how many did. Blocking; run
+/// it on a blocking thread.
+pub fn extract_vtt(
+    input: &str,
+    tracks: &[(usize, PathBuf)],
+    timeout: Duration,
+    cancel: &dyn Fn() -> bool,
+) -> Result<usize, String> {
+    if tracks.is_empty() {
+        return Ok(0);
+    }
+    if let Some(dir) = tracks[0].1.parent() {
+        std::fs::create_dir_all(dir)
+            .map_err(|e| format!("could not create the subtitle output dir: {e}"))?;
+    }
+    let tmps: Vec<PathBuf> = tracks.iter().map(|(_, out)| unique_tmp(out)).collect();
+    let outcome =
+        crate::ffmpeg_run::run_capturing_cancellable(pass(input, tracks, &tmps), timeout, cancel);
+    let mut landed = 0;
+    for ((_, out), tmp) in tracks.iter().zip(&tmps) {
+        match outcome.is_ok() && wrote_anything(tmp) && finalize(tmp, out) {
+            true => landed += 1,
+            false => drop(std::fs::remove_file(tmp)),
+        }
+    }
+    match outcome {
+        Ok(()) => Ok(landed),
+        Err(reason) => Err(reason),
+    }
+}
+
+// Text-subtitle decode is trivial; the cost is the demux read, so one thread
+// keeps this from competing with a live remux for cores.
+fn pass(input: &str, tracks: &[(usize, PathBuf)], tmps: &[PathBuf]) -> Command {
+    let mut cmd = Command::new("ffmpeg");
+    cmd.args(["-v", "error", "-nostdin", "-threads", "1", "-y"]);
+    cmd.arg("-i").arg(input);
+    for ((track, _), tmp) in tracks.iter().zip(tmps) {
+        cmd.arg("-map").arg(format!("0:s:{track}"));
+        cmd.args(["-f", "webvtt"]).arg(tmp);
+    }
+    cmd
+}
+
+fn wrote_anything(vtt: &std::path::Path) -> bool {
+    std::fs::metadata(vtt).is_ok_and(|m| m.len() > 0)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn worded(cmd: &Command) -> Vec<String> {
+        cmd.get_args()
+            .map(|a| a.to_string_lossy().into_owned())
+            .collect()
+    }
+
+    #[test]
+    fn a_picture_subtitle_is_never_offered_as_text() {
+        let text = ["subrip", "ass", "mov_text", "webvtt"].map(is_text_codec);
+        let picture =
+            ["pgs", "hdmv_pgs_subtitle", "dvd_subtitle", "dvb_subtitle"].map(is_text_codec);
+
+        assert_eq!(text, [true; 4]);
+        assert_eq!(picture, [false; 4]);
+    }
+
+    #[test]
+    fn a_pass_over_a_whole_input_neither_seeks_nor_bounds_what_it_reads() {
+        let tracks = [(2usize, PathBuf::from("/subs/a.vtt"))];
+
+        let args = worded(&pass("/films/one.mkv", &tracks, &[PathBuf::from("/t")]));
+
+        assert!(!args.contains(&"-ss".to_string()));
+        assert!(!args.contains(&"-to".to_string()));
+        assert_eq!(args.last().unwrap(), "/t");
+    }
+
+    #[test]
+    fn every_track_asked_for_is_mapped_into_the_one_read() {
+        let tracks = [
+            (0usize, PathBuf::from("/subs/a.vtt")),
+            (4usize, PathBuf::from("/subs/b.vtt")),
+        ];
+        let tmps = [PathBuf::from("/a.part"), PathBuf::from("/b.part")];
+
+        let args = worded(&pass("/films/one.mkv", &tracks, &tmps));
+
+        assert_eq!(args.iter().filter(|a| *a == "-map").count(), 2, "{args:?}");
+        assert!(args.contains(&"0:s:4".to_string()));
+    }
+
+    #[test]
+    fn nothing_asked_for_spawns_nothing() {
+        let ran = extract_vtt("/films/one.mkv", &[], Duration::ZERO, &|| false);
+
+        assert_eq!(ran.unwrap(), 0);
+    }
+
+    #[test]
+    fn a_budget_is_the_input_size_held_inside_the_band() {
+        let unknown = timeout_for("/films/nothing-here.mkv");
+
+        assert_eq!(unknown, Duration::from_secs(SHORTEST_BUDGET));
+    }
+}

package/rust/kroma-media/src/tmp_file.rs

@@ -0,0 +1,68 @@
+//! Writing a file something else is already serving, by way of a sibling temp.
+
+use std::path::{Path, PathBuf};
+use std::sync::atomic::{AtomicU64, Ordering};
+
+static SEQ: AtomicU64 = AtomicU64::new(0);
+
+/// A temp sibling of `out` that keeps `out`'s extension last, so ffmpeg and
+/// cwebp still infer the format from it, and that no concurrent writer of the
+/// same path can collide with.
+pub fn unique_tmp(out: &Path) -> PathBuf {
+    let seq = SEQ.fetch_add(1, Ordering::Relaxed);
+    let base = out
+        .file_name()
+        .and_then(std::ffi::OsStr::to_str)
+        .unwrap_or("out");
+    let ext = out
+        .extension()
+        .and_then(std::ffi::OsStr::to_str)
+        .unwrap_or("tmp");
+    out.with_file_name(format!("{base}.{}.{seq}.tmp.{ext}", std::process::id()))
+}
+
+/// Move a written temp onto the path readers ask for, cleaning the temp up when
+/// it cannot. Whether it landed.
+pub fn finalize(tmp: &Path, out: &Path) -> bool {
+    match std::fs::rename(tmp, out) {
+        Ok(()) => true,
+        Err(_) => {
+            let _ = std::fs::remove_file(tmp);
+            false
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn a_temp_keeps_the_extension_the_encoder_reads_the_format_from() {
+        let tmp = unique_tmp(Path::new("/cache/sheet.webp"));
+
+        assert_eq!(tmp.extension().unwrap(), "webp");
+        assert_eq!(tmp.parent().unwrap(), Path::new("/cache"));
+    }
+
+    #[test]
+    fn two_writers_of_one_path_are_given_two_temps() {
+        let out = Path::new("/cache/sheet.jpg");
+
+        let (first, second) = (unique_tmp(out), unique_tmp(out));
+
+        assert_ne!(first, second);
+    }
+
+    #[test]
+    fn a_temp_that_cannot_be_moved_is_not_left_behind() {
+        let dir = kroma_testing::temp_dir("tmp-file");
+        let tmp = dir.path().join("cue.vtt.part");
+        std::fs::write(&tmp, b"WEBVTT").unwrap();
+
+        let landed = finalize(&tmp, &dir.path().join("gone").join("cue.vtt"));
+
+        assert!(!landed);
+        assert!(!tmp.exists());
+    }
+}

package/rust/kroma-media/src/video.rs

@@ -0,0 +1,30 @@
+//! Which files hold a picture a player can open.
+
+use std::path::Path;
+
+pub const VIDEO_EXTENSIONS: &[&str] = &["mkv", "mp4", "m4v", "mov", "webm", "avi", "ts"];
+
+pub fn has_video_extension(path: &Path) -> bool {
+    path.extension()
+        .and_then(std::ffi::OsStr::to_str)
+        .is_some_and(|e| VIDEO_EXTENSIONS.contains(&e.to_ascii_lowercase().as_str()))
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn a_container_is_recognised_however_its_extension_is_cased() {
+        let held = ["/movies/Film.mkv", "/movies/Film.MP4", "clip.MOV", "a.ts"];
+
+        assert!(held.iter().all(|p| has_video_extension(Path::new(p))));
+    }
+
+    #[test]
+    fn a_file_beside_a_film_is_not_the_film() {
+        let beside = ["/movies/poster.jpg", "/movies/subs.srt", "/movies/noext"];
+
+        assert!(!beside.iter().any(|p| has_video_extension(Path::new(p))));
+    }
+}

package/rust/kroma-module-host/Cargo.toml

@@ -0,0 +1,36 @@
+[package]
+name = "kroma-module-host"
+version = "0.1.39-canary.3579809"
+edition = "2021"
+rust-version = "1.88"
+description = "The host seam: the narrow HostCtx contract + shared HTTP extractors a module crate's backend needs, without depending on the whole app (kroma-engine)."
+license = "AGPL-3.0-or-later"
+
+[features]
+# Compiles `testing::{StubHost, Recording}`, the shared HostCtx doubles. Off by
+# default so nothing reaches a release build; crates that test against the seam
+# turn it on in their dev-dependencies.
+testing = ["dep:kroma-testing", "storage", "test-serve", "kroma-sqlite/testing"]
+# `test_serve::{serve, blocking}` alone: an axum router on an ephemeral localhost
+# port. Separate from `testing` because a port contract's round-trip test needs
+# the wire and not a database.
+test-serve = []
+# The `storage` capability: `HostStorage`, the two pools, and the `query` helper.
+# Off by default, and that default is the point -- a module that declares no
+# storage must not link SQLite, whose bundled amalgamation is 1.5 MB of machine
+# code in a sidecar that never opens a database. Enabled through
+# `kroma-module-sdk/storage` by the modules that do.
+storage = ["dep:kroma-sqlite"]
+
+[dependencies]
+kroma-module-wire = { path = "../kroma-module-wire" }
+kroma-sqlite = { path = "../kroma-sqlite", optional = true }
+kroma-http = { path = "../kroma-http" }
+kroma-testing = { path = "../kroma-testing", optional = true }
+axum = { version = "0.8", features = ["macros", "ws"] }
+async-trait = { version = "0.1" }
+tokio = { version = "1", features = ["full"] }
+serde = { version = "1", features = ["derive"] }
+serde_json = { version = "1" }
+anyhow = { version = "1" }
+tracing = { version = "0.1" }