@kromatv/sdk 0.1.40-canary.0

package/rust/kroma-db/src/ingest/probe.rs

@@ -0,0 +1,171 @@
+//! What ffprobe found, and the representative columns it recomputes.
+
+use anyhow::Result;
+use rusqlite::params;
+
+use crate::Pool;
+
+/// (file_id, abs_path, owning item_id) for every file awaiting an ffprobe pass.
+pub fn unprobed_files(pool: &Pool) -> Result<Vec<(String, String, String)>> {
+    let conn = pool.get()?;
+    let mut stmt = conn.prepare("SELECT id, abs_path, item_id FROM files WHERE probed = 0")?;
+    let rows = stmt.query_map([], |r| {
+        Ok((
+            r.get::<_, String>(0)?,
+            r.get::<_, String>(1)?,
+            r.get::<_, String>(2)?,
+        ))
+    })?;
+    Ok(rows.collect::<rusqlite::Result<Vec<_>>>()?)
+}
+
+/// Item ids that are fully probed (≥1 file, none unprobed).
+pub fn probed_item_ids(pool: &Pool) -> Result<std::collections::HashSet<String>> {
+    let conn = pool.get()?;
+    let mut stmt = conn.prepare(
+        "SELECT item_id FROM files GROUP BY item_id \
+         HAVING SUM(CASE WHEN probed=0 THEN 1 ELSE 0 END)=0",
+    )?;
+    let rows = stmt.query_map([], |r| r.get::<_, String>(0))?;
+    Ok(rows.collect::<rusqlite::Result<_>>()?)
+}
+
+/// Whether every file of an item is probed (and it has at least one).
+pub fn item_probed(pool: &Pool, item_id: &str) -> Result<bool> {
+    let conn = pool.get()?;
+    let (total, unprobed): (i64, i64) = conn.query_row(
+        "SELECT COUNT(*), SUM(CASE WHEN probed=0 THEN 1 ELSE 0 END) FROM files WHERE item_id=?1",
+        params![item_id],
+        |r| Ok((r.get(0)?, r.get::<_, Option<i64>>(1)?.unwrap_or(0))),
+    )?;
+    Ok(total > 0 && unprobed == 0)
+}
+
+pub fn file_ids_for_item(pool: &Pool, item_id: &str) -> Result<Vec<String>> {
+    let conn = pool.get()?;
+    let mut stmt = conn.prepare("SELECT id FROM files WHERE item_id=?1")?;
+    let rows = stmt.query_map(params![item_id], |r| r.get::<_, String>(0))?;
+    Ok(rows.collect::<rusqlite::Result<Vec<_>>>()?)
+}
+
+pub fn unprobe_item_files(pool: &Pool, item_id: &str) -> Result<()> {
+    let conn = pool.get()?;
+    conn.execute(
+        "UPDATE files SET probed=0 WHERE item_id=?1",
+        params![item_id],
+    )?;
+    Ok(())
+}
+
+/// Every file's `(id, "mtime:size")`: a changed file gets a new signature and is
+/// re-probed.
+pub fn all_file_sigs(pool: &Pool) -> Result<Vec<(String, String)>> {
+    let conn = pool.get()?;
+    let mut stmt =
+        conn.prepare("SELECT id, COALESCE(mtime,0) || ':' || COALESCE(size,0) FROM files")?;
+    let rows = stmt.query_map([], |r| Ok((r.get::<_, String>(0)?, r.get::<_, String>(1)?)))?;
+    Ok(rows.collect::<rusqlite::Result<Vec<_>>>()?)
+}
+
+/// `(abs_path, item_id, probed)` for one file; `None` if the file row is gone.
+pub fn probe_target(pool: &Pool, file_id: &str) -> Result<Option<(String, String, bool)>> {
+    let conn = pool.get()?;
+    let mut stmt = conn.prepare("SELECT abs_path, item_id, probed FROM files WHERE id=?1")?;
+    let mut rows = stmt.query_map(params![file_id], |r| {
+        Ok((
+            r.get::<_, String>(0)?,
+            r.get::<_, String>(1)?,
+            r.get::<_, i64>(2)? != 0,
+        ))
+    })?;
+    match rows.next() {
+        Some(r) => Ok(Some(r?)),
+        None => Ok(None),
+    }
+}
+
+pub fn item_has_probed_file(pool: &Pool, item_id: &str) -> Result<bool> {
+    let conn = pool.get()?;
+    let n: i64 = conn.query_row(
+        "SELECT COUNT(*) FROM files WHERE item_id = ?1 AND probed = 1",
+        params![item_id],
+        |r| r.get(0),
+    )?;
+    Ok(n > 0)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::ingest::test_support::*;
+    use std::collections::HashMap;
+
+    #[test]
+    fn probe_helpers_over_seeded_files() {
+        let p = pool();
+        {
+            let conn = p.get().unwrap();
+            conn.execute(
+                "INSERT INTO libraries (id,name,kind,path,added_at) VALUES ('lib','L','movies','/x','t')",
+                [],
+            )
+            .unwrap();
+            for id in ["m1", "m2"] {
+                conn.execute(
+                    "INSERT INTO items (id,kind,title,container,library,added_at) VALUES (?1,'movie','T','mkv','lib','t')",
+                    params![id],
+                )
+                .unwrap();
+            }
+            // m1: one probed + one unprobed (not fully probed).
+            conn.execute(
+                "INSERT INTO files (id,item_id,abs_path,container,probed,mtime,size) VALUES ('f1','m1','/a',' ',1,100,2000)",
+                [],
+            )
+            .unwrap();
+            conn.execute(
+                "INSERT INTO files (id,item_id,abs_path,container,probed) VALUES ('f2','m1','/b','',0)",
+                [],
+            )
+            .unwrap();
+            // m2: a single probed file (fully probed).
+            conn.execute(
+                "INSERT INTO files (id,item_id,abs_path,container,probed) VALUES ('f3','m2','/c','',1)",
+                [],
+            )
+            .unwrap();
+        }
+
+        assert!(!item_probed(&p, "m1").unwrap());
+        assert!(item_probed(&p, "m2").unwrap());
+        assert!(!item_probed(&p, "missing").unwrap());
+
+        let fully = probed_item_ids(&p).unwrap();
+        assert!(fully.contains("m2") && !fully.contains("m1"));
+
+        let mut ids = file_ids_for_item(&p, "m1").unwrap();
+        ids.sort();
+        assert_eq!(ids, vec!["f1".to_string(), "f2".to_string()]);
+
+        assert_eq!(
+            probe_target(&p, "f1").unwrap(),
+            Some(("/a".to_string(), "m1".to_string(), true))
+        );
+        assert!(probe_target(&p, "gone").unwrap().is_none());
+        let sigs: HashMap<String, String> = all_file_sigs(&p).unwrap().into_iter().collect();
+        assert_eq!(sigs.get("f1").map(String::as_str), Some("100:2000"));
+        assert_eq!(sigs.get("f3").map(String::as_str), Some("0:0")); // null mtime/size
+
+        unprobe_item_files(&p, "m1").unwrap();
+        assert!(!item_has_probed_file(&p, "m1").unwrap());
+    }
+
+    #[test]
+    fn the_probe_state_readers_error_when_the_file_table_is_gone() {
+        let p = pool();
+        p.get().unwrap().execute_batch("DROP TABLE files").unwrap();
+
+        assert!(item_probed(&p, "m1").is_err());
+        assert!(item_has_probed_file(&p, "m1").is_err());
+    }
+}

package/rust/kroma-db/src/ingest/probe_result.rs

@@ -0,0 +1,178 @@
+//! Persisting one probe result, and the representative columns it moves.
+
+use anyhow::Result;
+use rusqlite::{params, Connection, OptionalExtension};
+
+use kroma_domain::{AudioStream, SubtitleTrack, VideoStream};
+
+use crate::Pool;
+
+/// Persists one file's probe result, then recomputes the owning item's representative columns.
+pub fn set_file_probe(
+    pool: &Pool,
+    file_id: &str,
+    duration_ms: Option<u64>,
+    video: Option<&VideoStream>,
+    audio: Option<&AudioStream>,
+    audio_tracks: &[AudioStream],
+    subtitles: &[SubtitleTrack],
+) -> Result<()> {
+    let conn = pool.get()?;
+    let subs = serde_json::to_string(subtitles).unwrap_or_else(|_| "[]".into());
+    let a_tracks = serde_json::to_string(audio_tracks).unwrap_or_else(|_| "[]".into());
+    conn.execute(
+        "UPDATE files SET probed=1, duration_ms=?2, \
+            v_codec=?3, v_width=?4, v_height=?5, v_hdr=?6, v_bit_depth=?7, \
+            a_codec=?8, a_channels=?9, a_language=?10, subtitles=?11, audio_tracks=?12 \
+         WHERE id = ?1",
+        params![
+            file_id,
+            duration_ms.map(|d| d as i64),
+            video.map(|v| v.codec.clone()),
+            video.and_then(|v| v.width),
+            video.and_then(|v| v.height),
+            video.map(|v| v.hdr as i64),
+            video.and_then(|v| v.bit_depth),
+            audio.map(|a| a.codec.clone()),
+            audio.and_then(|a| a.channels),
+            audio.and_then(|a| a.language.clone()),
+            subs,
+            a_tracks,
+        ],
+    )?;
+
+    let item_id: Option<String> = conn
+        .query_row(
+            "SELECT item_id FROM files WHERE id = ?1",
+            params![file_id],
+            |r| r.get(0),
+        )
+        .optional()?;
+    if let Some(item_id) = item_id {
+        recompute_item_representative(&conn, &item_id)?;
+    }
+    Ok(())
+}
+
+fn recompute_item_representative(conn: &Connection, item_id: &str) -> Result<()> {
+    let best: Option<(String, String, Option<String>, Option<i64>)> = conn
+        .query_row(
+            "SELECT abs_path, container, rel_path, duration_ms FROM files \
+             WHERE item_id = ?1 AND probed = 1 \
+             ORDER BY v_width DESC NULLS LAST, id LIMIT 1",
+            params![item_id],
+            |r| {
+                Ok((
+                    r.get::<_, String>(0)?,
+                    r.get::<_, String>(1)?,
+                    r.get::<_, Option<String>>(2)?,
+                    r.get::<_, Option<i64>>(3)?,
+                ))
+            },
+        )
+        .optional()?;
+
+    if let Some((abs, container, rel, duration)) = best {
+        conn.execute(
+            "UPDATE items SET container=?2, abs_path=?3, rel_path=?4, duration_ms=?5 WHERE id=?1",
+            params![item_id, container, abs, rel, duration],
+        )?;
+    }
+    Ok(())
+}
+
+pub(super) fn recompute_all_representatives(pool: &Pool) -> Result<()> {
+    let conn = pool.get()?;
+    let mut stmt = conn.prepare("SELECT DISTINCT item_id FROM files WHERE probed = 1")?;
+    let ids: Vec<String> = stmt
+        .query_map([], |r| r.get::<_, String>(0))?
+        .collect::<rusqlite::Result<Vec<_>>>()?;
+    for id in ids {
+        recompute_item_representative(&conn, &id)?;
+    }
+    Ok(())
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::ingest::test_support::*;
+    use crate::sync_all;
+    use crate::testing::TempPool;
+    use std::collections::HashMap;
+
+    fn pool_with_probed_movie() -> TempPool {
+        let p = pool();
+        sync_all(
+            &p,
+            &[lib("lib")],
+            &[],
+            &[movie(
+                "m1",
+                "Dune",
+                "lib",
+                vec![file("f1", "/media/m1.mkv", false)],
+            )],
+            &HashMap::new(),
+        )
+        .unwrap();
+        p
+    }
+
+    #[test]
+    fn a_refused_file_write_fails_the_probe_rather_than_reporting_success() {
+        let p = pool_with_probed_movie();
+        p.get()
+            .unwrap()
+            .execute_batch(
+                "CREATE TRIGGER no_file_update BEFORE UPDATE ON files \
+                 BEGIN SELECT RAISE(ABORT, 'refused'); END",
+            )
+            .unwrap();
+
+        assert!(set_file_probe(&p, "f1", Some(1), None, None, &[], &[]).is_err());
+    }
+
+    #[test]
+    fn a_refused_item_write_fails_the_probe_that_triggered_the_recompute() {
+        let p = pool_with_probed_movie();
+        p.get()
+            .unwrap()
+            .execute_batch(
+                "CREATE TRIGGER no_item_update BEFORE UPDATE ON items \
+                 BEGIN SELECT RAISE(ABORT, 'refused'); END",
+            )
+            .unwrap();
+
+        assert!(set_file_probe(&p, "f1", Some(1), None, None, &[], &[]).is_err());
+    }
+
+    #[test]
+    fn a_probe_result_for_a_file_that_is_gone_is_recorded_against_no_item() {
+        let p = pool_with_probed_movie();
+        set_file_probe(&p, "no-such-file", Some(1), None, None, &[], &[]).unwrap();
+
+        let container: String = p
+            .get()
+            .unwrap()
+            .query_row("SELECT container FROM items WHERE id='m1'", [], |r| {
+                r.get(0)
+            })
+            .unwrap();
+        assert_eq!(container, "mkv", "no item was recomputed");
+    }
+
+    #[test]
+    fn an_item_with_nothing_probed_yet_keeps_its_scanned_columns() {
+        let p = pool_with_probed_movie();
+        let conn = p.get().unwrap();
+        recompute_item_representative(&conn, "m1").unwrap();
+
+        let container: String = conn
+            .query_row("SELECT container FROM items WHERE id='m1'", [], |r| {
+                r.get(0)
+            })
+            .unwrap();
+        assert_eq!(container, "mkv");
+    }
+}

package/rust/kroma-db/src/ingest/scanned_files.rs

@@ -0,0 +1,235 @@
+//! The file rows a scan writes, and the probe data an unchanged file keeps.
+
+use std::collections::HashMap;
+
+use anyhow::Result;
+use rusqlite::params;
+
+use kroma_domain::{MediaFile, MediaItem};
+
+pub(super) fn sync_files(
+    tx: &rusqlite::Transaction,
+    items: &[MediaItem],
+    mtimes: &HashMap<String, Option<i64>>,
+) -> Result<()> {
+    let scanned: std::collections::HashSet<&str> = items
+        .iter()
+        .flat_map(|i| i.files.iter())
+        .filter_map(|f| f.abs_path.as_deref())
+        .collect();
+
+    delete_gone_files(tx, &scanned)?;
+    let prev = existing_file_sigs(tx)?;
+
+    let mut keep_stmt = tx.prepare(
+        "INSERT INTO files (id,item_id,abs_path,rel_path,container,size,mtime,edition) \
+         VALUES (?1,?2,?3,?4,?5,?6,?7,?8) \
+         ON CONFLICT(abs_path) DO UPDATE SET \
+             id=excluded.id, item_id=excluded.item_id, rel_path=excluded.rel_path, \
+             container=excluded.container, size=excluded.size, mtime=excluded.mtime, \
+             edition=excluded.edition",
+    )?;
+    let mut reset_stmt = tx.prepare(
+        "INSERT INTO files (id,item_id,abs_path,rel_path,container,size,mtime,edition,probed,\
+             duration_ms,v_codec,v_width,v_height,v_hdr,v_bit_depth,a_codec,a_channels,a_language,subtitles,audio_tracks) \
+         VALUES (?1,?2,?3,?4,?5,?6,?7,?8,0,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,'[]','[]') \
+         ON CONFLICT(abs_path) DO UPDATE SET \
+             id=excluded.id, item_id=excluded.item_id, rel_path=excluded.rel_path, \
+             container=excluded.container, size=excluded.size, mtime=excluded.mtime, \
+             edition=excluded.edition, probed=0, duration_ms=NULL, v_codec=NULL, v_width=NULL, \
+             v_height=NULL, v_hdr=NULL, v_bit_depth=NULL, a_codec=NULL, a_channels=NULL, \
+             a_language=NULL, subtitles='[]', audio_tracks='[]'",
+    )?;
+    // Pre-probed files (demo/seed content) skip the phase-2 probe pass.
+    let mut preprobed_stmt = tx.prepare(
+        "INSERT INTO files (id,item_id,abs_path,rel_path,container,size,mtime,edition,probed,\
+             duration_ms,v_codec,v_width,v_height,v_hdr,v_bit_depth,a_codec,a_channels,a_language,subtitles,audio_tracks) \
+         VALUES (?1,?2,?3,?4,?5,?6,?7,?8,1,?9,?10,?11,?12,?13,?14,?15,?16,?17,?18,?19) \
+         ON CONFLICT(abs_path) DO UPDATE SET \
+             id=excluded.id, item_id=excluded.item_id, rel_path=excluded.rel_path, \
+             container=excluded.container, size=excluded.size, mtime=excluded.mtime, \
+             edition=excluded.edition, probed=1, duration_ms=excluded.duration_ms, \
+             v_codec=excluded.v_codec, v_width=excluded.v_width, v_height=excluded.v_height, \
+             v_hdr=excluded.v_hdr, v_bit_depth=excluded.v_bit_depth, a_codec=excluded.a_codec, \
+             a_channels=excluded.a_channels, a_language=excluded.a_language, subtitles=excluded.subtitles, \
+             audio_tracks=excluded.audio_tracks",
+    )?;
+
+    for i in items {
+        for f in &i.files {
+            upsert_scanned_file(
+                &mut keep_stmt,
+                &mut reset_stmt,
+                &mut preprobed_stmt,
+                &prev,
+                mtimes,
+                i,
+                f,
+            )?;
+        }
+    }
+    Ok(())
+}
+
+fn delete_gone_files(
+    tx: &rusqlite::Transaction,
+    scanned: &std::collections::HashSet<&str>,
+) -> Result<()> {
+    let mut existing: Vec<(String, String)> = Vec::new();
+    {
+        let mut q = tx.prepare("SELECT id, abs_path FROM files")?;
+        let rows = q.query_map([], |r| Ok((r.get::<_, String>(0)?, r.get::<_, String>(1)?)))?;
+        for r in rows {
+            existing.push(r?);
+        }
+    }
+    let mut del = tx.prepare("DELETE FROM files WHERE id = ?1")?;
+    for (id, abs) in &existing {
+        if !scanned.contains(abs.as_str()) {
+            del.execute(params![id])?;
+        }
+    }
+    Ok(())
+}
+
+// What a stored file row has to still match for its probe result to be kept.
+struct FileSig {
+    size: Option<i64>,
+    mtime: Option<i64>,
+    probed: bool,
+}
+
+fn existing_file_sigs(tx: &rusqlite::Transaction) -> Result<HashMap<String, FileSig>> {
+    let mut prev: HashMap<String, FileSig> = HashMap::new();
+    let mut q = tx.prepare("SELECT abs_path, size, mtime, probed FROM files")?;
+    let rows = q.query_map([], |r| {
+        Ok((
+            r.get::<_, String>(0)?,
+            FileSig {
+                size: r.get(1)?,
+                mtime: r.get(2)?,
+                probed: r.get::<_, i64>(3)? != 0,
+            },
+        ))
+    })?;
+    for r in rows {
+        let (abs, sig) = r?;
+        prev.insert(abs, sig);
+    }
+    Ok(prev)
+}
+
+fn upsert_scanned_file<'a>(
+    keep_stmt: &mut rusqlite::Statement<'a>,
+    reset_stmt: &mut rusqlite::Statement<'a>,
+    preprobed_stmt: &mut rusqlite::Statement<'a>,
+    prev: &HashMap<String, FileSig>,
+    mtimes: &HashMap<String, Option<i64>>,
+    i: &MediaItem,
+    f: &MediaFile,
+) -> Result<()> {
+    let Some(abs) = f.abs_path.as_deref() else {
+        return Ok(());
+    };
+    let size = f.size.map(|s| s as i64);
+    let mtime = mtimes.get(&f.id).copied().flatten();
+
+    if f.probed {
+        let v = f.video.as_ref();
+        let a = f.audio.as_ref();
+        let subs = serde_json::to_string(&f.subtitles).unwrap_or_else(|_| "[]".into());
+        let a_tracks = serde_json::to_string(&f.audio_tracks).unwrap_or_else(|_| "[]".into());
+        preprobed_stmt.execute(params![
+            f.id,
+            i.id,
+            abs,
+            f.rel_path,
+            f.container,
+            size,
+            mtime,
+            f.edition,
+            f.duration_ms.map(|d| d as i64),
+            v.map(|v| v.codec.clone()),
+            v.and_then(|v| v.width),
+            v.and_then(|v| v.height),
+            v.map(|v| v.hdr as i64),
+            v.and_then(|v| v.bit_depth),
+            a.map(|a| a.codec.clone()),
+            a.and_then(|a| a.channels),
+            a.and_then(|a| a.language.clone()),
+            subs,
+            a_tracks,
+        ])?;
+        return Ok(());
+    }
+
+    let unchanged_probed = prev
+        .get(abs)
+        .is_some_and(|p| p.probed && p.size == size && p.mtime == mtime);
+    let stmt = if unchanged_probed {
+        keep_stmt
+    } else {
+        reset_stmt
+    };
+    stmt.execute(params![
+        f.id,
+        i.id,
+        abs,
+        f.rel_path,
+        f.container,
+        size,
+        mtime,
+        f.edition,
+    ])?;
+    Ok(())
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::ingest::test_support::*;
+    use crate::{item_has_probed_file, sync_all, unprobed_files};
+
+    #[test]
+    fn sync_preprobed_file_stores_streams_directly() {
+        let p = pool();
+        let items = vec![movie(
+            "m1",
+            "Demo",
+            "lib",
+            vec![file("f1", "demo://m1", true)],
+        )];
+        sync_all(&p, &[lib("lib")], &[], &items, &mtimes_of(&items, 100)).unwrap();
+        assert!(item_has_probed_file(&p, "m1").unwrap());
+        assert!(unprobed_files(&p).unwrap().is_empty());
+        let got = crate::get_item(&p, "m1").unwrap().unwrap();
+        assert_eq!(got.video.map(|v| v.codec), Some("hevc".to_string()));
+        // demo:// paths are not streamable, so abs_path stays cleared.
+        assert!(got.abs_path.is_none());
+    }
+
+    #[test]
+    fn a_scan_that_cannot_write_a_file_fails_on_both_the_probed_and_unprobed_paths() {
+        let p = pool();
+        p.get()
+            .unwrap()
+            .execute_batch(
+                "CREATE TRIGGER no_file_insert BEFORE INSERT ON files \
+                 BEGIN SELECT RAISE(ABORT, 'refused'); END",
+            )
+            .unwrap();
+
+        for probed in [false, true] {
+            let items = vec![movie(
+                "m1",
+                "Dune",
+                "lib",
+                vec![file("f1", "/media/m1.mkv", probed)],
+            )];
+            assert!(
+                sync_all(&p, &[lib("lib")], &[], &items, &HashMap::new()).is_err(),
+                "probed = {probed}"
+            );
+        }
+    }
+}