@kromatv/sdk 0.1.40-canary.0

package/rust/kroma-db/src/pipeline/ops/reconcile.rs

@@ -0,0 +1,274 @@
+//! Bringing a stage's ledger in line with what was just enumerated.
+
+use std::collections::{HashMap, HashSet};
+
+use anyhow::Result;
+use rusqlite::{params, TransactionBehavior};
+
+use super::{Subject, MAX_ATTEMPTS, UNREADABLE_SIG};
+use crate::pool::Pool;
+
+/// Reconcile a stage's ledger against the freshly-enumerated `subjects` (one
+/// transaction). Insert missing subjects as `pending`; re-`pending` any whose
+/// signature changed; retry `failed` rows under [`MAX_ATTEMPTS`]; delete rows for
+/// subjects that no longer exist. `done`-and-unchanged and `running` rows are
+/// left untouched.
+pub fn reconcile(
+    pool: &Pool,
+    stage: &str,
+    subject_kind: &str,
+    subjects: &[Subject],
+    now: i64,
+) -> Result<()> {
+    let mut conn = pool.get()?;
+    let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?;
+    let existing: HashMap<String, (Option<String>, String, i64, Option<i64>)> = {
+        let mut stmt = tx.prepare(
+            "SELECT subject_id, input_sig, status, attempts, next_retry_at \
+             FROM pipeline_tasks WHERE stage=?1",
+        )?;
+        let rows = stmt.query_map(params![stage], |r| {
+            Ok((
+                r.get::<_, String>(0)?,
+                (
+                    r.get::<_, Option<String>>(1)?,
+                    r.get::<_, String>(2)?,
+                    r.get::<_, i64>(3)?,
+                    r.get::<_, Option<i64>>(4)?,
+                ),
+            ))
+        })?;
+        rows.collect::<rusqlite::Result<HashMap<_, _>>>()?
+    };
+    let present: HashSet<&str> = subjects.iter().map(|(id, _)| id.as_str()).collect();
+    for (id, sig) in subjects {
+        reconcile_subject(&tx, stage, subject_kind, id, sig, existing.get(id), now)?;
+    }
+    for id in existing.keys() {
+        if !present.contains(id.as_str()) {
+            tx.execute(
+                "DELETE FROM pipeline_tasks WHERE stage=?1 AND subject_id=?2",
+                params![stage, id],
+            )?;
+        }
+    }
+    tx.commit()?;
+    Ok(())
+}
+
+type ExistingTask = (Option<String>, String, i64, Option<i64>);
+
+fn reconcile_subject(
+    tx: &rusqlite::Transaction,
+    stage: &str,
+    subject_kind: &str,
+    id: &str,
+    sig: &str,
+    existing: Option<&ExistingTask>,
+    now: i64,
+) -> Result<()> {
+    match existing {
+        None => {
+            // Unreadable inputs (mount blip): defer the first-ever processing to
+            // the next reconcile rather than creating a task from a bad signature.
+            if sig != UNREADABLE_SIG {
+                tx.execute(
+                    "INSERT INTO pipeline_tasks \
+                       (stage,subject_kind,subject_id,status,input_sig,attempts,priority,enqueued_at,updated_at) \
+                     VALUES (?1,?2,?3,'pending',?4,0,0,?5,?5)",
+                    params![stage, subject_kind, id, sig, now],
+                )?;
+            }
+        }
+        Some((old_sig, status, attempts, next_retry_at)) => {
+            reconcile_existing(
+                tx,
+                stage,
+                subject_kind,
+                id,
+                sig,
+                old_sig,
+                status,
+                *attempts,
+                *next_retry_at,
+                now,
+            )?;
+        }
+    }
+    Ok(())
+}
+
+#[allow(clippy::too_many_arguments)]
+fn reconcile_existing(
+    tx: &rusqlite::Transaction,
+    stage: &str,
+    subject_kind: &str,
+    id: &str,
+    sig: &str,
+    old_sig: &Option<String>,
+    status: &str,
+    attempts: i64,
+    next_retry_at: Option<i64>,
+    now: i64,
+) -> Result<()> {
+    if status == "running" {
+        return Ok(());
+    }
+    // Never a changed signature, or a flapping mount would re-queue the whole
+    // library on every blip.
+    if sig == UNREADABLE_SIG {
+        return Ok(());
+    }
+    // A NULL old sig (a task from `enqueue`, which omits input_sig) is not a
+    // change, or a just-finished reprocess would be re-run once more here.
+    let sig_changed = old_sig.is_some() && old_sig.as_deref() != Some(sig);
+    if sig_changed {
+        tx.execute(
+            "UPDATE pipeline_tasks SET status='pending', input_sig=?4, attempts=0, \
+               error=NULL, next_retry_at=NULL, enqueued_at=?5, updated_at=?5 \
+             WHERE stage=?1 AND subject_kind=?2 AND subject_id=?3",
+            params![stage, subject_kind, id, sig, now],
+        )?;
+        return Ok(());
+    }
+    if old_sig.is_none() {
+        tx.execute(
+            "UPDATE pipeline_tasks SET input_sig=?4, updated_at=?5 \
+             WHERE stage=?1 AND subject_kind=?2 AND subject_id=?3",
+            params![stage, subject_kind, id, sig, now],
+        )?;
+    }
+    let backoff_elapsed = next_retry_at.is_none_or(|t| t <= now);
+    if status == "failed" && attempts < MAX_ATTEMPTS && backoff_elapsed {
+        tx.execute(
+            "UPDATE pipeline_tasks SET status='pending', updated_at=?4 \
+             WHERE stage=?1 AND subject_kind=?2 AND subject_id=?3",
+            params![stage, subject_kind, id, now],
+        )?;
+    }
+    Ok(())
+}
+
+/// Force one subject to `pending` at (at least) `priority`, inserting the task if
+/// it doesn't exist yet. The signature is left for the stage's next reconcile to
+/// normalize.
+pub fn enqueue(
+    pool: &Pool,
+    stage: &str,
+    subject_kind: &str,
+    id: &str,
+    priority: i64,
+    now: i64,
+) -> Result<()> {
+    let conn = pool.get()?;
+    conn.execute(
+        "INSERT INTO pipeline_tasks \
+           (stage,subject_kind,subject_id,status,attempts,priority,enqueued_at,updated_at) \
+         VALUES (?1,?2,?3,'pending',0,?4,?5,?5) \
+         ON CONFLICT(stage,subject_kind,subject_id) DO UPDATE SET \
+           status='pending', attempts=0, error=NULL, next_retry_at=NULL, \
+           priority=MAX(priority, excluded.priority), \
+           enqueued_at=excluded.enqueued_at, updated_at=excluded.updated_at",
+        params![stage, subject_kind, id, priority, now],
+    )?;
+    Ok(())
+}
+
+/// Force every settled task of `stage` back to `pending`, clearing the stored
+/// signature, so the stage rebuilds the whole set on its next run. For when the
+/// stage's on-disk outputs were wiped out of band: the signature-based skip
+/// cannot see a missing output. Leaves `pending`/`running` alone.
+pub fn requeue_stage(pool: &Pool, stage: &str, now: i64) -> Result<usize> {
+    let conn = pool.get()?;
+    let n = conn.execute(
+        "UPDATE pipeline_tasks SET status='pending', attempts=0, error=NULL, input_sig=NULL, \
+           next_retry_at=NULL, updated_at=?2 \
+         WHERE stage=?1 AND status IN ('done','failed','blocked')",
+        params![stage, now],
+    )?;
+    Ok(n)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::pipeline::ops::test_support::*;
+    use crate::pipeline::ops::{claim_batch, finish_batch};
+
+    #[test]
+    fn enqueue_is_idempotent_and_keeps_the_higher_priority() {
+        let p = pool();
+        enqueue(&p, "probe", "item", "a", 5, 1).unwrap();
+        enqueue(&p, "probe", "item", "a", 1, 2).unwrap();
+        assert_eq!(row(&p, "probe", "a"), ("pending".to_string(), 0, 5));
+    }
+
+    #[test]
+    fn enqueue_revives_a_settled_task() {
+        let p = pool();
+        enqueue(&p, "probe", "item", "a", 0, 1).unwrap();
+        claim_batch(&p, "probe", 10, 2).unwrap();
+        finish_batch(&p, "probe", &[failed("a", "boom")], 3).unwrap();
+        assert_eq!(row(&p, "probe", "a").0, "failed");
+
+        enqueue(&p, "probe", "item", "a", 0, 4).unwrap();
+        let (status, attempts, _) = row(&p, "probe", "a");
+        assert_eq!(status, "pending");
+        assert_eq!(attempts, 0);
+        assert!(next_retry_at(&p, "probe", "a").is_none());
+    }
+
+    #[test]
+    fn requeue_stage_clears_the_signature_so_the_skip_cannot_hold() {
+        let p = pool();
+        enqueue(&p, "probe", "item", "a", 0, 1).unwrap();
+        claim_batch(&p, "probe", 10, 2).unwrap();
+        finish_batch(&p, "probe", &[ok("a")], 3).unwrap();
+        p.get()
+            .unwrap()
+            .execute(
+                "UPDATE pipeline_tasks SET input_sig='sig' WHERE subject_id='a'",
+                [],
+            )
+            .unwrap();
+
+        assert_eq!(requeue_stage(&p, "probe", 10).unwrap(), 1);
+        assert_eq!(row(&p, "probe", "a").0, "pending");
+        let sig: Option<String> = p
+            .get()
+            .unwrap()
+            .query_row(
+                "SELECT input_sig FROM pipeline_tasks WHERE subject_id='a'",
+                [],
+                |r| r.get(0),
+            )
+            .unwrap();
+        assert!(sig.is_none());
+    }
+
+    #[test]
+    fn requeue_stage_leaves_pending_and_running_alone() {
+        let p = pool();
+        enqueue(&p, "probe", "item", "pending1", 0, 1).unwrap();
+        enqueue(&p, "probe", "item", "running1", 0, 1).unwrap();
+        claim_batch(&p, "probe", 1, 2).unwrap();
+
+        assert_eq!(requeue_stage(&p, "probe", 10).unwrap(), 0);
+    }
+
+    #[test]
+    fn a_refused_update_aborts_the_whole_reconcile_instead_of_half_writing_it() {
+        let p = pool();
+        reconcile(&p, "probe", "item", &[("m1".into(), "sig-a".into())], 1_000).unwrap();
+        p.get()
+            .unwrap()
+            .execute_batch(
+                "CREATE TRIGGER no_update BEFORE UPDATE ON pipeline_tasks \
+                 BEGIN SELECT RAISE(ABORT, 'refused'); END",
+            )
+            .unwrap();
+
+        assert!(reconcile(&p, "probe", "item", &[("m1".into(), "sig-b".into())], 2_000).is_err());
+        assert_eq!(row(&p, "probe", "m1").0, "pending");
+    }
+}

package/rust/kroma-db/src/pipeline/ops/test_support.rs

@@ -0,0 +1,48 @@
+use rusqlite::params;
+
+use super::TaskResult;
+use crate::pool::Pool;
+use crate::testing::TempPool;
+
+pub(super) fn pool() -> TempPool {
+    crate::testing::temp_pool("ops")
+}
+
+pub(super) fn row(p: &Pool, stage: &str, id: &str) -> (String, i64, i64) {
+    p.get()
+        .unwrap()
+        .query_row(
+            "SELECT status, attempts, priority FROM pipeline_tasks \
+             WHERE stage=?1 AND subject_id=?2",
+            params![stage, id],
+            |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?)),
+        )
+        .unwrap()
+}
+
+pub(super) fn next_retry_at(p: &Pool, stage: &str, id: &str) -> Option<i64> {
+    p.get()
+        .unwrap()
+        .query_row(
+            "SELECT next_retry_at FROM pipeline_tasks WHERE stage=?1 AND subject_id=?2",
+            params![stage, id],
+            |r| r.get(0),
+        )
+        .unwrap()
+}
+
+pub(super) fn ok(id: &str) -> TaskResult {
+    TaskResult {
+        id: id.into(),
+        error: None,
+        duration_ms: 5,
+    }
+}
+
+pub(super) fn failed(id: &str, msg: &str) -> TaskResult {
+    TaskResult {
+        id: id.into(),
+        error: Some(msg.into()),
+        duration_ms: 5,
+    }
+}

package/rust/kroma-db/src/pipeline/ops.rs

@@ -0,0 +1,49 @@
+//! Ledger write / lifecycle operations. Every write is batched into a single
+//! IMMEDIATE transaction: it takes the write lock at BEGIN, so `busy_timeout`
+//! serializes concurrent stage drains, where a deferred transaction cannot
+//! upgrade mid-flight and fails `SQLITE_BUSY` instead of waiting. Timestamps
+//! are epoch milliseconds.
+
+mod drain;
+mod reconcile;
+
+#[cfg(test)]
+mod test_support;
+
+pub use drain::*;
+pub use reconcile::*;
+
+/// How many times a failed task is auto-retried across drains before it sticks
+/// as `failed` and waits for a manual retry.
+pub const MAX_ATTEMPTS: i64 = 3;
+
+const RETRY_BASE_MS: i64 = 5 * 60 * 1000;
+
+/// Quadratic backoff between auto-retries: attempt 1 -> 5 min, attempt 2 ->
+/// 20 min. Keeps a manually re-kicked stage from hammering a flaky dependency
+/// with back-to-back retries.
+pub fn retry_backoff_ms(attempts: i64) -> i64 {
+    RETRY_BASE_MS * attempts * attempts
+}
+
+/// Sentinel signature meaning the subject's inputs were unreadable at enumerate
+/// time; `reconcile` leaves such a task exactly as it is, so a flapping mount
+/// cannot re-queue the whole library. Cannot collide with a real signature.
+pub const UNREADABLE_SIG: &str = "\u{0}unreadable";
+
+/// A unit of work: the subject's id + a signature of its current inputs. A task
+/// is skipped while `status='done'` and its stored signature still matches, and
+/// re-queued the moment the signature changes.
+pub type Subject = (String, String);
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn backoff_grows_quadratically() {
+        assert_eq!(retry_backoff_ms(1), 5 * 60 * 1000);
+        assert_eq!(retry_backoff_ms(2), 20 * 60 * 1000);
+        assert!(retry_backoff_ms(2) > retry_backoff_ms(1));
+    }
+}

package/rust/kroma-db/src/pipeline/query/elements.rs

@@ -0,0 +1,200 @@
+//! The lean rows the elements list renders, and the titles behind their ids.
+
+use std::collections::HashMap;
+
+use anyhow::Result;
+
+use crate::pool::Pool;
+
+/// Lean item row for the elements list: only the columns the view needs, with
+/// poster/genre/has-metadata pulled out of the JSON via `json_extract` so we
+/// never deserialize the full (heavy) TMDB metadata blob per item.
+pub struct RawItem {
+    pub id: String,
+    pub kind: String,
+    pub title: String,
+    pub year: Option<i64>,
+    pub duration_ms: Option<i64>,
+    pub show_id: Option<String>,
+    pub show_title: Option<String>,
+    pub season: Option<i64>,
+    pub episode: Option<i64>,
+    pub episode_title: Option<String>,
+    pub has_meta: bool,
+    pub poster: Option<String>,
+    pub genre: Option<String>,
+}
+
+pub struct RawShow {
+    pub id: String,
+    pub title: String,
+    pub year: Option<i64>,
+    pub has_meta: bool,
+    pub poster: Option<String>,
+    pub genre: Option<String>,
+}
+
+/// All items, lean (no full-metadata parse). One query.
+pub fn raw_items(pool: &Pool) -> Result<Vec<RawItem>> {
+    let conn = pool.get()?;
+    let mut stmt = conn.prepare(
+        "SELECT id,kind,title,year,duration_ms,show_id,show_title,season,episode,episode_title,\
+           (metadata IS NOT NULL), json_extract(metadata,'$.posterUrl'), json_extract(metadata,'$.genres[0]') \
+         FROM items",
+    )?;
+    let rows = stmt.query_map([], |r| {
+        Ok(RawItem {
+            id: r.get(0)?,
+            kind: r.get(1)?,
+            title: r.get(2)?,
+            year: r.get(3)?,
+            duration_ms: r.get(4)?,
+            show_id: r.get(5)?,
+            show_title: r.get(6)?,
+            season: r.get(7)?,
+            episode: r.get(8)?,
+            episode_title: r.get(9)?,
+            has_meta: r.get::<_, i64>(10)? != 0,
+            poster: r.get(11)?,
+            genre: r.get(12)?,
+        })
+    })?;
+    Ok(rows.collect::<rusqlite::Result<Vec<_>>>()?)
+}
+
+/// All shows, lean. One query.
+pub fn raw_shows(pool: &Pool) -> Result<Vec<RawShow>> {
+    let conn = pool.get()?;
+    let mut stmt = conn.prepare(
+        "SELECT id,title,year,(metadata IS NOT NULL), \
+           json_extract(metadata,'$.posterUrl'), json_extract(metadata,'$.genres[0]') FROM shows",
+    )?;
+    let rows = stmt.query_map([], |r| {
+        Ok(RawShow {
+            id: r.get(0)?,
+            title: r.get(1)?,
+            year: r.get(2)?,
+            has_meta: r.get::<_, i64>(3)? != 0,
+            poster: r.get(4)?,
+            genre: r.get(5)?,
+        })
+    })?;
+    Ok(rows.collect::<rusqlite::Result<Vec<_>>>()?)
+}
+
+/// Titles for a set of item ids, one query (ids without a row are simply absent
+/// from the map). Batch resolver for the failed-task drill-down.
+pub fn item_titles(pool: &Pool, ids: &[String]) -> Result<HashMap<String, String>> {
+    titles_in(pool, "items", ids)
+}
+
+/// Titles for a set of show ids, one query. Batch resolver for the failed-task
+/// drill-down (metadata/embed subjects are item-kind but their id may be a show).
+pub fn show_titles(pool: &Pool, ids: &[String]) -> Result<HashMap<String, String>> {
+    titles_in(pool, "shows", ids)
+}
+
+// `id -> title` for the given ids from `table` (a fixed `"items"`/`"shows"`,
+// never user input), one `IN (...)` query per chunk. Empty ids yields an empty
+// map.
+fn titles_in(pool: &Pool, table: &str, ids: &[String]) -> Result<HashMap<String, String>> {
+    if ids.is_empty() {
+        return Ok(HashMap::new());
+    }
+    let conn = pool.get()?;
+    let mut titles = HashMap::with_capacity(ids.len());
+    for chunk in ids.chunks(crate::IN_CHUNK) {
+        let placeholders = vec!["?"; chunk.len()].join(",");
+        let sql = format!("SELECT id, title FROM {table} WHERE id IN ({placeholders})");
+        let mut stmt = conn.prepare(&sql)?;
+        let rows = stmt.query_map(rusqlite::params_from_iter(chunk.iter()), |r| {
+            Ok((r.get::<_, String>(0)?, r.get::<_, String>(1)?))
+        })?;
+        for row in rows {
+            let (id, title) = row?;
+            titles.insert(id, title);
+        }
+    }
+    Ok(titles)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::pipeline::query::test_support::*;
+
+    #[test]
+    fn raw_rows_and_title_lookups() {
+        let p = pool();
+        {
+            let conn = p.get().unwrap();
+            conn.execute("INSERT INTO libraries (id,name,kind,path,added_at) VALUES ('lib','L','movies','/x','t')", []).unwrap();
+            conn.execute(
+                "INSERT INTO items (id,kind,title,year,container,library,added_at,metadata) \
+                 VALUES ('m1','movie','Dune',2021,'mkv','lib','t','{\"posterUrl\":\"/p.webp\",\"genres\":[\"Horror\",\"Thriller\"]}')",
+                [],
+            )
+            .unwrap();
+            conn.execute(
+                "INSERT INTO items (id,kind,title,container,library,added_at) VALUES ('m2','movie','Bare','mkv','lib','t')",
+                [],
+            )
+            .unwrap();
+            conn.execute(
+                "INSERT INTO shows (id,library,title,added_at,metadata) VALUES ('s1','lib','Show','t','{\"posterUrl\":\"/s.webp\",\"genres\":[\"Drama\"]}')",
+                [],
+            )
+            .unwrap();
+        }
+        let items = raw_items(&p).unwrap();
+        let m1 = items.iter().find(|i| i.id == "m1").unwrap();
+        assert!(m1.has_meta);
+        assert_eq!(m1.poster.as_deref(), Some("/p.webp"));
+        assert_eq!(m1.genre.as_deref(), Some("Horror"));
+        assert_eq!(m1.year, Some(2021));
+        let m2 = items.iter().find(|i| i.id == "m2").unwrap();
+        assert!(!m2.has_meta);
+        assert!(m2.poster.is_none());
+
+        let shows = raw_shows(&p).unwrap();
+        assert_eq!(shows.len(), 1);
+        assert!(shows[0].has_meta);
+        assert_eq!(shows[0].genre.as_deref(), Some("Drama"));
+
+        let titles = item_titles(&p, &["m1".into(), "ghost".into()]).unwrap();
+        assert_eq!(titles.get("m1").map(String::as_str), Some("Dune"));
+        assert_eq!(titles.len(), 1);
+        assert_eq!(
+            show_titles(&p, &["s1".into()])
+                .unwrap()
+                .get("s1")
+                .map(String::as_str),
+            Some("Show")
+        );
+        assert!(item_titles(&p, &[]).unwrap().is_empty());
+    }
+
+    #[test]
+    fn a_title_lookup_longer_than_sqlite_can_bind_still_answers() {
+        const SQLITE_BIND_LIMIT: usize = 32_766;
+        let p = pool();
+        {
+            let conn = p.get().unwrap();
+            conn.execute("INSERT INTO libraries (id,name,kind,path,added_at) VALUES ('lib','L','movies','/x','t')", []).unwrap();
+            conn.execute(
+                "INSERT INTO items (id,kind,title,container,library,added_at) VALUES ('m1','movie','Dune','mkv','lib','t')",
+                [],
+            )
+            .unwrap();
+        }
+        let mut ids: Vec<String> = (0..=SQLITE_BIND_LIMIT)
+            .map(|n| format!("absent{n}"))
+            .collect();
+        ids.push("m1".to_string());
+
+        let titles = item_titles(&p, &ids).unwrap();
+
+        assert_eq!(titles.get("m1").map(String::as_str), Some("Dune"));
+        assert_eq!(titles.len(), 1);
+    }
+}

package/rust/kroma-db/src/pipeline/query/test_support.rs

@@ -0,0 +1,23 @@
+use rusqlite::{params, Connection};
+
+use crate::testing::TempPool;
+
+pub(super) fn pool() -> TempPool {
+    crate::testing::temp_pool("pq")
+}
+
+pub(super) fn task(
+    conn: &Connection,
+    stage: &str,
+    id: &str,
+    status: &str,
+    error: Option<&str>,
+    finished: Option<i64>,
+) {
+    conn.execute(
+        "INSERT INTO pipeline_tasks (stage,subject_kind,subject_id,status,error,attempts,enqueued_at,updated_at,finished_at) \
+         VALUES (?1,'file',?2,?3,?4,1,0,0,?5)",
+        params![stage, id, status, error, finished],
+    )
+    .unwrap();
+}