@kromatv/sdk 0.1.40-canary.0

package/rust/kroma-db/src/notifications.rs

@@ -0,0 +1,232 @@
+//! Notification persistence: insert, list newest-first, unread tally, read /
+//! delete transitions, retention, and the per-user delivery matrix. Rows store
+//! i18n keys, not rendered text; `services::notify::render` builds the wire shape.
+
+use std::collections::BTreeMap;
+
+use super::*;
+use kroma_domain::{ActionSpec, NotificationCategory, NotificationEvent, ParamValue, PushCategory};
+
+mod digest;
+mod preferences;
+mod read_state;
+
+#[cfg(test)]
+mod test_support;
+
+pub use digest::*;
+pub use preferences::*;
+pub use read_state::*;
+
+pub const RETENTION_PER_USER: usize = 200;
+
+// Column order must match `row_to_notification`.
+const NOTIFICATION_COLS: &str = "id, category, event, title_key, body_key, params, link, \
+    image_url, actions, push_category, read_at, created_at";
+
+/// One notification as stored: keys + params, not yet rendered.
+#[derive(Debug, Clone)]
+pub struct StoredNotification {
+    pub id: String,
+    pub category: NotificationCategory,
+    pub event: NotificationEvent,
+    pub title_key: String,
+    pub body_key: String,
+    pub params: BTreeMap<String, ParamValue>,
+    pub link: Option<String>,
+    pub image_url: Option<String>,
+    pub actions: Vec<ActionSpec>,
+    pub push_category: Option<PushCategory>,
+    pub read: bool,
+    pub created_at: i64,
+}
+
+fn row_to_notification(r: &Row) -> rusqlite::Result<StoredNotification> {
+    let category: String = r.get(1)?;
+    let event: String = r.get(2)?;
+    let params: String = r.get(5)?;
+    let actions: String = r.get(8)?;
+    let push_category: Option<String> = r.get(9)?;
+    let read_at: Option<i64> = r.get(10)?;
+    Ok(StoredNotification {
+        id: r.get(0)?,
+        // Unknown category (written by a newer build): fall back rather than drop the row.
+        category: NotificationCategory::parse(&category).unwrap_or(NotificationCategory::System),
+        // An unknown event is exactly what `Custom` means.
+        event: NotificationEvent::parse(&event).unwrap_or(NotificationEvent::Custom),
+        title_key: r.get(3)?,
+        body_key: r.get(4)?,
+        params: serde_json::from_str(&params).unwrap_or_default(),
+        link: r.get(6)?,
+        image_url: r.get(7)?,
+        actions: serde_json::from_str(&actions).unwrap_or_default(),
+        push_category: push_category.as_deref().and_then(PushCategory::parse),
+        read: read_at.is_some(),
+        created_at: r.get(11)?,
+    })
+}
+
+/// Every image a stored notification still points at, so the size-based cache
+/// trimmer knows an uploaded image is spoken for and must not evict it.
+pub fn referenced_images(conn: &Connection) -> Result<Vec<String>> {
+    let mut stmt =
+        conn.prepare("SELECT DISTINCT image_url FROM notifications WHERE image_url IS NOT NULL")?;
+    let rows = stmt.query_map([], |r| r.get::<_, String>(0))?;
+    Ok(rows.collect::<rusqlite::Result<Vec<_>>>()?)
+}
+
+/// Insert one notification for `user_id`, enforce [`RETENTION_PER_USER`], and
+/// return that user's new unread count.
+pub fn insert_notification(
+    conn: &Connection,
+    user_id: &str,
+    n: &StoredNotification,
+) -> Result<u32> {
+    // Serialization can't realistically fail; degrade to empty rather than lose the notification.
+    let params = serde_json::to_string(&n.params).unwrap_or_else(|_| "{}".into());
+    let actions = serde_json::to_string(&n.actions).unwrap_or_else(|_| "[]".into());
+    conn.execute(
+        "INSERT INTO notifications \
+         (id, user_id, category, event, title_key, body_key, params, link, image_url, actions, \
+          push_category, created_at) \
+         VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12)",
+        params![
+            n.id,
+            user_id,
+            // The spec's category, not the event's default: `Custom` reports `System`,
+            // while a module names the preference bucket its notification answers to.
+            n.category.as_str(),
+            n.event.as_str(),
+            n.title_key,
+            n.body_key,
+            params,
+            n.link,
+            n.image_url,
+            actions,
+            n.push_category.map(PushCategory::as_str),
+            n.created_at
+        ],
+    )?;
+    prune(conn, user_id)?;
+    Ok(unread_count(conn, user_id)?)
+}
+
+fn prune(conn: &Connection, user_id: &str) -> rusqlite::Result<()> {
+    conn.execute(
+        "DELETE FROM notifications WHERE user_id = ?1 AND id NOT IN \
+         (SELECT id FROM notifications WHERE user_id = ?1 ORDER BY created_at DESC LIMIT ?2)",
+        params![user_id, RETENTION_PER_USER as i64],
+    )?;
+    Ok(())
+}
+
+/// One user's notifications, newest first.
+pub fn list_notifications(
+    conn: &Connection,
+    user_id: &str,
+    limit: usize,
+    only_unread: bool,
+) -> rusqlite::Result<Vec<StoredNotification>> {
+    let unread = if only_unread {
+        " AND read_at IS NULL"
+    } else {
+        ""
+    };
+    let sql = format!(
+        "SELECT {NOTIFICATION_COLS} FROM notifications WHERE user_id = ?1{unread} \
+         ORDER BY created_at DESC LIMIT ?2"
+    );
+    let mut stmt = conn.prepare(&sql)?;
+    let rows = stmt.query_map(params![user_id, limit as i64], row_to_notification)?;
+    rows.collect()
+}
+
+/// Unread tally for the bell badge.
+pub fn unread_count(conn: &Connection, user_id: &str) -> rusqlite::Result<u32> {
+    conn.query_row(
+        "SELECT COUNT(*) FROM notifications WHERE user_id = ?1 AND read_at IS NULL",
+        params![user_id],
+        |r| r.get(0),
+    )
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::notifications::test_support::*;
+    use kroma_domain::ActionKind;
+
+    #[test]
+    fn insert_round_trips_every_rich_field() {
+        let (p, u1, _) = pool();
+        insert(&p, "n1", &u1, 1_000);
+        let conn = p.get().unwrap();
+        let list = list_notifications(&conn, &u1, 50, false).unwrap();
+        assert_eq!(list.len(), 1);
+        let n = &list[0];
+        assert_eq!(n.link.as_deref(), Some("/movie/ab12"));
+        assert_eq!(n.image_url.as_deref(), Some("https://img/p.jpg"));
+        assert_eq!(
+            n.params.get("title"),
+            Some(&ParamValue::Text("Dune".into()))
+        );
+        assert_eq!(n.actions.len(), 1);
+        assert_eq!(n.actions[0].id, "view");
+        assert_eq!(n.actions[0].kind, ActionKind::Link);
+        assert_eq!(n.push_category, Some(PushCategory::MediaAvailable));
+        // The category column is derived from the event, not passed in.
+        assert_eq!(n.category, NotificationCategory::Requests);
+        assert!(!n.read);
+    }
+
+    #[test]
+    fn listing_and_counting_are_scoped_to_one_user() {
+        let (p, u1, u2) = pool();
+        insert(&p, "n1", &u1, 1_000);
+        insert(&p, "n2", &u2, 1_000);
+        let conn = p.get().unwrap();
+        assert_eq!(list_notifications(&conn, &u1, 50, false).unwrap().len(), 1);
+        assert_eq!(unread_count(&conn, &u1).unwrap(), 1);
+        assert_eq!(unread_count(&conn, &u2).unwrap(), 1);
+    }
+
+    #[test]
+    fn retention_keeps_the_newest_and_drops_the_rest() {
+        let (p, u1, _) = pool();
+        for i in 0..(RETENTION_PER_USER + 5) {
+            insert(&p, &format!("n{i}"), &u1, i as i64);
+        }
+        let conn = p.get().unwrap();
+        let list = list_notifications(&conn, &u1, 1_000, false).unwrap();
+        assert_eq!(list.len(), RETENTION_PER_USER);
+        assert_eq!(list[0].id, format!("n{}", RETENTION_PER_USER + 4));
+        assert!(!list.iter().any(|n| n.id == "n0"));
+    }
+
+    #[test]
+    fn deleting_a_user_takes_their_notifications_with_them() {
+        let (p, u1, _) = pool();
+        insert(&p, "n1", &u1, 1_000);
+        let conn = p.get().unwrap();
+        conn.execute("DELETE FROM users WHERE id = ?1", params![u1])
+            .unwrap();
+        // ON DELETE CASCADE, so no orphan inbox survives the account.
+        assert_eq!(unread_count(&conn, &u1).unwrap(), 0);
+    }
+
+    #[test]
+    fn a_retention_sweep_that_is_refused_fails_the_write_it_belongs_to() {
+        let (p, u1, _) = pool();
+        let conn = p.get().unwrap();
+        for n in 0..=RETENTION_PER_USER {
+            insert_notification(&conn, &u1, &new(&format!("n{n}"), n as i64)).unwrap();
+        }
+        conn.execute_batch(
+            "CREATE TRIGGER no_prune BEFORE DELETE ON notifications \
+             BEGIN SELECT RAISE(ABORT, 'refused'); END",
+        )
+        .unwrap();
+
+        assert!(insert_notification(&conn, &u1, &new("over", 9_999)).is_err());
+    }
+}

package/rust/kroma-db/src/passkeys.rs

@@ -0,0 +1,163 @@
+//! Storage for WebAuthn credentials. The ceremonies live in `api::passkeys`,
+//! which serializes the webauthn-rs `Passkey` into the `credential` column.
+
+use super::*;
+
+use rusqlite::OptionalExtension;
+
+pub struct PasskeyRow {
+    // Credential id, base64url.
+    pub id: String,
+    pub name: String,
+    pub created_at: String,
+    pub last_used: Option<String>,
+}
+
+/// Returns the `created_at` timestamp written.
+pub fn insert_passkey(
+    pool: &Pool,
+    id: &str,
+    user_id: &str,
+    name: &str,
+    credential: &str,
+) -> Result<String> {
+    let conn = pool.get()?;
+    let created_at = now_or_blank();
+    conn.execute(
+        "INSERT INTO passkeys (id,user_id,name,credential,created_at,last_used) \
+         VALUES (?1,?2,?3,?4,?5,NULL)",
+        params![id, user_id, name, credential, created_at],
+    )?;
+    Ok(created_at)
+}
+
+/// Newest first.
+pub fn list_passkeys(pool: &Pool, user_id: &str) -> Result<Vec<PasskeyRow>> {
+    let conn = pool.get()?;
+    let mut stmt = conn.prepare(
+        "SELECT id,name,created_at,last_used FROM passkeys \
+         WHERE user_id = ?1 ORDER BY created_at DESC",
+    )?;
+    let rows = stmt.query_map(params![user_id], |r| {
+        Ok(PasskeyRow {
+            id: r.get(0)?,
+            name: r.get(1)?,
+            created_at: r.get(2)?,
+            last_used: r.get(3)?,
+        })
+    })?;
+    Ok(rows.collect::<rusqlite::Result<Vec<_>>>()?)
+}
+
+/// The serialized `Passkey` JSON blobs for a user.
+pub fn passkey_credentials(pool: &Pool, user_id: &str) -> Result<Vec<String>> {
+    let conn = pool.get()?;
+    let mut stmt = conn.prepare("SELECT credential FROM passkeys WHERE user_id = ?1")?;
+    let rows = stmt.query_map(params![user_id], |r| r.get::<_, String>(0))?;
+    Ok(rows.collect::<rusqlite::Result<Vec<_>>>()?)
+}
+
+/// Distinct account ids that have at least one passkey; discoverable sign-in
+/// matches the assertion's user handle against these.
+pub fn passkey_user_ids(pool: &Pool) -> Result<Vec<String>> {
+    let conn = pool.get()?;
+    let mut stmt = conn.prepare("SELECT DISTINCT user_id FROM passkeys")?;
+    let rows = stmt.query_map([], |r| r.get::<_, String>(0))?;
+    Ok(rows.collect::<rusqlite::Result<Vec<_>>>()?)
+}
+
+/// Marks a credential used, replacing the stored blob when the authenticator's
+/// signature counter advanced (webauthn-rs `needs_update`).
+pub fn touch_passkey(pool: &Pool, id: &str, credential: Option<&str>) -> Result<()> {
+    let conn = pool.get()?;
+    match credential {
+        Some(cred) => conn.execute(
+            "UPDATE passkeys SET last_used = ?2, credential = ?3 WHERE id = ?1",
+            params![id, now_or_blank(), cred],
+        )?,
+        None => conn.execute(
+            "UPDATE passkeys SET last_used = ?2 WHERE id = ?1",
+            params![id, now_or_blank()],
+        )?,
+    };
+    Ok(())
+}
+
+/// Scoped to `user_id` so a caller can only delete their own.
+pub fn delete_passkey(pool: &Pool, user_id: &str, id: &str) -> Result<bool> {
+    let conn = pool.get()?;
+    let n = conn.execute(
+        "DELETE FROM passkeys WHERE id = ?1 AND user_id = ?2",
+        params![id, user_id],
+    )?;
+    Ok(n > 0)
+}
+
+#[allow(dead_code)]
+pub fn passkey_exists(pool: &Pool, id: &str) -> Result<bool> {
+    let conn = pool.get()?;
+    let found: Option<i64> = conn
+        .query_row("SELECT 1 FROM passkeys WHERE id = ?1", params![id], |r| {
+            r.get(0)
+        })
+        .optional()?;
+    Ok(found.is_some())
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::testing::TempPool;
+    use kroma_domain::Permission;
+
+    fn pool_with_users() -> (TempPool, String, String) {
+        let pool = crate::testing::temp_pool("pk");
+        let a = crate::create_user(&pool, "a@b.c", "alice", "h", &[Permission::Playback]).unwrap();
+        let b = crate::create_user(&pool, "b@b.c", "bob", "h", &[Permission::Playback]).unwrap();
+        (pool, a.id, b.id)
+    }
+
+    #[test]
+    fn insert_list_credentials_and_ids() {
+        let (p, alice, bob) = pool_with_users();
+        insert_passkey(&p, "cred-a1", &alice, "Alice iPhone", "{\"blob\":1}").unwrap();
+        insert_passkey(&p, "cred-a2", &alice, "Alice Yubikey", "{\"blob\":2}").unwrap();
+        insert_passkey(&p, "cred-b1", &bob, "Bob laptop", "{\"blob\":3}").unwrap();
+
+        let alice_keys = list_passkeys(&p, &alice).unwrap();
+        assert_eq!(alice_keys.len(), 2);
+        assert!(alice_keys.iter().all(|k| k.last_used.is_none()));
+
+        assert_eq!(passkey_credentials(&p, &alice).unwrap().len(), 2);
+        let mut ids = passkey_user_ids(&p).unwrap();
+        ids.sort();
+        let mut want = vec![alice.clone(), bob.clone()];
+        want.sort();
+        assert_eq!(ids, want);
+        assert!(passkey_exists(&p, "cred-a1").unwrap());
+        assert!(!passkey_exists(&p, "nope").unwrap());
+    }
+
+    #[test]
+    fn touch_updates_and_delete_is_scoped() {
+        let (p, alice, bob) = pool_with_users();
+        insert_passkey(&p, "cred-a1", &alice, "Alice iPhone", "{\"blob\":1}").unwrap();
+
+        touch_passkey(&p, "cred-a1", Some("{\"blob\":9}")).unwrap();
+        assert!(list_passkeys(&p, &alice).unwrap()[0].last_used.is_some());
+        assert_eq!(
+            passkey_credentials(&p, &alice).unwrap(),
+            vec!["{\"blob\":9}".to_string()]
+        );
+        touch_passkey(&p, "cred-a1", None).unwrap();
+        assert_eq!(
+            passkey_credentials(&p, &alice).unwrap(),
+            vec!["{\"blob\":9}".to_string()]
+        );
+
+        assert!(!delete_passkey(&p, &bob, "cred-a1").unwrap());
+        assert!(delete_passkey(&p, &alice, "cred-a1").unwrap());
+        assert!(list_passkeys(&p, &alice).unwrap().is_empty());
+        assert!(passkey_user_ids(&p).unwrap().is_empty());
+    }
+}

package/rust/kroma-db/src/pipeline/ops/drain.rs

@@ -0,0 +1,266 @@
+//! Taking tasks off the ledger, writing their results back, and retrying.
+
+use anyhow::Result;
+use rusqlite::{params, TransactionBehavior};
+
+use super::{Subject, RETRY_BASE_MS};
+use crate::pool::Pool;
+
+/// Claim up to `limit` pending tasks for a stage: pick the highest-priority /
+/// oldest, flip them to `running`, and return `(subject_id, input_sig)` for the
+/// worker pool.
+pub fn claim_batch(pool: &Pool, stage: &str, limit: usize, now: i64) -> Result<Vec<Subject>> {
+    let mut conn = pool.get()?;
+    let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?;
+    let picked: Vec<Subject> = {
+        let mut stmt = tx.prepare(
+            "SELECT subject_id, COALESCE(input_sig,'') FROM pipeline_tasks \
+             WHERE stage=?1 AND status='pending' ORDER BY priority DESC, enqueued_at LIMIT ?2",
+        )?;
+        let rows = stmt.query_map(params![stage, limit as i64], |r| {
+            Ok((r.get::<_, String>(0)?, r.get::<_, String>(1)?))
+        })?;
+        rows.collect::<rusqlite::Result<Vec<_>>>()?
+    };
+    for (id, _) in &picked {
+        tx.execute(
+            "UPDATE pipeline_tasks SET status='running', started_at=?3, updated_at=?3 \
+             WHERE stage=?1 AND subject_id=?2",
+            params![stage, id, now],
+        )?;
+    }
+    tx.commit()?;
+    Ok(picked)
+}
+
+/// The outcome of processing one claimed task; `error: None` means success.
+pub struct TaskResult {
+    pub id: String,
+    pub error: Option<String>,
+    pub duration_ms: i64,
+}
+
+/// Write a batch of results back (one transaction). Success → `done`; failure →
+/// `failed` with `attempts` incremented (a later reconcile retries it while under
+/// [`MAX_ATTEMPTS`](super::MAX_ATTEMPTS)).
+pub fn finish_batch(pool: &Pool, stage: &str, results: &[TaskResult], now: i64) -> Result<()> {
+    let mut conn = pool.get()?;
+    let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?;
+    for r in results {
+        match &r.error {
+            None => tx.execute(
+                "UPDATE pipeline_tasks SET status='done', error=NULL, next_retry_at=NULL, \
+                   finished_at=?3, duration_ms=?4, updated_at=?3 WHERE stage=?1 AND subject_id=?2",
+                params![stage, r.id, now, r.duration_ms],
+            )?,
+            // `attempts` on the right-hand side reads the pre-update value, so the
+            // backoff is computed from the attempt this failure completes.
+            Some(e) => tx.execute(
+                "UPDATE pipeline_tasks SET status='failed', attempts=attempts+1, error=?3, \
+                   next_retry_at=?4 + ?6*(attempts+1)*(attempts+1), \
+                   finished_at=?4, duration_ms=?5, updated_at=?4 WHERE stage=?1 AND subject_id=?2",
+                params![stage, r.id, e, now, r.duration_ms, RETRY_BASE_MS],
+            )?,
+        };
+    }
+    tx.commit()?;
+    Ok(())
+}
+
+/// Flip `running` tasks back to `pending`, for a stage or all stages, so a crash
+/// or a cancelled drain cannot strand claimed-but-unprocessed tasks.
+pub fn reset_running(pool: &Pool, stage: Option<&str>) -> Result<usize> {
+    let conn = pool.get()?;
+    let now = kroma_primitives::now_ms();
+    let n = match stage {
+        Some(s) => conn.execute(
+            "UPDATE pipeline_tasks SET status='pending', updated_at=?2 \
+             WHERE stage=?1 AND status='running'",
+            params![s, now],
+        )?,
+        None => conn.execute(
+            "UPDATE pipeline_tasks SET status='pending', updated_at=?1 WHERE status='running'",
+            params![now],
+        )?,
+    };
+    Ok(n)
+}
+
+pub const RETRY_PRIORITY: i64 = 100;
+
+/// Reset `failed` tasks back to `pending` (attempts cleared) for a manual retry:
+/// the whole stage (`subject_id = None`) or one task. Bumps priority so the retry
+/// is claimed before the routine backlog.
+pub fn retry(pool: &Pool, stage: &str, subject_id: Option<&str>) -> Result<usize> {
+    let conn = pool.get()?;
+    let now = kroma_primitives::now_ms();
+    let n = match subject_id {
+        Some(id) => conn.execute(
+            "UPDATE pipeline_tasks SET status='pending', attempts=0, error=NULL, \
+               next_retry_at=NULL, priority=MAX(priority, ?3), updated_at=?4 \
+             WHERE stage=?1 AND subject_id=?2 AND status='failed'",
+            params![stage, id, RETRY_PRIORITY, now],
+        )?,
+        None => conn.execute(
+            "UPDATE pipeline_tasks SET status='pending', attempts=0, error=NULL, \
+               next_retry_at=NULL, priority=MAX(priority, ?2), updated_at=?3 \
+             WHERE stage=?1 AND status='failed'",
+            params![stage, RETRY_PRIORITY, now],
+        )?,
+    };
+    Ok(n)
+}
+
+/// Force a full re-run of a stage: every non-running task back to `pending`. This
+/// re-invokes the stage over all subjects; it does not delete cached artifacts, so
+/// each stage's own per-artifact skip still applies.
+pub fn reprocess(pool: &Pool, stage: &str) -> Result<usize> {
+    let conn = pool.get()?;
+    let now = kroma_primitives::now_ms();
+    let n = conn.execute(
+        "UPDATE pipeline_tasks SET status='pending', attempts=0, error=NULL, \
+           next_retry_at=NULL, updated_at=?2 \
+         WHERE stage=?1 AND status!='running'",
+        params![stage, now],
+    )?;
+    Ok(n)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::pipeline::ops::test_support::*;
+    use crate::pipeline::ops::{enqueue, retry_backoff_ms};
+
+    #[test]
+    fn claim_takes_the_highest_priority_then_the_oldest() {
+        let p = pool();
+        enqueue(&p, "probe", "item", "old", 0, 1).unwrap();
+        enqueue(&p, "probe", "item", "new", 0, 2).unwrap();
+        enqueue(&p, "probe", "item", "urgent", 9, 3).unwrap();
+
+        let picked = claim_batch(&p, "probe", 2, 10).unwrap();
+        let ids: Vec<&str> = picked.iter().map(|(id, _)| id.as_str()).collect();
+        assert_eq!(ids, vec!["urgent", "old"]);
+    }
+
+    #[test]
+    fn claim_flips_only_what_it_took() {
+        let p = pool();
+        enqueue(&p, "probe", "item", "a", 0, 1).unwrap();
+        enqueue(&p, "probe", "item", "b", 0, 2).unwrap();
+        claim_batch(&p, "probe", 1, 10).unwrap();
+
+        assert_eq!(row(&p, "probe", "a").0, "running");
+        assert_eq!(row(&p, "probe", "b").0, "pending");
+    }
+
+    #[test]
+    fn claim_ignores_another_stage_and_anything_not_pending() {
+        let p = pool();
+        enqueue(&p, "probe", "item", "a", 0, 1).unwrap();
+        enqueue(&p, "thumbs", "item", "b", 0, 1).unwrap();
+        claim_batch(&p, "probe", 10, 10).unwrap();
+
+        assert!(claim_batch(&p, "probe", 10, 11).unwrap().is_empty());
+        assert_eq!(row(&p, "thumbs", "b").0, "pending");
+    }
+
+    #[test]
+    fn finishing_a_success_clears_the_error_and_the_backoff() {
+        let p = pool();
+        enqueue(&p, "probe", "item", "a", 0, 1).unwrap();
+        claim_batch(&p, "probe", 10, 2).unwrap();
+        finish_batch(&p, "probe", &[failed("a", "boom")], 3).unwrap();
+        retry(&p, "probe", Some("a")).unwrap();
+        claim_batch(&p, "probe", 10, 4).unwrap();
+        finish_batch(&p, "probe", &[ok("a")], 5).unwrap();
+
+        assert_eq!(row(&p, "probe", "a").0, "done");
+        assert!(next_retry_at(&p, "probe", "a").is_none());
+    }
+
+    #[test]
+    fn each_failure_counts_an_attempt_and_backs_off_further() {
+        let p = pool();
+        enqueue(&p, "probe", "item", "a", 0, 1).unwrap();
+        claim_batch(&p, "probe", 10, 2).unwrap();
+        finish_batch(&p, "probe", &[failed("a", "boom")], 1_000).unwrap();
+
+        let (status, attempts, _) = row(&p, "probe", "a");
+        assert_eq!((status.as_str(), attempts), ("failed", 1));
+        // The backoff is computed from the attempt this failure completes, so the
+        // first failure waits 5 minutes rather than 0.
+        assert_eq!(
+            next_retry_at(&p, "probe", "a"),
+            Some(1_000 + retry_backoff_ms(1))
+        );
+
+        retry(&p, "probe", Some("a")).unwrap();
+        claim_batch(&p, "probe", 10, 2_000).unwrap();
+        finish_batch(&p, "probe", &[failed("a", "again")], 2_000).unwrap();
+        // `retry` resets attempts: the counter tracks consecutive automatic
+        // failures, not lifetime ones.
+        assert_eq!(row(&p, "probe", "a").1, 1);
+    }
+
+    #[test]
+    fn reset_running_rescues_tasks_stranded_by_a_crash() {
+        let p = pool();
+        enqueue(&p, "probe", "item", "a", 0, 1).unwrap();
+        enqueue(&p, "thumbs", "item", "b", 0, 1).unwrap();
+        claim_batch(&p, "probe", 10, 2).unwrap();
+        claim_batch(&p, "thumbs", 10, 2).unwrap();
+
+        assert_eq!(reset_running(&p, Some("probe")).unwrap(), 1);
+        assert_eq!(row(&p, "probe", "a").0, "pending");
+        assert_eq!(row(&p, "thumbs", "b").0, "running");
+
+        assert_eq!(reset_running(&p, None).unwrap(), 1);
+        assert_eq!(row(&p, "thumbs", "b").0, "pending");
+    }
+
+    #[test]
+    fn retry_touches_only_failed_tasks_and_raises_their_priority() {
+        let p = pool();
+        enqueue(&p, "probe", "item", "bad", 0, 1).unwrap();
+        enqueue(&p, "probe", "item", "good", 0, 1).unwrap();
+        claim_batch(&p, "probe", 10, 2).unwrap();
+        finish_batch(&p, "probe", &[failed("bad", "boom"), ok("good")], 3).unwrap();
+
+        assert_eq!(retry(&p, "probe", None).unwrap(), 1);
+        let (status, attempts, priority) = row(&p, "probe", "bad");
+        assert_eq!((status.as_str(), attempts), ("pending", 0));
+        assert!(priority > 0);
+        assert_eq!(row(&p, "probe", "good").0, "done");
+    }
+
+    #[test]
+    fn retry_can_name_a_single_subject() {
+        let p = pool();
+        for id in ["a", "b"] {
+            enqueue(&p, "probe", "item", id, 0, 1).unwrap();
+        }
+        claim_batch(&p, "probe", 10, 2).unwrap();
+        finish_batch(&p, "probe", &[failed("a", "x"), failed("b", "y")], 3).unwrap();
+
+        assert_eq!(retry(&p, "probe", Some("a")).unwrap(), 1);
+        assert_eq!(row(&p, "probe", "a").0, "pending");
+        assert_eq!(row(&p, "probe", "b").0, "failed");
+    }
+
+    #[test]
+    fn reprocess_requeues_everything_except_what_is_running() {
+        let p = pool();
+        for id in ["done1", "failed1", "running1"] {
+            enqueue(&p, "probe", "item", id, 0, 1).unwrap();
+        }
+        claim_batch(&p, "probe", 10, 2).unwrap();
+        finish_batch(&p, "probe", &[ok("done1"), failed("failed1", "x")], 3).unwrap();
+
+        assert_eq!(reprocess(&p, "probe").unwrap(), 2);
+        assert_eq!(row(&p, "probe", "done1").0, "pending");
+        assert_eq!(row(&p, "probe", "failed1").0, "pending");
+        assert_eq!(row(&p, "probe", "running1").0, "running");
+    }
+}