@synopackageland/templates 0.9.5 → 0.9.7

package/templates/native-rust/backend/src/app/background/tests.rs

@@ -0,0 +1,505 @@
+use std::sync::{Arc, Mutex};
+
+use serde_json::Value;
+use syno_pkg_land_sdk::BoxError;
+use syno_pkg_land_sdk::api::{self, Caller, Server};
+use syno_pkg_land_sdk::background::{
+    JOB_TIMEOUT, JobResult, MAX_QUEUED, Outcome, State, Store, WorkerInfo,
+};
+use syno_pkg_land_sdk::ctx::Context;
+use syno_pkg_land_sdk::filestation::{Entry, Page};
+use syno_pkg_land_sdk::http::Request;
+use syno_pkg_land_sdk::time::Timestamp;
+
+use super::*;
+use crate::app::test_support::{Stub, TempDir, alice, api_request, bob, nobody, respond, rule};
+
+const POLICY: &[u8] = br#"{"schema":1,"default":"administrators","methods":{}}"#;
+
+#[derive(Default)]
+pub(super) struct Seen {
+    pub calls: usize,
+    pub folder: String,
+    pub rel: String,
+    pub offset: i64,
+}
+
+/// File Station stand-in that records what the routes asked of it.
+pub(super) struct FakeFiles {
+    pub data: Mutex<Vec<u8>>,
+    pub err: Mutex<Option<api::Error>>,
+    pub page: Mutex<Page>,
+    pub seen: Mutex<Seen>,
+}
+
+impl FakeFiles {
+    fn answer<T>(&self, ok: T) -> Result<T, BoxError> {
+        match self.err.lock().unwrap().clone() {
+            Some(e) => Err(e.into()),
+            None => Ok(ok),
+        }
+    }
+
+    pub fn calls(&self) -> usize {
+        self.seen.lock().unwrap().calls
+    }
+}
+
+impl BackgroundFiles for FakeFiles {
+    fn browse(
+        &self,
+        _: &Context,
+        _: &Caller,
+        _: &Request,
+        folder: &str,
+        offset: i64,
+    ) -> Result<Page, BoxError> {
+        {
+            let mut seen = self.seen.lock().unwrap();
+            seen.calls += 1;
+            seen.folder = folder.into();
+            seen.offset = offset;
+        }
+        self.answer(self.page.lock().unwrap().clone())
+    }
+
+    fn read_file(
+        &self,
+        _: &Context,
+        _: &Caller,
+        _: &Request,
+        folder: &str,
+        rel: &str,
+    ) -> Result<Vec<u8>, BoxError> {
+        {
+            let mut seen = self.seen.lock().unwrap();
+            seen.calls += 1;
+            seen.folder = folder.into();
+            seen.rel = rel.into();
+        }
+        self.answer(self.data.lock().unwrap().clone())
+    }
+}
+
+type Lookup = Arc<dyn Fn(&str) -> String + Send + Sync>;
+
+pub(super) struct Env {
+    pub store: Arc<Store>,
+    pub files: Arc<FakeFiles>,
+    uid: Arc<Mutex<Lookup>>,
+    _dir: TempDir,
+}
+
+impl Env {
+    pub fn new() -> Self {
+        let dir = TempDir::new();
+        let files = Arc::new(FakeFiles {
+            data: Mutex::new(b"hello".to_vec()),
+            err: Mutex::new(None),
+            page: Mutex::new(Page::default()),
+            seen: Mutex::new(Seen::default()),
+        });
+        Env {
+            store: Arc::new(Store::new(dir.path().join("background"))),
+            files,
+            uid: Arc::new(Mutex::new(Arc::new(|_: &str| "1024".to_string()))),
+            _dir: dir,
+        }
+    }
+
+    /// Changes what the next request resolves DSM user names to.
+    pub fn set_uid(&self, lookup: impl Fn(&str) -> String + Send + Sync + 'static) {
+        *self.uid.lock().unwrap() = Arc::new(lookup);
+    }
+
+    pub fn api(&self) -> Arc<BackgroundApi> {
+        let uid = self.uid.clone();
+        BackgroundApi::new(
+            self.store.clone(),
+            self.files.clone(),
+            Arc::new(move |name| (uid.lock().unwrap().clone())(name)),
+        )
+    }
+
+    pub fn call(&self, auth: &Stub, method: &str, params: &str) -> (String, Value) {
+        call_routes(self.api().routes(), auth, method, params)
+    }
+}
+
+pub(super) fn call_routes(
+    routes: Vec<api::Route>,
+    auth: &Stub,
+    method: &str,
+    params: &str,
+) -> (String, Value) {
+    let server = Server::new_with_permissions(auth.clone(), POLICY, routes)
+        .expect("the background routes and policy are valid");
+    respond(&server, &mut api_request(method, params))
+}
+
+pub(super) fn data(out: &Value) -> &Value {
+    &out["data"]
+}
+
+pub(super) fn id_params(id: &str) -> String {
+    format!(r#"{{"id":"{id}"}}"#)
+}
+
+fn zeros() -> String {
+    "0".repeat(32)
+}
+
+pub(super) fn sample_params(method: &str) -> String {
+    let id = id_params(&"a".repeat(32));
+    match method {
+        "bg.schedules.list" | "bg.status" | "bg.jobs.list" => "{}".into(),
+        "bg.schedules.create" => {
+            r#"{"key":"schedule","jobId":"missing","intervalSeconds":3600,"consent":true}"#.into()
+        }
+        "bg.schedules.disable" | "bg.schedules.delete" => r#"{"id":"missing"}"#.into(),
+        "bg.schedules.enable" => r#"{"id":"missing","consent":true}"#.into(),
+        "bg.jobs.purge" => r#"{"owner":{"user":"bob","uid":"1025"}}"#.into(),
+        "bg.browse" => r#"{"folder":"/share","offset":0}"#.into(),
+        "bg.jobs.submit" => r#"{"key":"k1","folder":"/share","relative":"a.txt"}"#.into(),
+        "bg.jobs.get" | "bg.jobs.cancel" | "bg.jobs.pause" | "bg.jobs.resume"
+        | "bg.jobs.result" => id,
+        other => panic!("no sample for {other}"),
+    }
+}
+
+#[test]
+fn sdk_policies_authenticate_every_route() {
+    let env = Env::new();
+    let routes = env.api().routes();
+    assert_eq!(routes.len(), 15);
+    for route in &routes {
+        let method = route.method();
+        assert_ne!(rule(route), "public", "{method} must not be public");
+        let params = sample_params(method);
+        let (code, _) = env.call(&bob(), method, &params);
+        if method == "bg.jobs.purge" || method == "bg.schedules.enable" {
+            assert_eq!(code, "admin_required", "{method} as ordinary user");
+        } else {
+            assert!(
+                !["admin_required", "unsupported", "disabled"].contains(&code.as_str()),
+                "{method} unexpectedly blocked: {code}"
+            );
+        }
+        assert_eq!(
+            env.call(&nobody(), method, &params).0,
+            "session_required",
+            "{method} without a session"
+        );
+    }
+}
+
+#[test]
+fn status_reports_limits_and_worker() {
+    let env = Env::new();
+    let (code, out) = env.call(&alice(), "bg.status", "{}");
+    let d = data(&out);
+    assert_eq!(code, "", "{out}");
+    assert_eq!(d["algorithm"], "sha256-v1");
+    assert!(d["worker"].is_null());
+    let limits = &d["limits"];
+    assert_eq!(limits["maxInputBytes"], 512 * 1024);
+    assert_eq!(limits["maxQueued"], 16);
+    assert_eq!(limits["maxTotalBytes"], 16 << 20);
+    assert_eq!(limits["jobTimeoutSeconds"], JOB_TIMEOUT.as_secs());
+    env.store
+        .heartbeat(
+            &Context::background(),
+            WorkerInfo {
+                pid: 9,
+                uid: 255853,
+                username: "pkg".into(),
+                ..WorkerInfo::default()
+            },
+        )
+        .unwrap();
+    let (_, out) = env.call(&alice(), "bg.status", "{}");
+    let worker = &data(&out)["worker"];
+    assert_eq!(worker["alive"], true);
+    assert_eq!(worker["uid"], 255853);
+}
+
+#[test]
+fn submit_is_available_on_a_fresh_store() {
+    let env = Env::new();
+    let (code, _) = env.call(&alice(), "bg.jobs.submit", &sample_params("bg.jobs.submit"));
+    assert_eq!((code.as_str(), env.files.calls()), ("", 1));
+}
+
+#[test]
+fn submit_imports_the_selected_file_and_is_idempotent() {
+    let env = Env::new();
+    let params = r#"{"key":"k1","folder":"/share","relative":"docs/a.txt"}"#;
+    let (code, out) = env.call(&alice(), "bg.jobs.submit", params);
+    let job = &data(&out)["job"];
+    assert_eq!(code, "", "{out}");
+    assert_eq!(job["state"], "queued");
+    assert_eq!(job["name"], "a.txt");
+    assert_eq!(data(&out)["duplicate"], false);
+    {
+        let seen = env.files.seen.lock().unwrap();
+        assert_eq!(
+            (seen.folder.as_str(), seen.rel.as_str()),
+            ("/share", "docs/a.txt")
+        );
+    }
+    assert_eq!(job["submitter"]["user"], "alice");
+    assert_eq!(job["submitter"]["uid"], "1024");
+    let (_, again) = env.call(&alice(), "bg.jobs.submit", params);
+    assert_eq!(data(&again)["duplicate"], true);
+    assert_eq!(data(&again)["job"]["id"], job["id"]);
+    *env.files.data.lock().unwrap() = b"other".to_vec();
+    assert_eq!(env.call(&alice(), "bg.jobs.submit", params).0, "conflict");
+}
+
+#[test]
+fn submit_passes_adapter_refusals_through() {
+    let env = Env::new();
+    *env.files.err.lock().unwrap() = Some(api::Error::new("limit-exceeded", "x"));
+    let (code, _) = env.call(&alice(), "bg.jobs.submit", &sample_params("bg.jobs.submit"));
+    assert_eq!(code, "limit-exceeded");
+    let jobs = env.store.list(&Context::background()).unwrap();
+    assert!(
+        jobs.is_empty(),
+        "a failed import must never enter the queue"
+    );
+}
+
+#[test]
+fn submit_validates_params() {
+    let env = Env::new();
+    for params in [
+        r#"{}"#,
+        r#"{"key":"k","folder":"/share","relative":"a","extra":1}"#,
+        r#"{"key":"bad key","folder":"/share","relative":"a"}"#,
+        r#"{"key":"k","folder":"/share","relative":""}"#,
+    ] {
+        assert_eq!(
+            env.call(&alice(), "bg.jobs.submit", params).0,
+            "bad_request",
+            "{params}"
+        );
+    }
+}
+
+#[test]
+fn queue_full_has_a_stable_code() {
+    let env = Env::new();
+    for i in 0..MAX_QUEUED {
+        env.set_uid(move |_| (1024 + i / 2).to_string());
+        let params = format!(
+            r#"{{"key":"k{}","folder":"/share","relative":"a.txt"}}"#,
+            (b'a' + i as u8) as char
+        );
+        let (code, _) = env.call(&alice(), "bg.jobs.submit", &params);
+        assert_eq!(code, "", "{i}");
+    }
+    env.set_uid(|_| "9999".into());
+    let params = r#"{"key":"more","folder":"/share","relative":"a.txt"}"#;
+    assert_eq!(env.call(&alice(), "bg.jobs.submit", params).0, "queue-full");
+}
+
+fn submitted_id(env: &Env) -> String {
+    let (_, out) = env.call(
+        &alice(),
+        "bg.jobs.submit",
+        r#"{"key":"k1","folder":"/share","relative":"a.txt"}"#,
+    );
+    data(&out)["job"]["id"].as_str().unwrap().into()
+}
+
+#[test]
+fn list_get_cancel_and_result() {
+    let env = Env::new();
+    let id = submitted_id(&env);
+    let params = id_params(&id);
+
+    let (_, out) = env.call(&alice(), "bg.jobs.list", "{}");
+    assert_eq!(data(&out)["jobs"].as_array().unwrap().len(), 1);
+    let (code, out) = env.call(&alice(), "bg.jobs.get", &params);
+    assert_eq!(
+        (code.as_str(), &data(&out)["job"]["id"]),
+        ("", &Value::from(id.clone()))
+    );
+    assert_eq!(
+        env.call(&alice(), "bg.jobs.result", &params).0,
+        "not_ready",
+        "result of a queued job"
+    );
+    let ctx = Context::background();
+    env.store.claim(&ctx).unwrap();
+    let sum = "2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824";
+    env.store
+        .finish(
+            &ctx,
+            &id,
+            Outcome {
+                state: State::Succeeded,
+                result: Some(JobResult {
+                    sha256: sum.into(),
+                    bytes: 5,
+                }),
+                ..Outcome::default()
+            },
+        )
+        .unwrap();
+    let (code, out) = env.call(&alice(), "bg.jobs.result", &params);
+    let result = &data(&out)["result"];
+    assert_eq!(code, "", "{out}");
+    assert_eq!(result["sha256"], sum);
+    assert_eq!(result["bytes"], 5);
+    assert_eq!(result["worker"], "sha256-v1");
+    assert_eq!(result["jobId"], id.as_str());
+    assert_eq!(
+        env.call(&alice(), "bg.jobs.cancel", &params).0,
+        "conflict",
+        "cancel after completion"
+    );
+    for method in ["bg.jobs.get", "bg.jobs.cancel", "bg.jobs.result"] {
+        assert_eq!(
+            env.call(&alice(), method, &id_params(&zeros())).0,
+            "not_found",
+            "{method} unknown id"
+        );
+    }
+}
+
+#[test]
+fn cancel_a_queued_job() {
+    let env = Env::new();
+    let id = submitted_id(&env);
+    let (code, out) = env.call(&alice(), "bg.jobs.cancel", &id_params(&id));
+    assert_eq!(code, "", "{out}");
+    assert_eq!(data(&out)["job"]["state"], "canceled");
+}
+
+#[test]
+fn pause_and_resume_a_queued_job() {
+    let env = Env::new();
+    let params = id_params(&submitted_id(&env));
+    for (method, state) in [("bg.jobs.pause", "paused"), ("bg.jobs.resume", "queued")] {
+        let (code, out) = env.call(&alice(), method, &params);
+        assert_eq!(code, "", "{method}: {out}");
+        assert_eq!(data(&out)["job"]["state"], state, "{method}");
+    }
+    assert_eq!(env.call(&alice(), "bg.jobs.cancel", &params).0, "");
+    for method in ["bg.jobs.pause", "bg.jobs.resume"] {
+        assert_eq!(
+            env.call(&alice(), method, &params).0,
+            "conflict",
+            "{method} after cancel"
+        );
+        let unknown = id_params(&"0".repeat(32));
+        assert_eq!(
+            env.call(&alice(), method, &unknown).0,
+            "not_found",
+            "{method} unknown id"
+        );
+    }
+}
+
+#[test]
+fn worker_alive_expires() {
+    let at = |ago: i128| WorkerInfo {
+        at: Timestamp::now().add_nanos(-ago * 1_000_000_000),
+        ..WorkerInfo::default()
+    };
+    assert!(!worker_alive(&at(60), Timestamp::now()));
+    assert!(worker_alive(&at(0), Timestamp::now()));
+}
+
+#[test]
+fn browse_lists_through_the_callers_session() {
+    let env = Env::new();
+    *env.files.page.lock().unwrap() = Page {
+        files: vec![Entry {
+            name: "a.txt".into(),
+            path: "/share/a.txt".into(),
+            ..Entry::default()
+        }],
+        total: 1,
+        ..Page::default()
+    };
+    let (code, out) = env.call(&alice(), "bg.browse", r#"{"folder":"/share","offset":100}"#);
+    assert_eq!(code, "", "{out}");
+    {
+        let seen = env.files.seen.lock().unwrap();
+        assert_eq!((seen.folder.as_str(), seen.offset), ("/share", 100));
+    }
+    let files = data(&out)["files"].as_array().unwrap();
+    assert_eq!(files.len(), 1);
+    assert_eq!(files[0]["path"], "/share/a.txt");
+}
+
+#[test]
+fn browse_without_a_folder_lists_the_shares() {
+    let env = Env::new();
+    let (code, out) = env.call(&alice(), "bg.browse", "{}");
+    assert_eq!(code, "", "{out}");
+    let seen = env.files.seen.lock().unwrap();
+    assert_eq!((seen.folder.as_str(), seen.offset), ("", 0));
+}
+
+#[test]
+fn browse_validates_params_and_passes_refusals_through() {
+    let env = Env::new();
+    for params in [
+        r#"{"offset":-1}"#,
+        r#"{"folder":"/share","extra":1}"#,
+        r#"{"offset":"1"}"#,
+    ] {
+        assert_eq!(
+            env.call(&alice(), "bg.browse", params).0,
+            "bad_request",
+            "{params}"
+        );
+    }
+    assert_eq!(
+        env.files.calls(),
+        0,
+        "invalid params must not reach the NAS"
+    );
+    *env.files.err.lock().unwrap() = Some(api::Error::new("denied", "no"));
+    assert_eq!(
+        env.call(&alice(), "bg.browse", r#"{"folder":"/share"}"#).0,
+        "denied"
+    );
+}
+
+#[test]
+fn list_without_jobs_is_an_empty_array() {
+    let env = Env::new();
+    let (_, out) = env.call(&alice(), "bg.jobs.list", "{}");
+    assert_eq!(data(&out)["jobs"], serde_json::json!([]));
+}
+
+#[test]
+fn base_name_follows_gos_path_base() {
+    for (path, name) in [
+        ("docs/a.txt", "a.txt"),
+        ("a.txt", "a.txt"),
+        ("a/b/", "b"),
+        ("", "."),
+        ("///", "/"),
+    ] {
+        assert_eq!(base_name(path), name, "{path:?}");
+    }
+}
+
+#[test]
+fn background_error_maps_the_spool_and_keeps_context_errors() {
+    let code = |err: BoxError| {
+        api::find_error::<api::Error>(&*background_error(err))
+            .map(|e| e.code.clone())
+            .unwrap_or_default()
+    };
+    assert_eq!(code(Box::new(StoreError::Busy)), "jobs_busy");
+    assert_eq!(code(Box::new(StoreError::NotRunning)), "unavailable");
+    assert_eq!(code("disk on fire".into()), "unavailable");
+    let canceled = background_error(Box::new(CtxError::Canceled));
+    assert!(api::find_error::<CtxError>(&*canceled).is_some());
+}

package/templates/native-rust/backend/src/app/background/work.rs

@@ -0,0 +1,78 @@
+//! The App's own sha256-v1 Work, resumable after a pause or a package stop.
+
+use std::io::{self, ErrorKind, Read};
+
+use serde::{Deserialize, Serialize};
+use syno_pkg_land_sdk::BoxError;
+use syno_pkg_land_sdk::background::{JobResult, Progress, Task};
+use syno_pkg_land_sdk::ctx::Context;
+use syno_pkg_land_sdk::sha256::{Sha256, hex};
+
+/// Spaces [`hash_work`]'s checkpoints; a resumed job repeats at most this
+/// much hashing. A Work over larger input saves less often.
+pub(super) const HASH_CHECKPOINT_EVERY: i64 = 64 << 10;
+
+/// The checkpoint: the bytes already hashed and the hash state after them.
+#[derive(Serialize, Deserialize)]
+struct HashState {
+    offset: i64,
+    state: String,
+}
+
+/// Reports progress as it reads and checkpoints the hash state, so a job
+/// paused by its owner or by a package stop resumes where it left off
+/// instead of from the start.
+pub fn hash_work(ctx: &Context, t: &mut Task<'_>) -> Result<JobResult, BoxError> {
+    let (mut h, mut done) = resume_hash(&t.job.checkpoint, t.input)?;
+    let (total, mut saved) = (t.job.input_bytes, done);
+    let mut buf = vec![0; 32 * 1024];
+    loop {
+        if let Some(e) = ctx.err() {
+            return Err(Box::new(e));
+        }
+        let n = match t.input.read(&mut buf) {
+            Ok(n) => n,
+            Err(e) if e.kind() == ErrorKind::Interrupted => continue,
+            Err(e) => return Err(Box::new(e)),
+        };
+        h.update(&buf[..n]);
+        done += n as i64;
+        t.report(Progress {
+            bytes: done,
+            total_bytes: total,
+            ..Progress::default()
+        });
+        if n == 0 {
+            return Ok(JobResult {
+                sha256: hex(&h.finish()),
+                bytes: done,
+            });
+        }
+        // The hash state saves only after whole blocks; a short read waits for the next.
+        if done - saved >= HASH_CHECKPOINT_EVERY
+            && let Some(state) = save_hash(&h, done)
+        {
+            t.checkpoint(&state)?;
+            saved = done;
+        }
+    }
+}
+
+pub(super) fn save_hash(h: &Sha256, offset: i64) -> Option<String> {
+    let state = h.save()?;
+    serde_json::to_string(&HashState { offset, state }).ok()
+}
+
+/// Continues the hash a checkpoint saved, past the input it covered.
+fn resume_hash(checkpoint: &str, input: &mut dyn Read) -> Result<(Sha256, i64), BoxError> {
+    if checkpoint.is_empty() {
+        return Ok((Sha256::new(), 0));
+    }
+    let saved: HashState = serde_json::from_str(checkpoint)?;
+    let h = Sha256::restore(&saved.state).ok_or("checkpoint: invalid hash state")?;
+    let offset = u64::try_from(saved.offset).map_err(|_| "checkpoint: negative offset")?;
+    if io::copy(&mut input.take(offset), &mut io::sink())? != offset {
+        return Err("checkpoint: input shorter than its offset".into());
+    }
+    Ok((h, saved.offset))
+}

package/templates/native-rust/backend/src/app/background/work_tests.rs

@@ -0,0 +1,123 @@
+use std::io::{Read, empty};
+use std::sync::Arc;
+use std::time::Duration;
+
+use syno_pkg_land_sdk::background::{
+    Job, Runner, State, Store, SubmitRequest, Submitter, Task, WorkerInfo,
+};
+use syno_pkg_land_sdk::ctx::{Cancel, Context};
+use syno_pkg_land_sdk::sha256::{Sha256, hex_digest};
+
+use super::work::{HASH_CHECKPOINT_EVERY, hash_work, save_hash};
+use crate::app::test_support::TempDir;
+
+#[test]
+fn hash_work_resumes_from_its_checkpoint() {
+    let (head, tail) = (vec![b'h'; 1024], vec![b't'; 3000]);
+    let mut h = Sha256::new();
+    h.update(&head);
+    let saved = save_hash(&h, head.len() as i64).unwrap();
+    // The input's first bytes differ from what the checkpoint hashed: only a
+    // Work that skips them and continues the saved hash reaches this digest.
+    let mut input: &[u8] = &[vec![b'x'; head.len()], tail.clone()].concat();
+    let job = Job {
+        checkpoint: saved,
+        ..Job::default()
+    };
+    let got = hash_work(&Context::background(), &mut Task::new(job, &mut input)).unwrap();
+    assert_eq!(got.sha256, hex_digest(&[head, tail].concat()));
+    assert_eq!(got.bytes, 4024);
+}
+
+#[test]
+fn hash_work_refuses_a_foreign_checkpoint() {
+    for saved in [
+        "{",
+        r#"{"offset":-1,"state":""}"#,
+        r#"{"offset":3,"state":"00"}"#,
+    ] {
+        let job = Job {
+            checkpoint: saved.into(),
+            ..Job::default()
+        };
+        let mut input: &[u8] = b"abc";
+        let got = hash_work(&Context::background(), &mut Task::new(job, &mut input));
+        assert!(got.is_err(), "{saved}: accepted");
+    }
+}
+
+/// Ends the worker's context once `left` bytes have been read, like a
+/// package stop in the middle of a job.
+struct StopAfter<'a> {
+    inner: &'a mut dyn Read,
+    left: usize,
+    stop: Arc<Cancel>,
+}
+
+impl Read for StopAfter<'_> {
+    fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> {
+        let n = self.inner.read(buf)?;
+        self.left = self.left.saturating_sub(n);
+        if self.left == 0 {
+            self.stop.cancel();
+        }
+        Ok(n)
+    }
+}
+
+#[test]
+fn hash_work_reports_progress_and_checkpoints_before_a_stop() {
+    let dir = TempDir::new();
+    let store = Arc::new(Store::new(dir.path().join("background")));
+    let ctx = Context::background();
+    let data = b"0123456789abcdef".repeat(16 << 10);
+    let (job, _) = store
+        .submit(
+            &ctx,
+            SubmitRequest {
+                key: "k".into(),
+                name: "a.bin".into(),
+                data: data.clone(),
+                submitter: Submitter {
+                    user: "alice".into(),
+                    uid: "1024".into(),
+                },
+            },
+        )
+        .unwrap();
+    let (worker, stop) = ctx.with_cancel();
+    let stop = Arc::new(stop);
+    let mut runner = Runner {
+        store: store.clone(),
+        work: Arc::new(move |ctx: &Context, t: &mut Task<'_>| {
+            // The task's input lives as long as the task; the test leaks its wrapper.
+            let inner = std::mem::replace(&mut t.input, Box::leak(Box::new(empty())));
+            t.input = Box::leak(Box::new(StopAfter {
+                inner,
+                left: 3 * HASH_CHECKPOINT_EVERY as usize / 2,
+                stop: stop.clone(),
+            }));
+            hash_work(ctx, t)
+        }),
+        timeout: Duration::from_secs(60),
+        cancel_poll: Duration::from_millis(10),
+        ..Runner::new(dir.path().join("background"), WorkerInfo::default())
+    };
+    let claimed = store.claim(&ctx).unwrap().unwrap();
+    runner.process(&worker, &claimed);
+    let paused = store.get(&ctx, &job.id).unwrap();
+    let progress = paused.progress.clone().unwrap();
+    assert_eq!(paused.state, State::Paused, "{paused:?}");
+    assert!(!paused.checkpoint.is_empty());
+    assert!(progress.bytes >= HASH_CHECKPOINT_EVERY, "{progress:?}");
+    assert_eq!(progress.total_bytes, data.len() as i64);
+
+    store.recover(&ctx).unwrap();
+    let claimed = store.claim(&ctx).unwrap().unwrap();
+    assert_eq!(claimed.checkpoint, paused.checkpoint);
+    runner.work = Arc::new(hash_work);
+    runner.process(&ctx, &claimed);
+    let done = store.get(&ctx, &job.id).unwrap();
+    assert_eq!(done.state, State::Succeeded, "{done:?}");
+    assert_eq!(done.progress.unwrap().bytes, data.len() as i64);
+}