create-feltdb 0.11.7 → 0.11.8

package/dist/package-versions.js

@@ -1,4 +1,4 @@
 // One release train keeps generated applications installable. The repository
 // validation script checks these values against every workspace manifest.
-export const FELTDB_PACKAGE_VERSION = '0.11.7';
+export const FELTDB_PACKAGE_VERSION = '0.11.8';
 export const feltdbPackageRange = `^${FELTDB_PACKAGE_VERSION}`;

package/dist/server-source/crates/feltdb-server/src/backup.rs

@@ -4,11 +4,32 @@ use std::{
     fs::{self, File},
     io::{self, Read, Write},
     path::{Path, PathBuf},
-    time::{SystemTime, UNIX_EPOCH},
+    time::{Instant, SystemTime, UNIX_EPOCH},
 };
 
 pub const BACKUP_FORMAT: u32 = 1;
 
+/// Where the backed-up state came from. Optional in the manifest so that
+/// bundles written before provenance was recorded still verify: the format is
+/// unchanged, and an absent field is reported as unknown rather than invented.
+#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq, Eq)]
+pub struct BackupSource {
+    /// Absolute path of the durable state file the bundle was taken from.
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub data: Option<String>,
+    /// SHA-256 of the authoritative state artifact (`state.log`, or the
+    /// consistent `database.snapshot.json` of an online backup).
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub state_sha256: Option<String>,
+    /// The `feltdb-server` crate version that wrote the bundle (not the npm
+    /// package version).
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub server_version: Option<String>,
+    /// The durable format version of the build that wrote the bundle.
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub durable_format: Option<u32>,
+}
+
 #[derive(Clone, Debug, Serialize, Deserialize)]
 pub struct BackupEntry {
     pub path: String,
@@ -21,6 +42,93 @@ pub struct BackupManifest {
     pub format: u32,
     pub created_ms: u64,
     pub entries: Vec<BackupEntry>,
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub source: Option<BackupSource>,
+}
+
+/// The explicit outcome of a backup operation.
+///
+/// `status` is `ok` only when the bundle was written *and* independently
+/// verified; the CLI exit code follows it, so a successful backup is never
+/// reported as a failure and an unverifiable one is never reported as valid.
+#[derive(Clone, Debug, Serialize, Deserialize)]
+pub struct BackupReport {
+    pub operation: String,
+    pub status: String,
+    /// Absolute path of the bundle.
+    pub path: String,
+    /// SHA-256 of the bundle manifest: the bundle's stable identity.
+    pub backup_id: String,
+    pub source: BackupSource,
+    pub format: u32,
+    pub created_ms: u64,
+    pub files: usize,
+    pub bytes_written: u64,
+    /// `verified` or `failed`.
+    pub verification: String,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub error: Option<String>,
+    pub duration_ms: u64,
+}
+
+/// Resolve a user-supplied path against the working directory.
+///
+/// A bare relative path such as `backup` has an *empty* parent, not `.`; every
+/// operation that syncs or stages next to the output therefore works on the
+/// absolute form, which is what makes `backup`, `./backup` and
+/// `/abs/path/backup` behave identically.
+pub fn normalize_path(path: &Path) -> io::Result<PathBuf> {
+    if path.as_os_str().is_empty() {
+        return Err(io::Error::new(
+            io::ErrorKind::InvalidInput,
+            "backup path must not be empty",
+        ));
+    }
+    std::path::absolute(path)
+}
+
+/// SHA-256 of a bundle's manifest bytes, used as the bundle identity.
+pub fn backup_id(archive: &Path) -> io::Result<String> {
+    Ok(format!(
+        "{:x}",
+        Sha256::digest(fs::read(archive.join("manifest.json"))?)
+    ))
+}
+
+/// Verify a bundle and describe it as a report, without ever turning a
+/// verification failure into success.
+pub fn verification_report(archive: &Path, started: Instant, operation: &str) -> BackupReport {
+    let path = normalize_path(archive).unwrap_or_else(|_| archive.to_path_buf());
+    match verify_bundle(&path).and_then(|manifest| Ok((backup_id(&path)?, manifest))) {
+        Ok((id, manifest)) => BackupReport {
+            operation: operation.into(),
+            status: "ok".into(),
+            path: path.display().to_string(),
+            backup_id: id,
+            source: manifest.source.clone().unwrap_or_default(),
+            format: manifest.format,
+            created_ms: manifest.created_ms,
+            files: manifest.entries.len(),
+            bytes_written: manifest.entries.iter().map(|entry| entry.bytes).sum(),
+            verification: "verified".into(),
+            error: None,
+            duration_ms: started.elapsed().as_millis() as u64,
+        },
+        Err(error) => BackupReport {
+            operation: operation.into(),
+            status: "failed".into(),
+            path: path.display().to_string(),
+            backup_id: backup_id(&path).unwrap_or_default(),
+            source: BackupSource::default(),
+            format: 0,
+            created_ms: 0,
+            files: 0,
+            bytes_written: 0,
+            verification: "failed".into(),
+            error: Some(error.to_string()),
+            duration_ms: started.elapsed().as_millis() as u64,
+        },
+    }
 }
 
 pub fn server_artifacts(data: &Path, keys: &Path, audit: &Path) -> Vec<(String, PathBuf)> {
@@ -72,6 +180,20 @@ pub fn create_bundle_with_virtual(
     artifacts: &[(String, PathBuf)],
     virtual_files: &[(String, Vec<u8>)],
 ) -> io::Result<BackupManifest> {
+    create_bundle_with_source(output, artifacts, virtual_files, BackupSource::default())
+}
+
+/// Write a bundle and record where its state came from.
+///
+/// The bundle is staged beside the output and published with one rename, so a
+/// reader sees either no bundle or a complete one.
+pub fn create_bundle_with_source(
+    output: &Path,
+    artifacts: &[(String, PathBuf)],
+    virtual_files: &[(String, Vec<u8>)],
+    mut source: BackupSource,
+) -> io::Result<BackupManifest> {
+    let output = &normalize_path(output)?;
     if output.exists() {
         return Err(io::Error::new(
             io::ErrorKind::AlreadyExists,
@@ -114,6 +236,14 @@ pub fn create_bundle_with_virtual(
             });
         }
         entries.sort_by(|left, right| left.path.cmp(&right.path));
+        if source.state_sha256.is_none() {
+            source.state_sha256 = entries
+                .iter()
+                .find(|entry| {
+                    entry.path == "files/database.snapshot.json" || entry.path == "files/state.log"
+                })
+                .map(|entry| entry.sha256.clone());
+        }
         let manifest = BackupManifest {
             format: BACKUP_FORMAT,
             created_ms: SystemTime::now()
@@ -121,6 +251,7 @@ pub fn create_bundle_with_virtual(
                 .unwrap_or_default()
                 .as_millis() as u64,
             entries,
+            source: Some(source),
         };
         write_synced(
             &staging.join("manifest.json"),
@@ -128,9 +259,7 @@ pub fn create_bundle_with_virtual(
         )?;
         sync_directory(&staging)?;
         fs::rename(&staging, output)?;
-        if let Some(parent) = output.parent() {
-            sync_directory(parent)?;
-        }
+        sync_parent(output)?;
         Ok(manifest)
     })();
     if result.is_err() {
@@ -140,6 +269,7 @@ pub fn create_bundle_with_virtual(
 }
 
 pub fn verify_bundle(archive: &Path) -> io::Result<BackupManifest> {
+    let archive = &normalize_path(archive)?;
     if archive.with_extension("partial").exists() {
         return Err(io::Error::new(
             io::ErrorKind::InvalidData,
@@ -167,6 +297,25 @@ pub fn verify_bundle(archive: &Path) -> io::Result<BackupManifest> {
             ));
         }
     }
+    // Completeness runs both ways: every manifest entry is present and intact
+    // (above), and nothing unrecorded sits in the bundle. An extra file would be
+    // restored as state that no checksum ever covered.
+    let recorded: std::collections::BTreeSet<&str> = manifest
+        .entries
+        .iter()
+        .map(|entry| entry.path.as_str())
+        .collect();
+    let mut present = Vec::new();
+    collect_files(&archive.join("files"), "files", &mut present)?;
+    if let Some(extra) = present
+        .iter()
+        .find(|path| !recorded.contains(path.as_str()))
+    {
+        return Err(io::Error::new(
+            io::ErrorKind::InvalidData,
+            format!("backup contains a file absent from its manifest: {extra}"),
+        ));
+    }
     Ok(manifest)
 }
 
@@ -174,6 +323,7 @@ pub fn verify_bundle(archive: &Path) -> io::Result<BackupManifest> {
 /// does not exceed `at_ms`. Every manifest-bearing child is verified first so a
 /// corrupt recent backup cannot cause a silent fallback to older data.
 pub fn select_bundle_at(catalog: &Path, at_ms: u64) -> io::Result<PathBuf> {
+    let catalog = &normalize_path(catalog)?;
     let mut candidates = Vec::new();
     for entry in fs::read_dir(catalog)? {
         let path = entry?.path();
@@ -198,6 +348,8 @@ pub fn select_bundle_at(catalog: &Path, at_ms: u64) -> io::Result<PathBuf> {
 }
 
 pub fn restore_bundle(archive: &Path, output: &Path) -> io::Result<BackupManifest> {
+    let archive = &normalize_path(archive)?;
+    let output = &normalize_path(output)?;
     let manifest = verify_bundle(archive)?;
     if output.exists() {
         return Err(io::Error::new(
@@ -221,9 +373,7 @@ pub fn restore_bundle(archive: &Path, output: &Path) -> io::Result<BackupManifes
         )?;
         sync_directory(&staging)?;
         fs::rename(&staging, output)?;
-        if let Some(parent) = output.parent() {
-            sync_directory(parent)?;
-        }
+        sync_parent(output)?;
         Ok(manifest.clone())
     })();
     if result.is_err() {
@@ -271,6 +421,22 @@ fn copy_source(
     }
     Ok(())
 }
+fn collect_files(directory: &Path, relative: &str, found: &mut Vec<String>) -> io::Result<()> {
+    if !directory.exists() {
+        return Ok(());
+    }
+    for child in fs::read_dir(directory)? {
+        let child = child?;
+        let name = format!("{relative}/{}", child.file_name().to_string_lossy());
+        let kind = child.file_type()?;
+        if kind.is_dir() {
+            collect_files(&child.path(), &name, found)?;
+        } else {
+            found.push(name);
+        }
+    }
+    Ok(())
+}
 fn copy_tree(source: &Path, destination: &Path) -> io::Result<()> {
     if !source.exists() {
         return Ok(());
@@ -322,6 +488,14 @@ fn sync_file(path: &Path) -> io::Result<()> {
 fn sync_directory(path: &Path) -> io::Result<()> {
     File::open(path)?.sync_all()
 }
+/// Sync the directory that holds `path`. A relative path with no directory
+/// component lives in the working directory; its parent is `.`, never `""`.
+fn sync_parent(path: &Path) -> io::Result<()> {
+    match path.parent() {
+        Some(parent) if !parent.as_os_str().is_empty() => sync_directory(parent),
+        _ => sync_directory(Path::new(".")),
+    }
+}
 
 #[cfg(test)]
 mod tests {
@@ -409,4 +583,113 @@ mod tests {
         assert!(select_bundle_at(&root.join("catalog"), first.created_ms - 1).is_err());
         fs::remove_dir_all(root).unwrap();
     }
+
+    #[test]
+    fn relative_and_absolute_paths_normalize_identically() {
+        let cwd = std::env::current_dir().unwrap();
+        assert_eq!(
+            normalize_path(Path::new("backup")).unwrap(),
+            cwd.join("backup")
+        );
+        assert_eq!(
+            normalize_path(Path::new("./backup")).unwrap(),
+            cwd.join("./backup")
+        );
+        assert_eq!(
+            normalize_path(&cwd.join("backup")).unwrap(),
+            cwd.join("backup")
+        );
+        assert!(normalize_path(Path::new("")).is_err());
+        // The defect: a bare name has an empty parent, which cannot be opened
+        // for the post-rename directory sync. `sync_parent` must treat it as `.`.
+        assert_eq!(Path::new("backup").parent(), Some(Path::new("")));
+        sync_parent(Path::new("backup")).unwrap();
+    }
+
+    #[test]
+    fn backup_into_missing_parent_records_provenance_and_verifies() {
+        let root = std::env::temp_dir().join(format!(
+            "feltdb-provenance-{}-{}",
+            std::process::id(),
+            rand::random::<u64>()
+        ));
+        fs::create_dir_all(&root).unwrap();
+        fs::write(root.join("state.log"), b"state").unwrap();
+        let archive = root.join("does/not/exist/yet/archive");
+        let manifest = create_bundle_with_source(
+            &archive,
+            &[("state.log".into(), root.join("state.log"))],
+            &[],
+            BackupSource {
+                data: Some("/data/state.log".into()),
+                ..BackupSource::default()
+            },
+        )
+        .unwrap();
+        let source = manifest.source.unwrap();
+        assert_eq!(source.data.as_deref(), Some("/data/state.log"));
+        assert_eq!(
+            source.state_sha256.as_deref(),
+            Some(manifest.entries[0].sha256.as_str())
+        );
+        let report = verification_report(&archive, Instant::now(), "verify");
+        assert_eq!(report.status, "ok");
+        assert_eq!(report.verification, "verified");
+        assert_eq!(report.bytes_written, 5);
+        assert_eq!(report.backup_id, backup_id(&archive).unwrap());
+        fs::remove_dir_all(root).unwrap();
+    }
+
+    #[test]
+    fn unrecorded_files_and_missing_manifests_fail_verification() {
+        let root = std::env::temp_dir().join(format!(
+            "feltdb-completeness-{}-{}",
+            std::process::id(),
+            rand::random::<u64>()
+        ));
+        fs::create_dir_all(&root).unwrap();
+        fs::write(root.join("state.log"), b"state").unwrap();
+        let archive = root.join("archive");
+        create_bundle(&archive, &[("state.log".into(), root.join("state.log"))]).unwrap();
+        fs::write(archive.join("files/injected"), b"unrecorded").unwrap();
+        let report = verification_report(&archive, Instant::now(), "verify");
+        assert_eq!(report.status, "failed");
+        assert!(report.error.unwrap().contains("absent from its manifest"));
+        fs::remove_file(archive.join("files/injected")).unwrap();
+        fs::remove_file(archive.join("files/state.log")).unwrap();
+        assert_eq!(
+            verification_report(&archive, Instant::now(), "verify").status,
+            "failed"
+        );
+        fs::remove_file(archive.join("manifest.json")).unwrap();
+        assert_eq!(
+            verification_report(&archive, Instant::now(), "verify").verification,
+            "failed"
+        );
+        fs::remove_dir_all(root).unwrap();
+    }
+
+    #[test]
+    fn manifests_written_before_provenance_still_verify() {
+        let root = std::env::temp_dir().join(format!(
+            "feltdb-legacy-manifest-{}-{}",
+            std::process::id(),
+            rand::random::<u64>()
+        ));
+        fs::create_dir_all(root.join("archive/files")).unwrap();
+        fs::write(root.join("archive/files/state.log"), b"legacy").unwrap();
+        let digest = format!("{:x}", Sha256::digest(b"legacy"));
+        fs::write(
+            root.join("archive/manifest.json"),
+            format!(r#"{{"format":1,"created_ms":1,"entries":[{{"path":"files/state.log","bytes":6,"sha256":"{digest}"}}]}}"#),
+        )
+        .unwrap();
+        let manifest = verify_bundle(&root.join("archive")).unwrap();
+        assert!(manifest.source.is_none());
+        assert_eq!(
+            verification_report(&root.join("archive"), Instant::now(), "verify").source,
+            BackupSource::default()
+        );
+        fs::remove_dir_all(root).unwrap();
+    }
 }

package/dist/server-source/crates/feltdb-server/src/main.rs

@@ -75,7 +75,7 @@ use feltdb_server::{
     authenticated_principal::AuthenticatedPrincipal,
     authorization::{authorize, AuthorizationRequest, Subject},
     backup::{
-        create_bundle, create_bundle_with_virtual, restore_bundle, select_bundle_at,
+        restore_bundle, select_bundle_at,
         server_artifacts, verify_bundle,
     },
     causal::{
@@ -8529,6 +8529,9 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
     if manage_backup()? {
         return Ok(());
     }
+    if manage_doctor()? {
+        return Ok(());
+    }
     if manage_application_runtime()? {
         return Ok(());
     }
@@ -9989,8 +9992,16 @@ async fn health_ready(State(state): State<AppState>) -> axum::response::Response
     match outcome {
         Ok(Ok(Ok(health))) => {
             let ready = health.is_nominal();
+            let recovering = matches!(
+                state.cluster.proposal().map(|value| value.phase),
+                Some(ProposalPhase::Preparing | ProposalPhase::Prepared)
+            );
             let body = Json(json!({
                 "ready": ready,
+                // The explicit health model. A running server has already
+                // refused a corrupt or incompatible log at open, so `corrupt`
+                // is reported offline by `feltdb-server doctor`, never here.
+                "state": if !ready { "degraded" } else if recovering { "recovering" } else { "healthy" },
                 "storage": health.storage.label(),
                 "durable_format": health.durable_format.to_string(),
                 "probe_ms": waited_ms,
@@ -10006,6 +10017,7 @@ async fn health_ready(State(state): State<AppState>) -> axum::response::Response
             StatusCode::SERVICE_UNAVAILABLE,
             Json(json!({
                 "ready": false,
+                "state": "unavailable",
                 "error": error.to_string(),
                 "code": "STORAGE_UNAVAILABLE",
                 "probe_ms": waited_ms,
@@ -10016,6 +10028,7 @@ async fn health_ready(State(state): State<AppState>) -> axum::response::Response
             StatusCode::SERVICE_UNAVAILABLE,
             Json(json!({
                 "ready": false,
+                "state": "unavailable",
                 "error": error.to_string(),
                 "code": "STORAGE_PROBE_FAILED",
                 "probe_ms": waited_ms,
@@ -10026,6 +10039,7 @@ async fn health_ready(State(state): State<AppState>) -> axum::response::Response
             StatusCode::SERVICE_UNAVAILABLE,
             Json(json!({
                 "ready": false,
+                "state": "unavailable",
                 "error": "storage probe exceeded the server deadline",
                 "code": "STORAGE_STALLED",
                 "probe_ms": waited_ms,
@@ -10370,7 +10384,6 @@ fn manage_backup() -> Result<bool, Box<dyn std::error::Error>> {
     if arguments.get(1).map(String::as_str) != Some("backup") {
         return Ok(false);
     }
-    let command = arguments.get(2).map(String::as_str).unwrap_or("");
     let option = |name: &str| {
         arguments
             .iter()
@@ -10378,27 +10391,105 @@ fn manage_backup() -> Result<bool, Box<dyn std::error::Error>> {
             .and_then(|index| arguments.get(index + 1))
             .cloned()
     };
+    let json_output = arguments.iter().any(|value| value == "--json");
+    // `backup <path>` is `backup create --output <path>`; `backup verify <path>`
+    // is `backup verify --archive <path>`. The flag forms remain supported.
+    let positional = |from: usize| {
+        let mut index = from;
+        while let Some(value) = arguments.get(index) {
+            if value == "--json" {
+                index += 1;
+            } else if value.starts_with("--") {
+                index += 2;
+            } else {
+                return Some(value.clone());
+            }
+        }
+        None
+    };
+    let (command, target) = match arguments.get(2).map(String::as_str) {
+        Some(command @ ("create" | "verify" | "restore")) => (command, positional(3)),
+        Some(value) if !value.starts_with("--") => ("create", Some(value.to_string())),
+        _ if option("--output").is_some() && option("--archive").is_none() => ("create", None),
+        _ => ("", None),
+    };
+    let usage = "usage: feltdb-server backup [create] <output> | backup verify <archive> | backup restore --archive <archive> --output <directory> [--at-ms timestamp]  (options: --data path --keys path --audit path --json)";
+    let started = Instant::now();
+    let emit = |report: &feltdb_server::backup::BackupReport, human: String| {
+        if json_output {
+            println!("{}", serde_json::to_string_pretty(report).unwrap_or_default());
+        } else if report.status == "ok" {
+            println!("{human}");
+        } else {
+            eprintln!(
+                "Backup {} failed: {} ({})",
+                report.operation,
+                report.path,
+                report.error.as_deref().unwrap_or("unknown error")
+            );
+        }
+    };
     match command {
         "create" => {
             let data = PathBuf::from(option("--data").unwrap_or_else(|| "./data/feltdb.log".into()));
             let keys = PathBuf::from(option("--keys").unwrap_or_else(|| "./data/api-keys.json".into()));
             let audit = PathBuf::from(option("--audit").unwrap_or_else(|| data.with_extension("audit.log").to_string_lossy().into_owned()));
-            let output = PathBuf::from(option("--output").ok_or("backup create requires --output")?);
-            let manifest = create_bundle(&output, &server_artifacts(&data, &keys, &audit))?;
-            println!("Backup created: {} (format {}, {} files)", output.display(), manifest.format, manifest.entries.len());
+            let output = option("--output").or(target).ok_or("backup create requires an output path")?;
+            let output = feltdb_server::backup::normalize_path(std::path::Path::new(&output))?;
+            let data = feltdb_server::backup::normalize_path(&data)?;
+            if !data.exists() {
+                return Err(format!("backup source does not exist: {}", data.display()).into());
+            }
+            let source = feltdb_server::backup::BackupSource {
+                data: Some(data.display().to_string()),
+                state_sha256: None,
+                server_version: Some(env!("CARGO_PKG_VERSION").into()),
+                durable_format: Some(feltdb::DURABLE_FORMAT_VERSION),
+            };
+            feltdb_server::backup::create_bundle_with_source(&output, &server_artifacts(&data, &keys, &audit), &[], source)?;
+            // A backup is reported as successful only after it verifies
+            // independently of the code path that wrote it.
+            let mut report = feltdb_server::backup::verification_report(&output, started, "create");
+            if report.status == "ok" {
+                if let Err(error) = verify_restorable(&output) {
+                    report.status = "failed".into();
+                    report.verification = "failed".into();
+                    report.error = Some(error.to_string());
+                }
+            }
+            let human = format!("Backup created: {} (id {}, format {}, {} files, {} bytes, verified, {} ms)", report.path, &report.backup_id[..report.backup_id.len().min(12)], report.format, report.files, report.bytes_written, report.duration_ms);
+            emit(&report, human);
+            if report.status != "ok" {
+                std::process::exit(1);
+            }
         }
         "verify" => {
-            let archive = PathBuf::from(option("--archive").ok_or("backup verify requires --archive")?);
-            let manifest = verify_bundle(&archive)?;
-            println!("Backup verified: {} (format {}, {} files)", archive.display(), manifest.format, manifest.entries.len());
+            let archive = option("--archive").or(target).ok_or("backup verify requires an archive path")?;
+            let mut report = feltdb_server::backup::verification_report(std::path::Path::new(&archive), started, "verify");
+            let mut restorable = None;
+            if report.status == "ok" {
+                match verify_restorable(std::path::Path::new(&report.path)) {
+                    Ok(detail) => restorable = Some(detail),
+                    Err(error) => {
+                        report.status = "failed".into();
+                        report.verification = "failed".into();
+                        report.error = Some(error.to_string());
+                    }
+                }
+            }
+            let human = format!("Backup verified: {} (id {}, format {}, {} files, {} bytes{})", report.path, &report.backup_id[..report.backup_id.len().min(12)], report.format, report.files, report.bytes_written, restorable.map(|detail| format!(", {detail}")).unwrap_or_default());
+            emit(&report, human);
+            if report.status != "ok" {
+                std::process::exit(1);
+            }
         }
         "restore" => {
-            let archive = PathBuf::from(option("--archive").ok_or("backup restore requires --archive")?);
+            let archive = PathBuf::from(option("--archive").or(target).ok_or("backup restore requires --archive")?);
             let archive = match option("--at-ms") {
                 Some(value) => select_bundle_at(&archive, value.parse().map_err(|_| "backup restore --at-ms must be an unsigned millisecond timestamp")?)?,
                 None => archive,
             };
-            let output = PathBuf::from(option("--output").ok_or("backup restore requires --output")?);
+            let output = feltdb_server::backup::normalize_path(std::path::Path::new(&option("--output").ok_or("backup restore requires --output")?))?;
             let manifest = restore_bundle(&archive, &output)?;
             let snapshot_path = output.join("files/database.snapshot.json");
             if snapshot_path.exists() {
@@ -10406,13 +10497,199 @@ fn manage_backup() -> Result<bool, Box<dyn std::error::Error>> {
                 let database = FeltDb::open(output.join("files/state.log"))?;
                 database.install_snapshot(snapshot)?;
             }
-            println!("Backup restored: {} (format {}, {} files)", output.display(), manifest.format, manifest.entries.len());
+            let mut report = feltdb_server::backup::verification_report(&archive, started, "restore");
+            report.path = output.display().to_string();
+            emit(&report, format!("Backup restored: {} (format {}, {} files)", output.display(), manifest.format, manifest.entries.len()));
+            if report.status != "ok" {
+                std::process::exit(1);
+            }
         }
-        _ => return Err("usage: feltdb-server backup <create|verify|restore> [--data path] [--keys path] [--audit path] --output directory | --archive directory [--at-ms timestamp]".into()),
+        _ => return Err(usage.into()),
     }
     Ok(true)
 }
 
+/// `feltdb-server doctor` — what an operator needs to know, offline.
+///
+/// Answers, for a data path: is the database healthy; is the durable log
+/// valid; what state/version is it in; is recovery pending; can a backup be
+/// created; can that backup be restored; which server and format versions are
+/// active. It never opens the live files: the state is bundled exactly as
+/// `backup create` would, and every probe — open, read, write — runs against a
+/// private restored copy. `--backup <archive>` additionally verifies an
+/// existing bundle.
+///
+/// The health state is one of `healthy`, `recovering`, `degraded`,
+/// `unavailable`, or `corrupt`, and `healthy` requires every probe to pass:
+/// a readable durable log in a supported format, a clean replay, a working
+/// read, a working write, and a restorable backup.
+fn manage_doctor() -> Result<bool, Box<dyn std::error::Error>> {
+    let arguments: Vec<String> = std::env::args().collect();
+    if arguments.get(1).map(String::as_str) != Some("doctor") {
+        return Ok(false);
+    }
+    let option = |name: &str| {
+        arguments
+            .iter()
+            .position(|value| value == name)
+            .and_then(|index| arguments.get(index + 1))
+            .cloned()
+    };
+    let json_output = arguments.iter().any(|value| value == "--json");
+    let data = feltdb_server::backup::normalize_path(&PathBuf::from(
+        option("--data").unwrap_or_else(|| "./data/feltdb.log".into()),
+    ))?;
+    let keys = PathBuf::from(option("--keys").unwrap_or_else(|| "./data/api-keys.json".into()));
+    let audit = PathBuf::from(option("--audit").unwrap_or_else(|| data.with_extension("audit.log").to_string_lossy().into_owned()));
+
+    let mut checks: Vec<Value> = Vec::new();
+    let mut check = |name: &str, ok: bool, detail: String| {
+        checks.push(json!({ "name": name, "status": if ok { "PASS" } else { "FAIL" }, "detail": detail }));
+        ok
+    };
+    let scratch = std::env::temp_dir().join(format!(
+        "feltdb-doctor-{}-{}",
+        std::process::id(),
+        rand::random::<u64>()
+    ));
+    let bundle = scratch.join("bundle");
+    let restored = scratch.join("restored");
+    let mut state = "healthy";
+    let mut facts = json!({
+        "data": data.display().to_string(),
+        "server_version": env!("CARGO_PKG_VERSION"),
+        "supported_durable_format": feltdb::DURABLE_FORMAT_VERSION,
+        "backup_format": feltdb_server::backup::BACKUP_FORMAT,
+    });
+
+    if !check("data present", data.exists(), data.display().to_string()) {
+        state = "unavailable";
+    } else {
+        std::fs::create_dir_all(&scratch)?;
+        let source = feltdb_server::backup::BackupSource {
+            data: Some(data.display().to_string()),
+            state_sha256: None,
+            server_version: Some(env!("CARGO_PKG_VERSION").into()),
+            durable_format: Some(feltdb::DURABLE_FORMAT_VERSION),
+        };
+        let bundled = feltdb_server::backup::create_bundle_with_source(&bundle, &server_artifacts(&data, &keys, &audit), &[], source);
+        if !check("backup can be created", bundled.is_ok(), bundled.as_ref().map(|manifest| format!("{} files", manifest.entries.len())).unwrap_or_else(|error| error.to_string())) {
+            state = "unavailable";
+        } else {
+            let restorable = restore_bundle(&bundle, &restored);
+            check("backup can be restored", restorable.is_ok(), restorable.as_ref().map(|_| "restored to a private copy".to_string()).unwrap_or_else(|error| error.to_string()));
+            match FeltDb::open(restored.join("files/state.log")) {
+                Err(error) => {
+                    let corrupt = matches!(error, feltdb::FlowError::CorruptLogLine(_) | feltdb::FlowError::IncompatibleFormat(_));
+                    check("durable log valid", false, error.to_string());
+                    state = if corrupt { "corrupt" } else { "unavailable" };
+                }
+                Ok(database) => {
+                    let health = database.health();
+                    facts["durable_format"] = json!(health.durable_format.to_string());
+                    facts["storage"] = json!(health.storage.label());
+                    facts["recovery"] = json!(health.recovery.to_string());
+                    facts["sequence"] = json!(database.sequence().unwrap_or(0));
+                    check("durable log valid", true, format!("durable format {}", health.durable_format));
+                    if !check("replay clean", health.storage.is_clean(), health.recovery.to_string()) {
+                        state = "degraded";
+                    }
+                    let read = database.list_collection_page("_flow_capabilities", None, 1);
+                    if !check("read path", read.is_ok(), read.as_ref().map(|_| "bounded read succeeded".to_string()).unwrap_or_else(|error| error.to_string())) {
+                        state = "unavailable";
+                    }
+                    let probe_key = "_feltdb_doctor:probe";
+                    let wrote = database
+                        .insert(probe_key, json!({ "probe": true }))
+                        .and_then(|_| database.get_value(probe_key))
+                        .map(|value| value.is_some());
+                    if !check("mutation path", matches!(wrote, Ok(true)), match &wrote { Ok(true) => "write and read-back on the private copy".into(), Ok(false) => "write was not readable".into(), Err(error) => error.to_string() }) {
+                        state = "unavailable";
+                    }
+                }
+            }
+        }
+        if let Ok(cluster) = ClusterStore::load(data.with_extension("cluster.json"), Vec::new()) {
+            let pending = matches!(cluster.proposal().map(|value| value.phase), Some(ProposalPhase::Preparing | ProposalPhase::Prepared));
+            facts["pending_recovery"] = json!(pending);
+            check("no pending membership recovery", !pending, if pending { "a membership change is mid-flight".into() } else { "none".into() });
+            if pending && state == "healthy" {
+                state = "recovering";
+            }
+        }
+    }
+    if let Some(archive) = option("--backup") {
+        let started = Instant::now();
+        let report = feltdb_server::backup::verification_report(std::path::Path::new(&archive), started, "verify");
+        let restorable = if report.status == "ok" { verify_restorable(std::path::Path::new(&report.path)).map_err(|error| error.to_string()) } else { Err(report.error.clone().unwrap_or_default()) };
+        check("supplied backup restorable", restorable.is_ok(), restorable.unwrap_or_else(|error| error));
+        facts["backup"] = json!(report);
+    }
+    let _ = std::fs::remove_dir_all(&scratch);
+    let healthy_requires_all = checks.iter().all(|entry| entry["status"] == "PASS");
+    if state == "healthy" && !healthy_requires_all {
+        state = "degraded";
+    }
+    let report = json!({ "command": "doctor", "state": state, "facts": facts, "checks": checks });
+    if json_output {
+        println!("{}", serde_json::to_string_pretty(&report)?);
+    } else {
+        println!("FeltDB doctor: {state}");
+        for entry in report["checks"].as_array().into_iter().flatten() {
+            println!("  {} {}: {}", if entry["status"] == "PASS" { "✓" } else { "✗" }, entry["name"].as_str().unwrap_or(""), entry["detail"].as_str().unwrap_or(""));
+        }
+    }
+    if state != "healthy" {
+        std::process::exit(1);
+    }
+    Ok(true)
+}
+
+/// Prove a bundle restores into a database this build can open.
+///
+/// Checksums prove the bytes are the ones recorded; they cannot prove the
+/// recorded bytes are a FeltDB database in a format this build understands.
+/// This restores into a private temporary directory, opens it, and reads it —
+/// the same path `backup restore` and a subsequent server start take — and
+/// never touches the bundle or any live database.
+fn verify_restorable(archive: &std::path::Path) -> Result<String, Box<dyn std::error::Error>> {
+    let manifest = verify_bundle(archive)?;
+    let has_log = manifest.entries.iter().any(|entry| entry.path == "files/state.log");
+    let has_snapshot = manifest
+        .entries
+        .iter()
+        .any(|entry| entry.path == "files/database.snapshot.json");
+    if !has_log && !has_snapshot {
+        return Err("backup holds no authoritative state (neither state.log nor database.snapshot.json)".into());
+    }
+    let scratch = std::env::temp_dir().join(format!(
+        "feltdb-backup-verify-{}-{}",
+        std::process::id(),
+        rand::random::<u64>()
+    ));
+    let restored = scratch.join("restored");
+    let result = (|| -> Result<String, Box<dyn std::error::Error>> {
+        std::fs::create_dir_all(&scratch)?;
+        restore_bundle(archive, &restored)?;
+        let database = FeltDb::open(restored.join("files/state.log"))?;
+        if has_snapshot {
+            let snapshot: DatabaseSnapshot = serde_json::from_slice(&std::fs::read(
+                restored.join("files/database.snapshot.json"),
+            )?)?;
+            database.install_snapshot(snapshot)?;
+        }
+        let health = database.health();
+        Ok(format!(
+            "restorable: durable format {}, storage {}, sequence {}",
+            health.durable_format,
+            health.storage.label(),
+            database.sequence()?
+        ))
+    })();
+    let _ = std::fs::remove_dir_all(&scratch);
+    result
+}
+
 fn manage_application_runtime() -> Result<bool, Box<dyn std::error::Error>> {
     let arguments: Vec<String> = std::env::args().collect();
     if arguments.get(1).map(String::as_str) != Some("app") {
@@ -10515,12 +10792,24 @@ async fn create_online_backup(
         .cloned()
         .collect();
     let output = PathBuf::from(request.output);
+    let source = feltdb_server::backup::BackupSource {
+        data: state
+            .backup_artifacts
+            .iter()
+            .find(|(label, _)| label == "state.log")
+            .and_then(|(_, path)| feltdb_server::backup::normalize_path(path).ok())
+            .map(|path| path.display().to_string()),
+        state_sha256: None,
+        server_version: Some(env!("CARGO_PKG_VERSION").into()),
+        durable_format: Some(feltdb::DURABLE_FORMAT_VERSION),
+    };
     let manifest = tokio::task::spawn_blocking(move || {
         let _guard = guard;
-        create_bundle_with_virtual(
+        feltdb_server::backup::create_bundle_with_source(
             &output,
             &artifacts,
             &[("database.snapshot.json".to_string(), snapshot_bytes)],
+            source,
         )
     })
     .await

package/package.json

@@ -2,7 +2,7 @@
   "name": "create-feltdb",
   "private": false,
   "description": "Create a new FeltDB application with one command",
-  "version": "0.11.7",
+  "version": "0.11.8",
   "license": "MIT",
   "bin": {
     "create-feltdb": "bin/create-feltdb.js"