create-feltdb 0.11.7 → 0.11.9

package/dist/server-source/crates/feltdb-server/src/main.rs

@@ -75,7 +75,7 @@ use feltdb_server::{
     authenticated_principal::AuthenticatedPrincipal,
     authorization::{authorize, AuthorizationRequest, Subject},
     backup::{
-        create_bundle, create_bundle_with_virtual, restore_bundle, select_bundle_at,
+        restore_bundle, select_bundle_at,
         server_artifacts, verify_bundle,
     },
     causal::{
@@ -601,6 +601,14 @@ struct CanonicalTransactionRequest {
     environment: String,
     transaction: TransactionRequest,
 }
+#[derive(Deserialize)]
+struct CanonicalReadBasisRequest {
+    application_id: String,
+    revision_id: String,
+    #[serde(default = "production_environment")]
+    environment: String,
+    reads: feltdb::ReadSet,
+}
 fn production_environment() -> String {
     "production".into()
 }
@@ -1207,6 +1215,7 @@ async fn push_sync(
             ),
             operations: vec![operation.operation.clone()],
             preconditions: vec![],
+            fences: vec![],
         };
         let _transaction_permit = state
             .state_transaction_admission
@@ -4620,12 +4629,15 @@ fn runtime_inventory(
         .filter(|value| value.status == ConnectionStatus::Active)
         .map(|value| value.id)
         .collect();
-    let snapshot = state.db.export_snapshot()?;
+    // Every managed request resolves its contract through here, so this reads
+    // only the capability registry's own rows. A whole-database snapshot would
+    // clone, sort, and hash every tenant's state per request.
     let mut capabilities = BTreeSet::new();
-    for row in snapshot
-        .rows
+    for row in state
+        .db
+        .list_collection("_flow_capabilities")?
         .iter()
-        .filter(|row| row.capability == "_flow_capabilities" && !row.deleted)
+        .filter(|row| !row.deleted)
     {
         if let Some(name) = row.value.get("name").and_then(Value::as_str) {
             capabilities.insert(name.to_string());
@@ -4653,6 +4665,8 @@ fn revision_contract(
     revision_id: &str,
     environment: &str,
 ) -> Result<ApplicationRuntimeContract, ApiError> {
+    let _span =
+        feltdb::workload_diagnostics::span(feltdb::workload_diagnostics::Phase::ContractResolution);
     let tenant = application_scope(state, actor, application_id, "application:revision:read")?;
     let revision = state
         .applications
@@ -7209,6 +7223,7 @@ async fn run_runtime_action(
             if_version,
         }],
         preconditions: vec![],
+        fences: vec![],
     };
     let write_policy = contract
         .policies
@@ -7658,11 +7673,34 @@ async fn execute_canonical_query(
         &input.revision_id,
         &input.environment,
     )?;
+    let authorization = if feltdb::state_contract::is_substrate_collection(&input.query.collection)
+    {
+        let mut authorization = state_authorization_legacy(
+            &principal,
+            &tenant,
+            &input.application_id,
+            &contract.revision_id,
+            "state:denied",
+        );
+        if application_scope(
+            &state,
+            &principal.key_id,
+            &input.application_id,
+            "composition:read",
+        )
+        .is_ok()
+        {
+            authorization.capabilities.insert("composition:read".into());
+        }
+        authorization
+    } else {
+        state_authorization(&principal, &contract, &input.query.collection, "read")
+    };
     let context = feltdb::state_contract::begin_query_read(
         &state.db,
         &contract.state.schema,
         &contract.environment.state_namespace,
-        state_authorization(&principal, &contract, &input.query.collection, "read"),
+        authorization,
     )
     .map_err(state_contract_error)?;
 
@@ -7885,6 +7923,76 @@ async fn execute_revision_recovery(
     Ok(Json(response))
 }
 
+/// Whether `principal` may read every record of `collection` — the condition
+/// for a read basis, captured or committed, to cover it.
+///
+/// A token digests every row it covers, and a refused fence reveals that one
+/// of them changed. Both are reads, so both require the right to read the whole
+/// collection: `state:read` under a read policy that does not depend on the
+/// record (none, or `authenticated`), or `composition:read` for a substrate
+/// collection. Owner, member, and self policies filter rows per actor; a read
+/// basis over such a collection is refused rather than digested.
+fn read_basis_readable(
+    state: &AppState,
+    principal: &Principal,
+    contract: &ApplicationRuntimeContract,
+    collection: &str,
+) -> bool {
+    if feltdb::state_contract::is_substrate_collection(collection) {
+        return application_scope(
+            state,
+            &principal.key_id,
+            &contract.application_id,
+            "composition:read",
+        )
+        .is_ok();
+    }
+    if !state_authorization(principal, contract, collection, "read")
+        .capabilities
+        .contains("state:read")
+    {
+        return false;
+    }
+    matches!(
+        matching_state_policy(contract, collection)
+            .and_then(|policy| policy.read.as_deref())
+            .and_then(PolicySubject::from_str),
+        None | Some(PolicySubject::Authenticated)
+    )
+}
+
+/// Capture a read basis over the application's state namespace.
+async fn capture_canonical_read_basis(
+    State(state): State<AppState>,
+    Extension(principal): Extension<Principal>,
+    headers: HeaderMap,
+    Json(input): Json<CanonicalReadBasisRequest>,
+) -> Result<Json<Value>, ApiError> {
+    let tenant = application_scope(
+        &state,
+        &principal.key_id,
+        &input.application_id,
+        "state:read",
+    )?;
+    require_bound_tenant(&headers, &tenant)?;
+    let contract = revision_contract(
+        &state,
+        &principal.key_id,
+        &input.application_id,
+        &input.revision_id,
+        &input.environment,
+    )?;
+    let basis = feltdb::state_contract::capture_read_basis(
+        &state.db,
+        &contract.state.schema,
+        &contract.environment.state_namespace,
+        &input.reads,
+        &|collection| read_basis_readable(&state, &principal, &contract, collection),
+    )
+    .map_err(state_contract_error)?;
+    Ok(Json(json!(basis)))
+}
+
 async fn execute_canonical_transaction(
     State(state): State<AppState>,
     Extension(principal): Extension<Principal>,
@@ -7921,14 +8029,49 @@ async fn execute_canonical_transaction(
     input.transaction.revision_id = contract.revision_id.clone();
     input.transaction.schema_version = contract.state.schema.schema_version;
     input.transaction.state_namespace = Some(contract.environment.state_namespace.clone());
-    let collections = input
+    let (substrate, collections): (Vec<String>, Vec<String>) = input
         .transaction
         .operations
         .iter()
         .map(|operation| operation.collection.clone())
-        .collect::<Vec<_>>();
-    let (authorized, write_policies) =
-        authorize_transaction_collections(&principal, &contract, &tenant, &collections);
+        .partition(|collection| feltdb::state_contract::is_substrate_collection(collection));
+    let (authorized, write_policies) = if collections.is_empty() && !substrate.is_empty() {
+        (tenant == contract.tenant_id, BTreeMap::new())
+    } else {
+        authorize_transaction_collections(&principal, &contract, &tenant, &collections)
+    };
+    // Substrate records are written only under the dedicated capability.
+    let substrate_authorized = substrate.is_empty()
+        || application_scope(
+            &state,
+            &principal.key_id,
+            &input.application_id,
+            "composition:write",
+        )
+        .is_ok();
+    // A fence is a read of everything it covers: the committer must be able to
+    // read all of it, exactly as capturing it required.
+    for fence in &input.transaction.fences {
+        for collection in fence
+            .reads
+            .records
+            .iter()
+            .map(|record| record.collection.as_str())
+            .chain(
+                fence
+                    .reads
+                    .predicates
+                    .iter()
+                    .map(|predicate| predicate.collection.as_str()),
+            )
+        {
+            if !read_basis_readable(&state, &principal, &contract, collection) {
+                return Err(state_contract_error(
+                    feltdb::state_contract::read_basis_denied(collection),
+                ));
+            }
+        }
+    }
     input.transaction.authorization = state_authorization_legacy(
         &principal,
         &tenant,
@@ -7940,6 +8083,13 @@ async fn execute_canonical_transaction(
             "state:denied"
         },
     );
+    if authorized && substrate_authorized && !substrate.is_empty() {
+        input
+            .transaction
+            .authorization
+            .capabilities
+            .insert("composition:write".into());
+    }
     let _transaction_permit = state
         .state_transaction_admission
         .acquire()
@@ -8529,6 +8679,9 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
     if manage_backup()? {
         return Ok(());
     }
+    if manage_doctor()? {
+        return Ok(());
+    }
     if manage_application_runtime()? {
         return Ok(());
     }
@@ -9033,6 +9186,10 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
             "/v1/transactions",
             axum::routing::post(execute_canonical_transaction),
         )
+        .route(
+            "/v1/transactions/read-basis",
+            axum::routing::post(capture_canonical_read_basis),
+        )
         .route(
             "/v1/revision/recover",
             axum::routing::post(execute_revision_recovery),
@@ -9309,6 +9466,10 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
             axum::routing::delete(delete_connection_control),
         )
         .route("/transactions", axum::routing::post(commit_transaction))
+        .route(
+            "/transactions/read-basis",
+            axum::routing::post(capture_read_basis),
+        )
         .route("/query", axum::routing::post(execute_bounded_query))
         .route("/revision", get(get_revision))
         .route(
@@ -9628,6 +9789,8 @@ async fn authenticate(
         || path == "/v1/state/conflicts"
         || path == "/v1/state/reconciliation/plan"
         || path == "/v1/experimental/semantic-decisions"
+        || path == "/transactions/read-basis"
+        || path == "/v1/transactions/read-basis"
     {
         // Describing state is reading it. These are POSTs because they take
         // three revision identities and a plan, not because they change
@@ -9989,8 +10152,16 @@ async fn health_ready(State(state): State<AppState>) -> axum::response::Response
     match outcome {
         Ok(Ok(Ok(health))) => {
             let ready = health.is_nominal();
+            let recovering = matches!(
+                state.cluster.proposal().map(|value| value.phase),
+                Some(ProposalPhase::Preparing | ProposalPhase::Prepared)
+            );
             let body = Json(json!({
                 "ready": ready,
+                // The explicit health model. A running server has already
+                // refused a corrupt or incompatible log at open, so `corrupt`
+                // is reported offline by `feltdb-server doctor`, never here.
+                "state": if !ready { "degraded" } else if recovering { "recovering" } else { "healthy" },
                 "storage": health.storage.label(),
                 "durable_format": health.durable_format.to_string(),
                 "probe_ms": waited_ms,
@@ -10006,6 +10177,7 @@ async fn health_ready(State(state): State<AppState>) -> axum::response::Response
             StatusCode::SERVICE_UNAVAILABLE,
             Json(json!({
                 "ready": false,
+                "state": "unavailable",
                 "error": error.to_string(),
                 "code": "STORAGE_UNAVAILABLE",
                 "probe_ms": waited_ms,
@@ -10016,6 +10188,7 @@ async fn health_ready(State(state): State<AppState>) -> axum::response::Response
             StatusCode::SERVICE_UNAVAILABLE,
             Json(json!({
                 "ready": false,
+                "state": "unavailable",
                 "error": error.to_string(),
                 "code": "STORAGE_PROBE_FAILED",
                 "probe_ms": waited_ms,
@@ -10026,6 +10199,7 @@ async fn health_ready(State(state): State<AppState>) -> axum::response::Response
             StatusCode::SERVICE_UNAVAILABLE,
             Json(json!({
                 "ready": false,
+                "state": "unavailable",
                 "error": "storage probe exceeded the server deadline",
                 "code": "STORAGE_STALLED",
                 "probe_ms": waited_ms,
@@ -10370,7 +10544,6 @@ fn manage_backup() -> Result<bool, Box<dyn std::error::Error>> {
     if arguments.get(1).map(String::as_str) != Some("backup") {
         return Ok(false);
     }
-    let command = arguments.get(2).map(String::as_str).unwrap_or("");
     let option = |name: &str| {
         arguments
             .iter()
@@ -10378,27 +10551,105 @@ fn manage_backup() -> Result<bool, Box<dyn std::error::Error>> {
             .and_then(|index| arguments.get(index + 1))
             .cloned()
     };
+    let json_output = arguments.iter().any(|value| value == "--json");
+    // `backup <path>` is `backup create --output <path>`; `backup verify <path>`
+    // is `backup verify --archive <path>`. The flag forms remain supported.
+    let positional = |from: usize| {
+        let mut index = from;
+        while let Some(value) = arguments.get(index) {
+            if value == "--json" {
+                index += 1;
+            } else if value.starts_with("--") {
+                index += 2;
+            } else {
+                return Some(value.clone());
+            }
+        }
+        None
+    };
+    let (command, target) = match arguments.get(2).map(String::as_str) {
+        Some(command @ ("create" | "verify" | "restore")) => (command, positional(3)),
+        Some(value) if !value.starts_with("--") => ("create", Some(value.to_string())),
+        _ if option("--output").is_some() && option("--archive").is_none() => ("create", None),
+        _ => ("", None),
+    };
+    let usage = "usage: feltdb-server backup [create] <output> | backup verify <archive> | backup restore --archive <archive> --output <directory> [--at-ms timestamp]  (options: --data path --keys path --audit path --json)";
+    let started = Instant::now();
+    let emit = |report: &feltdb_server::backup::BackupReport, human: String| {
+        if json_output {
+            println!("{}", serde_json::to_string_pretty(report).unwrap_or_default());
+        } else if report.status == "ok" {
+            println!("{human}");
+        } else {
+            eprintln!(
+                "Backup {} failed: {} ({})",
+                report.operation,
+                report.path,
+                report.error.as_deref().unwrap_or("unknown error")
+            );
+        }
+    };
     match command {
         "create" => {
             let data = PathBuf::from(option("--data").unwrap_or_else(|| "./data/feltdb.log".into()));
             let keys = PathBuf::from(option("--keys").unwrap_or_else(|| "./data/api-keys.json".into()));
             let audit = PathBuf::from(option("--audit").unwrap_or_else(|| data.with_extension("audit.log").to_string_lossy().into_owned()));
-            let output = PathBuf::from(option("--output").ok_or("backup create requires --output")?);
-            let manifest = create_bundle(&output, &server_artifacts(&data, &keys, &audit))?;
-            println!("Backup created: {} (format {}, {} files)", output.display(), manifest.format, manifest.entries.len());
+            let output = option("--output").or(target).ok_or("backup create requires an output path")?;
+            let output = feltdb_server::backup::normalize_path(std::path::Path::new(&output))?;
+            let data = feltdb_server::backup::normalize_path(&data)?;
+            if !data.exists() {
+                return Err(format!("backup source does not exist: {}", data.display()).into());
+            }
+            let source = feltdb_server::backup::BackupSource {
+                data: Some(data.display().to_string()),
+                state_sha256: None,
+                server_version: Some(env!("CARGO_PKG_VERSION").into()),
+                durable_format: Some(feltdb::DURABLE_FORMAT_VERSION),
+            };
+            feltdb_server::backup::create_bundle_with_source(&output, &server_artifacts(&data, &keys, &audit), &[], source)?;
+            // A backup is reported as successful only after it verifies
+            // independently of the code path that wrote it.
+            let mut report = feltdb_server::backup::verification_report(&output, started, "create");
+            if report.status == "ok" {
+                if let Err(error) = verify_restorable(&output) {
+                    report.status = "failed".into();
+                    report.verification = "failed".into();
+                    report.error = Some(error.to_string());
+                }
+            }
+            let human = format!("Backup created: {} (id {}, format {}, {} files, {} bytes, verified, {} ms)", report.path, &report.backup_id[..report.backup_id.len().min(12)], report.format, report.files, report.bytes_written, report.duration_ms);
+            emit(&report, human);
+            if report.status != "ok" {
+                std::process::exit(1);
+            }
         }
         "verify" => {
-            let archive = PathBuf::from(option("--archive").ok_or("backup verify requires --archive")?);
-            let manifest = verify_bundle(&archive)?;
-            println!("Backup verified: {} (format {}, {} files)", archive.display(), manifest.format, manifest.entries.len());
+            let archive = option("--archive").or(target).ok_or("backup verify requires an archive path")?;
+            let mut report = feltdb_server::backup::verification_report(std::path::Path::new(&archive), started, "verify");
+            let mut restorable = None;
+            if report.status == "ok" {
+                match verify_restorable(std::path::Path::new(&report.path)) {
+                    Ok(detail) => restorable = Some(detail),
+                    Err(error) => {
+                        report.status = "failed".into();
+                        report.verification = "failed".into();
+                        report.error = Some(error.to_string());
+                    }
+                }
+            }
+            let human = format!("Backup verified: {} (id {}, format {}, {} files, {} bytes{})", report.path, &report.backup_id[..report.backup_id.len().min(12)], report.format, report.files, report.bytes_written, restorable.map(|detail| format!(", {detail}")).unwrap_or_default());
+            emit(&report, human);
+            if report.status != "ok" {
+                std::process::exit(1);
+            }
         }
         "restore" => {
-            let archive = PathBuf::from(option("--archive").ok_or("backup restore requires --archive")?);
+            let archive = PathBuf::from(option("--archive").or(target).ok_or("backup restore requires --archive")?);
             let archive = match option("--at-ms") {
                 Some(value) => select_bundle_at(&archive, value.parse().map_err(|_| "backup restore --at-ms must be an unsigned millisecond timestamp")?)?,
                 None => archive,
             };
-            let output = PathBuf::from(option("--output").ok_or("backup restore requires --output")?);
+            let output = feltdb_server::backup::normalize_path(std::path::Path::new(&option("--output").ok_or("backup restore requires --output")?))?;
             let manifest = restore_bundle(&archive, &output)?;
             let snapshot_path = output.join("files/database.snapshot.json");
             if snapshot_path.exists() {
@@ -10406,13 +10657,199 @@ fn manage_backup() -> Result<bool, Box<dyn std::error::Error>> {
                 let database = FeltDb::open(output.join("files/state.log"))?;
                 database.install_snapshot(snapshot)?;
             }
-            println!("Backup restored: {} (format {}, {} files)", output.display(), manifest.format, manifest.entries.len());
+            let mut report = feltdb_server::backup::verification_report(&archive, started, "restore");
+            report.path = output.display().to_string();
+            emit(&report, format!("Backup restored: {} (format {}, {} files)", output.display(), manifest.format, manifest.entries.len()));
+            if report.status != "ok" {
+                std::process::exit(1);
+            }
         }
-        _ => return Err("usage: feltdb-server backup <create|verify|restore> [--data path] [--keys path] [--audit path] --output directory | --archive directory [--at-ms timestamp]".into()),
+        _ => return Err(usage.into()),
     }
     Ok(true)
 }
 
+/// `feltdb-server doctor` — what an operator needs to know, offline.
+///
+/// Answers, for a data path: is the database healthy; is the durable log
+/// valid; what state/version is it in; is recovery pending; can a backup be
+/// created; can that backup be restored; which server and format versions are
+/// active. It never opens the live files: the state is bundled exactly as
+/// `backup create` would, and every probe — open, read, write — runs against a
+/// private restored copy. `--backup <archive>` additionally verifies an
+/// existing bundle.
+///
+/// The health state is one of `healthy`, `recovering`, `degraded`,
+/// `unavailable`, or `corrupt`, and `healthy` requires every probe to pass:
+/// a readable durable log in a supported format, a clean replay, a working
+/// read, a working write, and a restorable backup.
+fn manage_doctor() -> Result<bool, Box<dyn std::error::Error>> {
+    let arguments: Vec<String> = std::env::args().collect();
+    if arguments.get(1).map(String::as_str) != Some("doctor") {
+        return Ok(false);
+    }
+    let option = |name: &str| {
+        arguments
+            .iter()
+            .position(|value| value == name)
+            .and_then(|index| arguments.get(index + 1))
+            .cloned()
+    };
+    let json_output = arguments.iter().any(|value| value == "--json");
+    let data = feltdb_server::backup::normalize_path(&PathBuf::from(
+        option("--data").unwrap_or_else(|| "./data/feltdb.log".into()),
+    ))?;
+    let keys = PathBuf::from(option("--keys").unwrap_or_else(|| "./data/api-keys.json".into()));
+    let audit = PathBuf::from(option("--audit").unwrap_or_else(|| data.with_extension("audit.log").to_string_lossy().into_owned()));
+
+    let mut checks: Vec<Value> = Vec::new();
+    let mut check = |name: &str, ok: bool, detail: String| {
+        checks.push(json!({ "name": name, "status": if ok { "PASS" } else { "FAIL" }, "detail": detail }));
+        ok
+    };
+    let scratch = std::env::temp_dir().join(format!(
+        "feltdb-doctor-{}-{}",
+        std::process::id(),
+        rand::random::<u64>()
+    ));
+    let bundle = scratch.join("bundle");
+    let restored = scratch.join("restored");
+    let mut state = "healthy";
+    let mut facts = json!({
+        "data": data.display().to_string(),
+        "server_version": env!("CARGO_PKG_VERSION"),
+        "supported_durable_format": feltdb::DURABLE_FORMAT_VERSION,
+        "backup_format": feltdb_server::backup::BACKUP_FORMAT,
+    });
+
+    if !check("data present", data.exists(), data.display().to_string()) {
+        state = "unavailable";
+    } else {
+        std::fs::create_dir_all(&scratch)?;
+        let source = feltdb_server::backup::BackupSource {
+            data: Some(data.display().to_string()),
+            state_sha256: None,
+            server_version: Some(env!("CARGO_PKG_VERSION").into()),
+            durable_format: Some(feltdb::DURABLE_FORMAT_VERSION),
+        };
+        let bundled = feltdb_server::backup::create_bundle_with_source(&bundle, &server_artifacts(&data, &keys, &audit), &[], source);
+        if !check("backup can be created", bundled.is_ok(), bundled.as_ref().map(|manifest| format!("{} files", manifest.entries.len())).unwrap_or_else(|error| error.to_string())) {
+            state = "unavailable";
+        } else {
+            let restorable = restore_bundle(&bundle, &restored);
+            check("backup can be restored", restorable.is_ok(), restorable.as_ref().map(|_| "restored to a private copy".to_string()).unwrap_or_else(|error| error.to_string()));
+            match FeltDb::open(restored.join("files/state.log")) {
+                Err(error) => {
+                    let corrupt = matches!(error, feltdb::FlowError::CorruptLogLine(_) | feltdb::FlowError::IncompatibleFormat(_));
+                    check("durable log valid", false, error.to_string());
+                    state = if corrupt { "corrupt" } else { "unavailable" };
+                }
+                Ok(database) => {
+                    let health = database.health();
+                    facts["durable_format"] = json!(health.durable_format.to_string());
+                    facts["storage"] = json!(health.storage.label());
+                    facts["recovery"] = json!(health.recovery.to_string());
+                    facts["sequence"] = json!(database.sequence().unwrap_or(0));
+                    check("durable log valid", true, format!("durable format {}", health.durable_format));
+                    if !check("replay clean", health.storage.is_clean(), health.recovery.to_string()) {
+                        state = "degraded";
+                    }
+                    let read = database.list_collection_page("_flow_capabilities", None, 1);
+                    if !check("read path", read.is_ok(), read.as_ref().map(|_| "bounded read succeeded".to_string()).unwrap_or_else(|error| error.to_string())) {
+                        state = "unavailable";
+                    }
+                    let probe_key = "_feltdb_doctor:probe";
+                    let wrote = database
+                        .insert(probe_key, json!({ "probe": true }))
+                        .and_then(|_| database.get_value(probe_key))
+                        .map(|value| value.is_some());
+                    if !check("mutation path", matches!(wrote, Ok(true)), match &wrote { Ok(true) => "write and read-back on the private copy".into(), Ok(false) => "write was not readable".into(), Err(error) => error.to_string() }) {
+                        state = "unavailable";
+                    }
+                }
+            }
+        }
+        if let Ok(cluster) = ClusterStore::load(data.with_extension("cluster.json"), Vec::new()) {
+            let pending = matches!(cluster.proposal().map(|value| value.phase), Some(ProposalPhase::Preparing | ProposalPhase::Prepared));
+            facts["pending_recovery"] = json!(pending);
+            check("no pending membership recovery", !pending, if pending { "a membership change is mid-flight".into() } else { "none".into() });
+            if pending && state == "healthy" {
+                state = "recovering";
+            }
+        }
+    }
+    if let Some(archive) = option("--backup") {
+        let started = Instant::now();
+        let report = feltdb_server::backup::verification_report(std::path::Path::new(&archive), started, "verify");
+        let restorable = if report.status == "ok" { verify_restorable(std::path::Path::new(&report.path)).map_err(|error| error.to_string()) } else { Err(report.error.clone().unwrap_or_default()) };
+        check("supplied backup restorable", restorable.is_ok(), restorable.unwrap_or_else(|error| error));
+        facts["backup"] = json!(report);
+    }
+    let _ = std::fs::remove_dir_all(&scratch);
+    let healthy_requires_all = checks.iter().all(|entry| entry["status"] == "PASS");
+    if state == "healthy" && !healthy_requires_all {
+        state = "degraded";
+    }
+    let report = json!({ "command": "doctor", "state": state, "facts": facts, "checks": checks });
+    if json_output {
+        println!("{}", serde_json::to_string_pretty(&report)?);
+    } else {
+        println!("FeltDB doctor: {state}");
+        for entry in report["checks"].as_array().into_iter().flatten() {
+            println!("  {} {}: {}", if entry["status"] == "PASS" { "✓" } else { "✗" }, entry["name"].as_str().unwrap_or(""), entry["detail"].as_str().unwrap_or(""));
+        }
+    }
+    if state != "healthy" {
+        std::process::exit(1);
+    }
+    Ok(true)
+}
+
+/// Prove a bundle restores into a database this build can open.
+///
+/// Checksums prove the bytes are the ones recorded; they cannot prove the
+/// recorded bytes are a FeltDB database in a format this build understands.
+/// This restores into a private temporary directory, opens it, and reads it —
+/// the same path `backup restore` and a subsequent server start take — and
+/// never touches the bundle or any live database.
+fn verify_restorable(archive: &std::path::Path) -> Result<String, Box<dyn std::error::Error>> {
+    let manifest = verify_bundle(archive)?;
+    let has_log = manifest.entries.iter().any(|entry| entry.path == "files/state.log");
+    let has_snapshot = manifest
+        .entries
+        .iter()
+        .any(|entry| entry.path == "files/database.snapshot.json");
+    if !has_log && !has_snapshot {
+        return Err("backup holds no authoritative state (neither state.log nor database.snapshot.json)".into());
+    }
+    let scratch = std::env::temp_dir().join(format!(
+        "feltdb-backup-verify-{}-{}",
+        std::process::id(),
+        rand::random::<u64>()
+    ));
+    let restored = scratch.join("restored");
+    let result = (|| -> Result<String, Box<dyn std::error::Error>> {
+        std::fs::create_dir_all(&scratch)?;
+        restore_bundle(archive, &restored)?;
+        let database = FeltDb::open(restored.join("files/state.log"))?;
+        if has_snapshot {
+            let snapshot: DatabaseSnapshot = serde_json::from_slice(&std::fs::read(
+                restored.join("files/database.snapshot.json"),
+            )?)?;
+            database.install_snapshot(snapshot)?;
+        }
+        let health = database.health();
+        Ok(format!(
+            "restorable: durable format {}, storage {}, sequence {}",
+            health.durable_format,
+            health.storage.label(),
+            database.sequence()?
+        ))
+    })();
+    let _ = std::fs::remove_dir_all(&scratch);
+    result
+}
+
 fn manage_application_runtime() -> Result<bool, Box<dyn std::error::Error>> {
     let arguments: Vec<String> = std::env::args().collect();
     if arguments.get(1).map(String::as_str) != Some("app") {
@@ -10515,12 +10952,24 @@ async fn create_online_backup(
         .cloned()
         .collect();
     let output = PathBuf::from(request.output);
+    let source = feltdb_server::backup::BackupSource {
+        data: state
+            .backup_artifacts
+            .iter()
+            .find(|(label, _)| label == "state.log")
+            .and_then(|(_, path)| feltdb_server::backup::normalize_path(path).ok())
+            .map(|path| path.display().to_string()),
+        state_sha256: None,
+        server_version: Some(env!("CARGO_PKG_VERSION").into()),
+        durable_format: Some(feltdb::DURABLE_FORMAT_VERSION),
+    };
     let manifest = tokio::task::spawn_blocking(move || {
         let _guard = guard;
-        create_bundle_with_virtual(
+        feltdb_server::backup::create_bundle_with_source(
             &output,
             &artifacts,
             &[("database.snapshot.json".to_string(), snapshot_bytes)],
+            source,
         )
     })
     .await
@@ -13584,6 +14033,28 @@ struct AtomicTxRequest {
     /// exactly as before.
     #[serde(default)]
     preconditions: Vec<AtomicTxPreconditionRequest>,
+    /// Read bases this authority issued (`POST /transactions/read-basis`),
+    /// re-evaluated inside the commit lock. Absent means no fence.
+    #[serde(default, skip_serializing_if = "Vec::is_empty")]
+    fences: Vec<feltdb::ReadBasis>,
+}
+
+/// Capture a read basis: what the authority holds, now, for a set of records
+/// and equality predicates. Reading, so it requires `state:read`.
+async fn capture_read_basis(
+    State(state): State<AppState>,
+    Json(reads): Json<feltdb::ReadSet>,
+) -> Result<Json<feltdb::ReadBasis>, ApiError> {
+    state
+        .db
+        .read_basis(&reads)
+        .map(Json)
+        .map_err(|error| match error {
+            FlowError::CapabilityError(message) if message.starts_with("INVALID_READ_SET:") => {
+                ApiError(StatusCode::BAD_REQUEST, message)
+            }
+            other => ApiError(StatusCode::INTERNAL_SERVER_ERROR, other.to_string()),
+        })
 }
 
 #[derive(Serialize)]
@@ -13645,6 +14116,8 @@ async fn commit_transaction(
     let mut record_preconditions: Vec<RecordPrecondition> = Vec::new();
     let mut seen = std::collections::HashSet::new();
 
+    let fences = request.fences.clone();
+
     // Transaction-level preconditions first, on records the transaction need
     // not write at all.
     for condition in &request.preconditions {
@@ -13722,12 +14195,13 @@ async fn commit_transaction(
     let operations = mutations.len();
     let commit = state
         .db
-        .apply_atomic_transaction_guarded(
+        .apply_atomic_transaction_fenced(
             &request.transaction_id,
             Some(&payload_hash),
             None,
             &[],
             &record_preconditions,
+            &fences,
             &mutations,
             None,
         )
@@ -13747,6 +14221,9 @@ async fn commit_transaction(
                         "failure": *failure,
                     }),
                 ),
+                FlowError::CapabilityError(message) if message.starts_with("INVALID_READ_SET:") => {
+                    ApiError(StatusCode::BAD_REQUEST, message)
+                }
                 FlowError::CapabilityError(message)
                     if message.starts_with("TRANSACTION_ID_PAYLOAD_MISMATCH:") =>
                 {