@feltdb/core 0.12.0 → 0.12.2

package/dist/create/server-source/crates/feltdb/tests/authority_transaction_identity.rs

@@ -0,0 +1,534 @@
+//! Transaction identity at the process authority's commit boundary.
+//!
+//! A transaction id names one payload. The authority, not its clients, binds an
+//! id to the payload it first committed, so that an id reused for different
+//! contents is refused instead of being answered "duplicate" and silently
+//! discarded. These tests hold `AuthorityClient::commit` and
+//! `EmbeddedAuthority::commit` to that contract over the real socket, across
+//! restart, and against logs written before the binding existed.
+#![cfg(unix)]
+
+use feltdb::{
+    authority::{
+        AuthorityClient, AuthorityError, AuthorityServer, AuthorityTransaction, EmbeddedAuthority,
+        FeltDbAuthority,
+    },
+    AtomicCommit, AtomicMutation, AtomicPrecondition, FeltDb, RecordPrecondition,
+};
+use serde_json::{json, Value};
+use std::{
+    path::{Path, PathBuf},
+    process::{Child, Command},
+    sync::Arc,
+    time::Duration,
+};
+use tempfile::TempDir;
+use tokio::{
+    io::{AsyncBufReadExt, AsyncWriteExt, BufReader},
+    net::UnixListener,
+};
+
+static PROCESS_TESTS: std::sync::Mutex<()> = std::sync::Mutex::new(());
+
+struct RunningAuthority {
+    child: Child,
+    socket: PathBuf,
+    storage: PathBuf,
+}
+
+impl Drop for RunningAuthority {
+    fn drop(&mut self) {
+        let _ = self.child.kill();
+        let _ = self.child.wait();
+    }
+}
+
+fn start(directory: &TempDir) -> RunningAuthority {
+    let socket = directory.path().join("authority.sock");
+    let storage = directory.path().join("authority.db");
+    let child = Command::new(env!("CARGO_BIN_EXE_feltdb-authority"))
+        .arg(&socket)
+        .arg(&storage)
+        .spawn()
+        .expect("authority starts");
+    for _ in 0..400 {
+        if socket.exists() {
+            return RunningAuthority {
+                child,
+                socket,
+                storage,
+            };
+        }
+        std::thread::sleep(Duration::from_millis(5));
+    }
+    panic!("authority socket was not ready");
+}
+
+fn stop(mut authority: RunningAuthority) -> PathBuf {
+    authority.child.kill().unwrap();
+    authority.child.wait().unwrap();
+    let _ = std::fs::remove_file(&authority.socket);
+    authority.storage.clone()
+}
+
+/// A transaction writing `value` to one record. The id is the caller's.
+fn put(id: &str, key: &str, value: Value) -> AuthorityTransaction {
+    AuthorityTransaction {
+        transaction_id: id.into(),
+        base_revision: None,
+        preconditions: vec![],
+        record_preconditions: vec![],
+        mutations: vec![AtomicMutation {
+            capability: "items".into(),
+            key: key.into(),
+            value: Some(value),
+        }],
+        audit: None,
+    }
+}
+
+/// `payload_binding` as the wire carries it, read without depending on the Rust field.
+fn binding(commit: &AtomicCommit) -> Value {
+    serde_json::to_value(commit).unwrap()["payload_binding"].clone()
+}
+
+async fn rows(authority: &impl FeltDbAuthority) -> Vec<Value> {
+    authority
+        .read_authoritative()
+        .await
+        .unwrap()
+        .rows
+        .into_iter()
+        .map(|row| json!({"key": row.key, "value": row.value}))
+        .collect()
+}
+
+fn code(error: &AuthorityError) -> &str {
+    &error.code
+}
+
+/* ----------------------------- 1. same id, same payload ----------------------------- */
+
+#[tokio::test]
+async fn same_id_same_payload_is_a_verified_duplicate_of_the_original() {
+    let directory = TempDir::new().unwrap();
+    let authority = EmbeddedAuthority::open(directory.path().join("a.db")).unwrap();
+    let first = authority
+        .commit(put("tx-1", "items:a", json!({"n": 1})))
+        .await
+        .unwrap();
+    assert!(!first.duplicate);
+    // Later commits move the authority on; the duplicate still describes the original.
+    for i in 0..3 {
+        authority
+            .commit(put(&format!("later-{i}"), &format!("items:l{i}"), json!({"i": i})))
+            .await
+            .unwrap();
+    }
+    let again = authority
+        .commit(put("tx-1", "items:a", json!({"n": 1})))
+        .await
+        .unwrap();
+    assert!(again.duplicate);
+    assert_eq!(binding(&again), json!("verified"));
+    assert_eq!(again.commit_revision, first.commit_revision);
+    assert_eq!(again.state_before, first.state_before);
+    assert_eq!(again.state_after, first.state_after);
+    assert_eq!(authority.current_revision().await.unwrap(), 4);
+}
+
+/* --------------------------- 2. same id, different payload -------------------------- */
+
+#[tokio::test]
+async fn same_id_different_payload_is_an_idempotency_conflict_and_applies_nothing() {
+    let directory = TempDir::new().unwrap();
+    let authority = EmbeddedAuthority::open(directory.path().join("a.db")).unwrap();
+    authority
+        .commit(put("tx-1", "items:a", json!({"n": 1})))
+        .await
+        .unwrap();
+    let before_rows = rows(&authority).await;
+    let before_revision = authority.current_revision().await.unwrap();
+
+    let mut variants = vec![
+        put("tx-1", "items:a", json!({"n": 2})),
+        put("tx-1", "items:other", json!({"n": 1})),
+    ];
+    let mut with_base = put("tx-1", "items:a", json!({"n": 1}));
+    with_base.base_revision = Some(1);
+    variants.push(with_base);
+    let mut with_audit = put("tx-1", "items:a", json!({"n": 1}));
+    with_audit.audit = Some(json!({"by": "someone else"}));
+    variants.push(with_audit);
+    let mut with_precondition = put("tx-1", "items:a", json!({"n": 1}));
+    with_precondition.preconditions = vec![AtomicPrecondition {
+        capability: "items".into(),
+        key: "items:a".into(),
+        expected_version: None,
+    }];
+    variants.push(with_precondition);
+    let mut with_record_precondition = put("tx-1", "items:a", json!({"n": 1}));
+    with_record_precondition.record_preconditions = vec![RecordPrecondition {
+        capability: "items".into(),
+        key: "items:a".into(),
+        require_absent: false,
+        expected_version: Some(1),
+        expected_epoch: None,
+        expected_lease_id: None,
+    }];
+    variants.push(with_record_precondition);
+    let mut two_mutations = put("tx-1", "items:a", json!({"n": 1}));
+    two_mutations.mutations.push(AtomicMutation {
+        capability: "items".into(),
+        key: "items:extra".into(),
+        value: Some(json!({})),
+    });
+    variants.push(two_mutations);
+
+    for (index, variant) in variants.into_iter().enumerate() {
+        let error = authority.commit(variant).await.unwrap_err();
+        assert_eq!(code(&error), "IDEMPOTENCY_CONFLICT", "variant {index}: {error}");
+        assert!(error.message.contains("tx-1"), "the id is named: {error}");
+    }
+    assert_eq!(rows(&authority).await, before_rows, "a conflict writes nothing");
+    assert_eq!(authority.current_revision().await.unwrap(), before_revision);
+    let original = authority
+        .commit(put("tx-1", "items:a", json!({"n": 1})))
+        .await
+        .unwrap();
+    assert_eq!(binding(&original), json!("verified"), "the recorded identity is unchanged");
+}
+
+#[tokio::test]
+async fn property_order_in_the_payload_does_not_change_its_identity() {
+    let directory = TempDir::new().unwrap();
+    let authority = EmbeddedAuthority::open(directory.path().join("a.db")).unwrap();
+    let a: Value = serde_json::from_str(r#"{"x":1,"y":{"p":1,"q":[1,{"b":2,"a":1}]}}"#).unwrap();
+    let b: Value = serde_json::from_str(r#"{"y":{"q":[1,{"a":1,"b":2}],"p":1},"x":1}"#).unwrap();
+    authority.commit(put("tx-1", "items:a", a)).await.unwrap();
+    let again = authority.commit(put("tx-1", "items:a", b)).await.unwrap();
+    assert_eq!(binding(&again), json!("verified"));
+    // Array order does mean something.
+    let c: Value = serde_json::from_str(r#"{"x":1,"y":{"p":1,"q":[{"a":1,"b":2},1]}}"#).unwrap();
+    let error = authority.commit(put("tx-1", "items:a", c)).await.unwrap_err();
+    assert_eq!(code(&error), "IDEMPOTENCY_CONFLICT");
+}
+
+/* ---------------------------------- 3-4. concurrency --------------------------------- */
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
+async fn concurrent_identical_requests_apply_once_and_the_rest_observe_it() {
+    let directory = TempDir::new().unwrap();
+    let authority = Arc::new(EmbeddedAuthority::open(directory.path().join("a.db")).unwrap());
+    let tasks: Vec<_> = (0..24)
+        .map(|_| {
+            let authority = authority.clone();
+            tokio::spawn(async move { authority.commit(put("race", "items:k", json!({"who": 0}))).await })
+        })
+        .collect();
+    let mut applied = 0;
+    for task in tasks {
+        let commit = task.await.unwrap().unwrap();
+        if commit.duplicate {
+            assert_eq!(binding(&commit), json!("verified"));
+        } else {
+            applied += 1;
+        }
+    }
+    assert_eq!(applied, 1, "exactly one application");
+    assert_eq!(authority.current_revision().await.unwrap(), 1);
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
+async fn concurrent_conflicting_requests_apply_at_most_one_payload() {
+    let directory = TempDir::new().unwrap();
+    let authority = Arc::new(EmbeddedAuthority::open(directory.path().join("a.db")).unwrap());
+    let tasks: Vec<_> = (0..30)
+        .map(|index| {
+            let authority = authority.clone();
+            tokio::spawn(async move {
+                let who = index % 3;
+                (who, authority.commit(put("race", "items:k", json!({"who": who}))).await)
+            })
+        })
+        .collect();
+    let mut outcomes = vec![];
+    for task in tasks {
+        outcomes.push(task.await.unwrap());
+    }
+    let applied: Vec<_> = outcomes
+        .iter()
+        .filter(|(_, result)| matches!(result, Ok(commit) if !commit.duplicate))
+        .collect();
+    assert_eq!(applied.len(), 1, "exactly one payload is applied");
+    let winner = applied[0].0;
+    for (who, result) in &outcomes {
+        match result {
+            Ok(commit) => {
+                assert_eq!(*who, winner, "only the winning payload is ever acknowledged");
+                assert!(!commit.duplicate || binding(commit) == json!("verified"));
+            }
+            Err(error) => {
+                assert_ne!(*who, winner);
+                assert_eq!(code(error), "IDEMPOTENCY_CONFLICT");
+            }
+        }
+    }
+    let stored = rows(authority.as_ref()).await;
+    assert_eq!(stored.len(), 1);
+    assert_eq!(stored[0]["value"]["who"], json!(winner));
+    assert_eq!(authority.current_revision().await.unwrap(), 1);
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
+async fn concurrent_clients_over_the_socket_get_the_same_answers() {
+    let _process_test = PROCESS_TESTS.lock().unwrap_or_else(|e| e.into_inner());
+    let directory = TempDir::new().unwrap();
+    let authority = start(&directory);
+    let socket = authority.socket.clone();
+    let tasks: Vec<_> = (0..20)
+        .map(|index| {
+            let socket = socket.clone();
+            tokio::spawn(async move {
+                let who = index % 2;
+                (who, AuthorityClient::connect(socket).commit(put("race", "items:k", json!({"who": who}))).await)
+            })
+        })
+        .collect();
+    let mut applied = vec![];
+    let mut conflicts = 0;
+    let mut duplicates = 0;
+    let mut outcomes = vec![];
+    for task in tasks {
+        outcomes.push(task.await.unwrap());
+    }
+    for (who, result) in &outcomes {
+        match result {
+            Ok(commit) if commit.duplicate => duplicates += 1,
+            Ok(_) => applied.push(*who),
+            Err(error) => {
+                assert_eq!(code(error), "IDEMPOTENCY_CONFLICT", "{error}");
+                conflicts += 1;
+            }
+        }
+    }
+    assert_eq!(applied.len(), 1);
+    assert_eq!(applied.len() + duplicates + conflicts, 20);
+    let winner = applied[0];
+    assert!(outcomes.iter().filter(|(who, _)| *who != winner).all(|(_, r)| r.is_err()));
+    let client = AuthorityClient::connect(&socket);
+    let stored = rows(&client).await;
+    assert_eq!(stored.len(), 1);
+    assert_eq!(stored[0]["value"]["who"], json!(winner));
+}
+
+/* --------------------------------- 5. crash and restart ------------------------------- */
+
+#[tokio::test]
+async fn identity_survives_a_killed_authority() {
+    let _process_test = PROCESS_TESTS.lock().unwrap_or_else(|e| e.into_inner());
+    let directory = TempDir::new().unwrap();
+    let authority = start(&directory);
+    let first = AuthorityClient::connect(&authority.socket)
+        .commit(put("tx-1", "items:a", json!({"n": 1})))
+        .await
+        .unwrap();
+    stop(authority);
+
+    let restarted = start(&directory);
+    let client = AuthorityClient::connect(&restarted.socket);
+    let again = client.commit(put("tx-1", "items:a", json!({"n": 1}))).await.unwrap();
+    assert!(again.duplicate);
+    assert_eq!(binding(&again), json!("verified"));
+    assert_eq!(again.commit_revision, first.commit_revision);
+    let error = client.commit(put("tx-1", "items:a", json!({"n": 2}))).await.unwrap_err();
+    assert_eq!(code(&error), "IDEMPOTENCY_CONFLICT");
+    assert_eq!(client.current_revision().await.unwrap(), 1);
+}
+
+#[tokio::test]
+async fn a_torn_commit_was_never_acknowledged_so_its_retry_is_a_first_use() {
+    use std::io::Write;
+    let _process_test = PROCESS_TESTS.lock().unwrap_or_else(|e| e.into_inner());
+    let directory = TempDir::new().unwrap();
+    let authority = start(&directory);
+    AuthorityClient::connect(&authority.socket)
+        .commit(put("tx-1", "items:a", json!({"n": 1})))
+        .await
+        .unwrap();
+    let storage = stop(authority);
+    // A crash inside the next transaction's record, after its payload hash was written.
+    let mut log = std::fs::OpenOptions::new().append(true).open(&storage).unwrap();
+    log.write_all(br#"{"record_type":"feltdb.transaction.v1","transaction_id":"tx-2","payload_hash":"sha256:abc""#)
+        .unwrap();
+    log.sync_all().unwrap();
+
+    let recovered = start(&directory);
+    let client = AuthorityClient::connect(&recovered.socket);
+    let retried = client.commit(put("tx-2", "items:b", json!({"n": 2}))).await.unwrap();
+    assert!(!retried.duplicate, "the torn record was discarded, not remembered");
+    assert_eq!(client.current_revision().await.unwrap(), 2);
+}
+
+/* --------------------------- 6. logs written before the binding ------------------------ */
+
+#[tokio::test]
+async fn an_id_committed_without_a_hash_is_legacy_unverified_and_never_adopted() {
+    let directory = TempDir::new().unwrap();
+    let path = directory.path().join("legacy.db");
+    {
+        // What the authority did before it bound ids: no payload hash.
+        let db = FeltDb::open(&path).unwrap();
+        db.apply_atomic_transaction(
+            "old-1",
+            None,
+            &[],
+            &[AtomicMutation {
+                capability: "items".into(),
+                key: "items:a".into(),
+                value: Some(json!({"n": 1})),
+            }],
+            None,
+        )
+        .unwrap();
+    }
+    for pass in 0..2 {
+        // Pass 1 reopens the log after pass 0's retries, to show they recorded nothing.
+        let authority = EmbeddedAuthority::open(&path).unwrap();
+        let same = authority.commit(put("old-1", "items:a", json!({"n": 1}))).await.unwrap();
+        assert!(same.duplicate, "pass {pass}");
+        assert_eq!(binding(&same), json!("legacy_unverified"), "pass {pass}");
+        // There is nothing to compare a different payload against: it is the same answer,
+        // labelled, and it applies nothing.
+        let different = authority.commit(put("old-1", "items:a", json!({"n": 99}))).await.unwrap();
+        assert!(different.duplicate);
+        assert_eq!(binding(&different), json!("legacy_unverified"));
+        assert_eq!(rows(&authority).await[0]["value"]["n"], json!(1));
+    }
+    let db = FeltDb::open(&path).unwrap();
+    assert_eq!(db.applied_transaction_payload_hash("old-1").unwrap(), None, "the retry did not give the legacy id a hash");
+    // New ids written afterwards are bound.
+    let authority = EmbeddedAuthority::from_db(db);
+    authority.commit(put("new-1", "items:b", json!({"n": 1}))).await.unwrap();
+    let error = authority.commit(put("new-1", "items:b", json!({"n": 2}))).await.unwrap_err();
+    assert_eq!(code(&error), "IDEMPOTENCY_CONFLICT");
+}
+
+#[tokio::test]
+async fn a_recorded_hash_this_build_cannot_judge_fails_closed() {
+    let directory = TempDir::new().unwrap();
+    let path = directory.path().join("future.db");
+    {
+        let db = FeltDb::open(&path).unwrap();
+        for (id, hash) in [
+            ("future", "authority-tx9:sha256:0000000000000000000000000000000000000000000000000000000000000000"),
+            ("other-form", "sha256:abc"),
+        ] {
+            db.apply_atomic_transaction_content_addressed(
+                id,
+                Some(hash),
+                None,
+                &[],
+                &[AtomicMutation {
+                    capability: "items".into(),
+                    key: format!("items:{id}"),
+                    value: Some(json!({"id": id})),
+                }],
+                None,
+            )
+            .unwrap();
+        }
+    }
+    let authority = EmbeddedAuthority::open(&path).unwrap();
+    for id in ["future", "other-form"] {
+        let error = authority
+            .commit(put(id, &format!("items:{id}"), json!({"id": id})))
+            .await
+            .unwrap_err();
+        assert_eq!(code(&error), "IDEMPOTENCY_UNVERIFIABLE", "{id}: {error}");
+    }
+    assert_eq!(authority.current_revision().await.unwrap(), 2, "nothing was applied");
+}
+
+/* ---------------------- 7. outcomes a caller can be misled about ----------------------- */
+
+#[tokio::test]
+async fn a_conflict_crosses_the_socket_as_a_conflict_not_as_a_generic_error() {
+    let _process_test = PROCESS_TESTS.lock().unwrap_or_else(|e| e.into_inner());
+    let directory = TempDir::new().unwrap();
+    let authority = start(&directory);
+    let client = AuthorityClient::connect(&authority.socket);
+    client.commit(put("tx-1", "items:a", json!({"n": 1}))).await.unwrap();
+    let error = client.commit(put("tx-1", "items:a", json!({"n": 2}))).await.unwrap_err();
+    assert_eq!(code(&error), "IDEMPOTENCY_CONFLICT");
+    assert_ne!(code(&error), "AUTHORITY_ERROR");
+    assert_ne!(code(&error), "AUTHORITY_UNAVAILABLE");
+}
+
+/// A socket that reads one request line and then does `after` with the connection.
+async fn fake_authority(path: &Path, reply: Option<Vec<u8>>) {
+    let listener = UnixListener::bind(path).unwrap();
+    tokio::spawn(async move {
+        let (stream, _) = listener.accept().await.unwrap();
+        let (reader, mut writer) = stream.into_split();
+        let mut lines = BufReader::new(reader).lines();
+        let _ = lines.next_line().await;
+        if let Some(reply) = reply {
+            let _ = writer.write_all(&reply).await;
+        }
+        // dropping the stream closes it
+    });
+}
+
+#[tokio::test]
+async fn a_commit_that_may_have_reached_the_authority_is_reported_as_unknown_not_failed() {
+    let directory = TempDir::new().unwrap();
+    // The authority read the request and went away without answering.
+    let silent = directory.path().join("silent.sock");
+    fake_authority(&silent, None).await;
+    let error = AuthorityClient::connect(&silent)
+        .commit(put("tx-1", "items:a", json!({})))
+        .await
+        .unwrap_err();
+    assert_eq!(code(&error), "COMMIT_OUTCOME_UNKNOWN", "{error}");
+
+    // It answered with something that is not a response.
+    let garbled = directory.path().join("garbled.sock");
+    fake_authority(&garbled, Some(b"not json\n".to_vec())).await;
+    let error = AuthorityClient::connect(&garbled)
+        .commit(put("tx-1", "items:a", json!({})))
+        .await
+        .unwrap_err();
+    assert_eq!(code(&error), "COMMIT_OUTCOME_UNKNOWN", "{error}");
+
+    // Nothing was listening: the request never left, so a plain failure is truthful.
+    let error = AuthorityClient::connect(directory.path().join("absent.sock"))
+        .commit(put("tx-1", "items:a", json!({})))
+        .await
+        .unwrap_err();
+    assert_eq!(code(&error), "AUTHORITY_UNAVAILABLE");
+
+    // Reads have no outcome to be unsure about; they keep their codes.
+    let error = AuthorityClient::connect(&silent).current_revision().await.unwrap_err();
+    assert_ne!(code(&error), "COMMIT_OUTCOME_UNKNOWN");
+}
+
+#[tokio::test]
+async fn a_server_in_this_process_binds_ids_too() {
+    // The same contract through AuthorityServer::open, without the binary.
+    let directory = TempDir::new().unwrap();
+    let socket = directory.path().join("s.sock");
+    let server = AuthorityServer::open(&socket, directory.path().join("s.db")).unwrap();
+    let handle = tokio::spawn(server.serve());
+    for _ in 0..400 {
+        if socket.exists() {
+            break;
+        }
+        tokio::time::sleep(Duration::from_millis(5)).await;
+    }
+    let client = AuthorityClient::connect(&socket);
+    client.commit(put("tx-1", "items:a", json!({"n": 1}))).await.unwrap();
+    let error = client.commit(put("tx-1", "items:a", json!({"n": 2}))).await.unwrap_err();
+    assert_eq!(code(&error), "IDEMPOTENCY_CONFLICT");
+    handle.abort();
+}