@feltdb/core 0.6.8 → 0.6.10

package/dist/create/server-source/crates/feltdb/src/bin/feltdb_node.rs

@@ -1,11 +1,45 @@
-/// Standalone FeltDB node for OS process boundary testing
-///
-/// Simplified runtime that:
-/// 1. Accepts command via stdin (execute-tx, query-state, shutdown)
-/// 2. Outputs state to stdout in JSON format
-/// 3. Runs replication over TCP in background
-/// 4. Executes local transactions and replicates to peers
-/// 5. Receives and applies replicated transactions from peers
+//! A FeltDB replication node, as an operating-system process.
+//!
+//! This exists so replication can be proved across a real process and network
+//! boundary rather than between executors sharing a heap. It is a harness, not
+//! a product surface: it speaks a line protocol on stdin/stdout so a test can
+//! drive it, and it implements no replication semantics of its own. Every
+//! decision about ordering, deduplication, durability, and convergence is made
+//! by `DistributedTransactionExecutor`; this binary only moves bytes between
+//! processes and reports what happened.
+//!
+//! ## Links are directed, one TCP connection each
+//!
+//! `TcpTransport` holds a single stream, and its `receive()` holds the stream
+//! lock while awaiting the next message. A bidirectional link would therefore
+//! deadlock the first `send()` behind the receive that is parked on the socket.
+//!
+//! So each direction gets its own connection: for a link X -> Y, Y binds a port
+//! dedicated to X and only ever receives on it, and X connects to that port and
+//! only ever sends. Three nodes in a full mesh is six directed links.
+//!
+//! That is a topology choice made here in the harness. It changes nothing about
+//! the transport or the protocol.
+//!
+//! ## Usage
+//!
+//! ```text
+//! feltdb_node --node-id A --data DIR --recv B=127.0.0.1:9111 --send B=127.0.0.1:9121
+//! ```
+//!
+//! `--recv PEER=ADDR` binds ADDR and accepts the connection PEER makes to it.
+//! `--send PEER=ADDR` connects to the address PEER is listening on for us.
+//!
+//! Commands on stdin, one per line:
+//!
+//! ```text
+//! execute-tx RECORD   commit a local transaction and replicate it to every peer
+//! resend-last         put the identical envelope back on the wire, to test dedup
+//! query-state         the local replica's state hash and applied-operation count
+//! metrics             per-link byte and message counters, straight from the transport
+//! peers               per-link connection state
+//! shutdown            exit cleanly
+//! ```
 
 use feltdb::convergence::VectorClock;
 use feltdb::distributed_transactions::{DistributedTransactionExecutor, ReplicationMessage};
@@ -18,246 +52,442 @@ use feltdb::transactions::{
 use serde_json::json;
 use std::collections::HashMap;
 use std::io::{self, BufRead, Write};
+use std::path::PathBuf;
 use std::sync::atomic::{AtomicBool, Ordering};
 use std::sync::Arc;
 use tokio::sync::{mpsc, Mutex};
 use tokio::time::{sleep, Duration};
 
+/// One outbound connection to a peer, used only for sending.
+///
+/// `control` exists because `connect` needs `&mut self`. Everything else goes
+/// through `io`, a clone: `TcpTransport` holds its stream and metrics behind
+/// `Arc`s, so a clone is the same connection, and reading counters or sending
+/// does not queue behind whatever else holds the mutex.
+#[derive(Clone)]
+struct SendLink {
+    peer: String,
+    addr: String,
+    control: Arc<Mutex<TcpTransport>>,
+    io: TcpTransport,
+}
+
+/// One inbound connection from a peer, used only for receiving.
+///
+/// The distinction matters more here: `receive()` parks on the socket holding
+/// the transport's internal stream lock, so anything sharing an outer mutex
+/// with it would block until a message happened to arrive.
 #[derive(Clone)]
-struct NodeState {
-    executor: Arc<Mutex<DistributedTransactionExecutor>>,
-    node_id: String,
-    transport: Arc<Mutex<TcpTransport>>,
-    tx_sequence: Arc<Mutex<u64>>,
-    is_server: bool,
-    should_exit: Arc<AtomicBool>,
+struct RecvLink {
+    peer: String,
+    #[allow(dead_code)]
+    addr: String,
+    control: Arc<Mutex<TcpTransport>>,
+    io: TcpTransport,
+}
+
+fn parse_link(value: &str) -> Result<(String, String), String> {
+    let (peer, addr) = value
+        .split_once('=')
+        .ok_or_else(|| format!("expected PEER=ADDR, got {value}"))?;
+    if peer.is_empty() || addr.is_empty() {
+        return Err(format!("expected PEER=ADDR, got {value}"));
+    }
+    Ok((peer.to_string(), addr.to_string()))
+}
+
+/// Connect with retries: the peer's listener may not be bound yet, and after a
+/// peer restarts its old socket is gone and this is how the link comes back.
+async fn connect_with_retry(link: &SendLink, attempts: u32) -> bool {
+    for attempt in 0..attempts {
+        let mut control = link.control.lock().await;
+        if control.connect(&link.addr).await.is_ok() {
+            return true;
+        }
+        drop(control);
+        sleep(Duration::from_millis(25 * (attempt as u64 + 1))).await;
+    }
+    false
 }
 
 #[tokio::main]
 async fn main() -> Result<(), Box<dyn std::error::Error>> {
     let args: Vec<String> = std::env::args().collect();
 
-    // Parse arguments
-    let mut mode = "server";
-    let mut replication_port = 9001u16;
-    let mut peer_addr = String::new();
-
-    let mut i = 1;
-    while i < args.len() {
-        match args[i].as_str() {
-            "--mode" => {
-                i += 1;
-                mode = &args[i];
+    let mut node_id = String::new();
+    let mut data_dir: Option<PathBuf> = None;
+    let mut send_specs: Vec<(String, String)> = Vec::new();
+    let mut recv_specs: Vec<(String, String)> = Vec::new();
+
+    let mut index = 1;
+    while index < args.len() {
+        match args[index].as_str() {
+            "--node-id" => {
+                index += 1;
+                node_id = args
+                    .get(index)
+                    .ok_or("--node-id requires a value")?
+                    .clone();
+            }
+            "--data" => {
+                index += 1;
+                data_dir = Some(PathBuf::from(
+                    args.get(index).ok_or("--data requires a value")?,
+                ));
             }
-            "--port" => {
-                i += 1;
-                replication_port = args[i].parse()?;
+            "--send" => {
+                index += 1;
+                send_specs.push(parse_link(args.get(index).ok_or("--send requires PEER=ADDR")?)?);
             }
-            "--peer" => {
-                i += 1;
-                peer_addr = args[i].clone();
+            "--recv" => {
+                index += 1;
+                recv_specs.push(parse_link(args.get(index).ok_or("--recv requires PEER=ADDR")?)?);
             }
-            _ => {}
+            other => return Err(format!("unknown argument {other}").into()),
         }
-        i += 1;
+        index += 1;
     }
 
-    let is_server = mode == "server";
-    let node_id = if is_server {
-        "node_a".to_string()
-    } else {
-        "node_b".to_string()
-    };
+    if node_id.is_empty() {
+        return Err("--node-id is required".into());
+    }
+
+    // Durability is the executor's, not this binary's. The log path is per
+    // node, so each process owns its own directory on disk.
+    let log_path = data_dir.as_ref().map(|dir| {
+        std::fs::create_dir_all(dir).ok();
+        dir.join("operations.jsonl")
+    });
 
-    // Initialize executor
-    let initial_state = StateHash::from_hex("hash_initial".to_string());
+    let initial_state = StateHash::from_hex("0".repeat(64));
     let mut executor =
-        DistributedTransactionExecutor::new(node_id.clone(), initial_state.clone());
-    executor.register_replica("node_a".to_string(), initial_state.clone());
-    executor.register_replica("node_b".to_string(), initial_state.clone());
-
-    let transport = if is_server {
-        TcpTransport::new(format!("127.0.0.1:{}", replication_port))
-    } else {
-        TcpTransport::new("127.0.0.1:0".to_string())
-    };
-
-    let node_state = NodeState {
-        executor: Arc::new(Mutex::new(executor)),
-        node_id: node_id.clone(),
-        transport: Arc::new(Mutex::new(transport)),
-        tx_sequence: Arc::new(Mutex::new(1)),
-        is_server,
-        should_exit: Arc::new(AtomicBool::new(false)),
-    };
-
-    // Start TCP replication in background
-    if is_server {
-        let replication_addr = format!("127.0.0.1:{}", replication_port);
-        let ns = node_state.clone();
-        tokio::spawn(async move {
-            let mut transport = ns.transport.lock().await;
-            if let Err(e) = transport.listen().await {
-                eprintln!("ERROR: listen failed: {}", e);
-            }
-        });
-        println!("STARTED server on port {}", replication_port);
-        let _ = std::io::stdout().flush();
-    } else {
-        let ns = node_state.clone();
-        let peer = peer_addr.clone();
-        tokio::spawn(async move {
-            let mut transport = ns.transport.lock().await;
-            if let Err(e) = transport.connect(&peer).await {
-                eprintln!("ERROR: connect failed: {}", e);
-            }
-        });
-        println!("STARTED client connecting to {}", peer_addr);
-        let _ = std::io::stdout().flush();
+        DistributedTransactionExecutor::with_log(node_id.clone(), initial_state.clone(), log_path.clone())?;
+
+    // Replaying the log restores the deduplication set, so a restarted node
+    // does not re-apply operations it already durably recorded.
+    let mut replayed = 0usize;
+    if let Some(path) = &log_path {
+        if path.exists() {
+            replayed = executor.load_from_disk(path.clone())?;
+        }
     }
 
-    // Spawn background task to receive and apply replicated messages
-    let ns = node_state.clone();
-    tokio::spawn(async move {
-        loop {
-            if ns.should_exit.load(Ordering::Relaxed) {
-                break;
-            }
+    for (peer, _) in send_specs.iter().chain(recv_specs.iter()) {
+        if peer != &node_id {
+            executor.register_replica(peer.clone(), initial_state.clone());
+        }
+    }
+
+    // The running state version, advanced by every applied transaction.
+    //
+    // This matters more than it looks. `execute_transaction` derives the next
+    // hash as H(parent_hash, command), so passing a fresh zero-hash parent each
+    // time — as this binary and the old in-process tests both did — makes the
+    // resulting "state hash" a function of the last command alone, carrying no
+    // history. Two nodes that applied completely different sets of operations
+    // then report identical hashes as long as the most recent one matches, and
+    // a convergence check built on that comparison cannot fail.
+    //
+    // Chaining it makes the hash mean what its name says, and makes a node that
+    // missed an operation observably divergent. It lives beside the executor
+    // under one lock so a local commit and an arriving replication cannot
+    // interleave and chain onto a parent that is already stale.
+    let core = Arc::new(Mutex::new((executor, StateVersion::new(VectorClock::new(), "0".repeat(64)))));
+    let should_exit = Arc::new(AtomicBool::new(false));
+    let sequence = Arc::new(Mutex::new(1u64));
+    // Kept so `resend-last` can put the identical envelope back on the wire.
+    let last_envelope: Arc<Mutex<Option<feltdb::distributed_transactions::TransactionEnvelope>>> =
+        Arc::new(Mutex::new(None));
+
+    let send_links: Vec<SendLink> = send_specs
+        .into_iter()
+        .map(|(peer, addr)| {
+            let transport = TcpTransport::new("127.0.0.1:0".to_string());
+            SendLink { peer, addr, control: Arc::new(Mutex::new(transport.clone())), io: transport }
+        })
+        .collect();
+
+    let recv_links: Vec<RecvLink> = recv_specs
+        .into_iter()
+        .map(|(peer, addr)| {
+            let transport = TcpTransport::new(addr.clone());
+            RecvLink { peer, addr, control: Arc::new(Mutex::new(transport.clone())), io: transport }
+        })
+        .collect();
 
-            let messages = {
-                let transport = ns.transport.lock().await;
-                match transport.receive().await {
-                    Ok(msgs) => msgs,
-                    Err(_) => {
+    // Accept, serve, and accept again. One task per link, because each parks
+    // on its own socket and must not block the others.
+    //
+    // The re-listen is what makes a peer restart survivable. `TcpTransport`'s
+    // `listen()` binds, accepts exactly one connection, and drops the listener,
+    // so after that first accept the port is closed and a peer that reconnects
+    // is refused. Looping back to `listen()` once the connection ends rebinds
+    // the port for the next one. That is a harness-level fix: the transport
+    // keeps its single-connection shape and the protocol is untouched.
+    for link in &recv_links {
+        let control = link.control.clone();
+        let transport = link.io.clone();
+        let core = core.clone();
+        let exit = should_exit.clone();
+        let peer = link.peer.clone();
+        tokio::spawn(async move {
+            while !exit.load(Ordering::Relaxed) {
+                {
+                    let mut control = control.lock().await;
+                    if let Err(error) = control.listen().await {
+                        eprintln!("listen for {peer} failed: {error}");
+                        drop(control);
                         sleep(Duration::from_millis(100)).await;
                         continue;
                     }
                 }
-            };
 
-            for msg in messages {
-                let mut executor = ns.executor.lock().await;
-                let mut parent_vc = VectorClock::new();
-                parent_vc.increment(&ns.node_id);
-                let parent_version = StateVersion::new(parent_vc, "hash_initial".to_string());
-
-                if let Err(e) = executor.receive_replicated_transaction(msg, parent_version) {
-                    eprintln!("ERROR: apply replication failed: {}", e);
+                // Serve this connection until it ends, then rebind for the next.
+                loop {
+                    if exit.load(Ordering::Relaxed) {
+                        return;
+                    }
+                    match transport.receive().await {
+                        Ok(messages) => {
+                            for message in messages {
+                                let mut guard = core.lock().await;
+                                let (executor, version) = &mut *guard;
+                                let origin = message.envelope.originating_node.clone();
+                                let parent = version.clone();
+                                // A duplicate is refused by the executor, which
+                                // is the behaviour under test, not an error
+                                // here — and a refused message must not advance
+                                // the version.
+                                if let Ok(applied) =
+                                    executor.receive_replicated_transaction(message, parent)
+                                {
+                                    version.state_hash = applied.to_string();
+                                    version.vector_clock.increment(&origin);
+                                }
+                            }
+                        }
+                        Err(_) => break,
+                    }
                 }
             }
+        });
+    }
 
-            sleep(Duration::from_millis(50)).await;
-        }
-    });
-
-    // Give replication setup time to establish
-    sleep(Duration::from_millis(500)).await;
+    // Establish outbound links before announcing readiness, so a test that
+    // writes immediately is not racing the topology.
+    for link in &send_links {
+        connect_with_retry(link, 20).await;
+    }
 
-    // Create a channel for stdin commands
-    let (tx, mut rx) = mpsc::channel::<String>(100);
+    println!(
+        "STARTED {}",
+        json!({
+            "node": node_id,
+            "replayed": replayed,
+            "send": send_links.iter().map(|l| l.peer.clone()).collect::<Vec<_>>(),
+            "recv": recv_links.iter().map(|l| l.peer.clone()).collect::<Vec<_>>(),
+        })
+    );
+    io::stdout().flush().ok();
 
-    // Spawn a blocking task to read stdin
+    let (commands, mut incoming) = mpsc::channel::<String>(100);
     tokio::task::spawn_blocking(move || {
-        let stdin = io::stdin();
-        let handle = stdin.lock();
-
-        for line in handle.lines() {
-            if let Ok(cmd) = line {
-                let cmd_str = cmd.trim().to_string();
-                if !cmd_str.is_empty() {
-                    // Non-blocking send, ignore if receiver is gone
-                    let _ = tx.blocking_send(cmd_str);
-                }
+        for line in io::stdin().lock().lines().map_while(Result::ok) {
+            let line = line.trim().to_string();
+            if !line.is_empty() && commands.blocking_send(line).is_err() {
+                break;
             }
         }
     });
 
-    // Process commands from the channel
-    while let Some(cmd) = rx.recv().await {
-        match cmd.trim() {
+    while let Some(command) = incoming.recv().await {
+        let (verb, argument) = match command.split_once(' ') {
+            Some((verb, rest)) => (verb, rest.trim().to_string()),
+            None => (command.as_str(), String::new()),
+        };
+
+        match verb {
             "execute-tx" => {
+                let record = if argument.is_empty() { "item".to_string() } else { argument };
                 let seq = {
-                    let mut s = node_state.tx_sequence.lock().await;
-                    let current = *s;
-                    *s += 1;
+                    let mut guard = sequence.lock().await;
+                    let current = *guard;
+                    *guard += 1;
                     current
                 };
 
-                // Create a test operation
-                let mut parent_vc = VectorClock::new();
-                parent_vc.increment(&node_state.node_id);
-                let parent_version = StateVersion::new(parent_vc, "hash_initial".to_string());
-
                 let mut fields = HashMap::new();
-                fields.insert("key".to_string(), json!("value"));
+                fields.insert("record".to_string(), json!(record.clone()));
 
-                let op = Operation::new(
-                    OperationId::new(node_state.node_id.clone(), seq),
-                    parent_version.clone(),
-                    format!("tx_{}", seq),
+                let parent = { core.lock().await.1.clone() };
+                let operation = Operation::new(
+                    OperationId::new(node_id.clone(), seq),
+                    parent.clone(),
+                    format!("tx_{record}"),
                     OperationCommand {
                         op_type: "set".to_string(),
                         collection: "items".to_string(),
-                        record_id: format!("item{}", seq),
+                        record_id: record.clone(),
                         fields,
                     },
-                    node_state.node_id.clone(),
+                    node_id.clone(),
                 );
 
-                {
-                    let mut executor = node_state.executor.lock().await;
-                    match executor.execute_local_transaction(
+                let envelope = {
+                    let mut guard = core.lock().await;
+                    let (executor, version) = &mut *guard;
+                    let result = executor.execute_local_transaction(
                         seq,
-                        format!("tx_{}", seq),
-                        parent_version,
-                        vec![op],
+                        format!("tx_{record}"),
+                        version.clone(),
+                        vec![operation],
                         ConsistencyContract::local(),
-                    ) {
-                        Ok(envelope) => {
-                            let replication_msg = ReplicationMessage::new(
-                                envelope,
-                                node_state.node_id.clone(),
-                                if node_state.node_id == "node_a" {
-                                    "node_b".to_string()
-                                } else {
-                                    "node_a".to_string()
-                                },
+                    );
+                    if result.is_ok() {
+                        if let Some(replica) = executor.get_replica_state(&node_id) {
+                            version.state_hash = replica.state_hash.to_string();
+                            version.vector_clock.increment(&node_id);
+                        }
+                    }
+                    result
+                };
+
+                match envelope {
+                    Ok(envelope) => {
+                        *last_envelope.lock().await = Some(envelope.clone());
+                        let mut delivered = Vec::new();
+                        let mut failed = Vec::new();
+                        for link in &send_links {
+                            let message = ReplicationMessage::new(
+                                envelope.clone(),
+                                node_id.clone(),
+                                link.peer.clone(),
                                 seq,
                             );
-
-                            let transport = node_state.transport.lock().await;
-                            if let Err(e) = transport.send(replication_msg).await {
-                                println!("ERROR: send replication failed: {}", e);
-                            } else {
-                                println!("TX_EXECUTED");
+                            let sent = link.io.send(message.clone()).await;
+                            match sent {
+                                Ok(()) => delivered.push(link.peer.clone()),
+                                Err(_) => {
+                                    // The peer may have restarted, which drops
+                                    // our end of the socket. Reconnect and try
+                                    // once more before reporting a failure.
+                                    if connect_with_retry(link, 3).await {
+                                        if link.io.send(message).await.is_ok() {
+                                            delivered.push(link.peer.clone());
+                                            continue;
+                                        }
+                                    }
+                                    failed.push(link.peer.clone());
+                                }
                             }
                         }
-                        Err(e) => {
-                            println!("ERROR: execute transaction failed: {}", e);
+                        println!(
+                            "TX_EXECUTED {}",
+                            json!({ "record": record, "sequence": seq, "delivered": delivered, "failed": failed })
+                        );
+                    }
+                    Err(error) => println!("ERROR {}", json!({ "reason": error })),
+                }
+                io::stdout().flush().ok();
+            }
+
+            // Re-send the most recent envelope, unchanged, to every peer.
+            //
+            // Deduplication is a claimed property of the executor, and it is
+            // only interesting if the duplicate actually crosses the wire.
+            // Issuing another `execute-tx` would not test it: that produces a
+            // new envelope id and is a different transaction.
+            "resend-last" => {
+                let envelope = last_envelope.lock().await.clone();
+                match envelope {
+                    Some(envelope) => {
+                        let mut delivered = Vec::new();
+                        for link in &send_links {
+                            let message = ReplicationMessage::new(
+                                envelope.clone(),
+                                node_id.clone(),
+                                link.peer.clone(),
+                                envelope.envelope_id.sequence,
+                            );
+                            if link.io.send(message).await.is_ok() {
+                                delivered.push(link.peer.clone());
+                            }
                         }
+                        println!(
+                            "RESENT {}",
+                            json!({ "envelope": envelope.envelope_id, "delivered": delivered })
+                        );
                     }
+                    None => println!("ERROR {}", json!({ "reason": "nothing sent yet" })),
                 }
-                let _ = std::io::stdout().flush();
+                io::stdout().flush().ok();
             }
+
             "query-state" => {
-                let executor = node_state.executor.lock().await;
-                if let Some(replica) = executor.get_replica_state(&node_state.node_id) {
-                    println!("STATE:{},{}", replica.state_hash, replica.operations_applied);
-                } else {
-                    println!("ERROR: node not found");
+                let guard = core.lock().await;
+                let (executor, _) = &*guard;
+                match executor.get_replica_state(&node_id) {
+                    Some(replica) => println!(
+                        "STATE {}",
+                        json!({
+                            "hash": replica.state_hash.to_string(),
+                            "operations_applied": replica.operations_applied,
+                        })
+                    ),
+                    None => println!("ERROR {}", json!({ "reason": "no local replica" })),
+                }
+                io::stdout().flush().ok();
+            }
+
+            // The proof that the network carried the work. Counters come from
+            // the transport itself, so a test can assert bytes actually moved
+            // rather than inferring it from converged state.
+            "metrics" => {
+                let mut sent = json!({});
+                let mut received = json!({});
+                for link in &send_links {
+                    let metrics = link.io.metrics();
+                    sent[&link.peer] = json!({
+                        "bytes": metrics.get_bytes_sent(),
+                        "messages": metrics.get_messages_sent(),
+                        "connection_changes": metrics.get_connection_state_changes(),
+                    });
+                }
+                for link in &recv_links {
+                    let metrics = link.io.metrics();
+                    received[&link.peer] = json!({
+                        "bytes": metrics.get_bytes_received(),
+                        "messages": metrics.get_messages_received(),
+                        "connection_changes": metrics.get_connection_state_changes(),
+                    });
                 }
-                let _ = std::io::stdout().flush();
+                println!("METRICS {}", json!({ "sent": sent, "received": received }));
+                io::stdout().flush().ok();
             }
+
+            "peers" => {
+                let mut state = json!({});
+                for link in &send_links {
+                    state[format!("send:{}", link.peer)] = json!(link.io.is_connected());
+                }
+                for link in &recv_links {
+                    state[format!("recv:{}", link.peer)] = json!(link.io.is_connected());
+                }
+                println!("PEERS {state}");
+                io::stdout().flush().ok();
+            }
+
             "shutdown" => {
-                node_state.should_exit.store(true, Ordering::Relaxed);
+                should_exit.store(true, Ordering::Relaxed);
                 println!("SHUTDOWN");
-                let _ = std::io::stdout().flush();
-                sleep(Duration::from_millis(100)).await;
+                io::stdout().flush().ok();
+                sleep(Duration::from_millis(50)).await;
                 std::process::exit(0);
             }
+
             _ => {
-                println!("ERROR: unknown command");
-                let _ = std::io::stdout().flush();
+                println!("ERROR {}", json!({ "reason": format!("unknown command {verb}") }));
+                io::stdout().flush().ok();
             }
         }
     }