librats 0.7.1 → 0.8.0

package/native-src/src/librats.h

@@ -9,6 +9,9 @@
 #include "file_transfer.h" // File transfer functionality
 #include "noise.h" // Noise Protocol encryption
 #include "ice.h"   // ICE-lite NAT traversal
+#include "io_poller.h" // Platform-optimal I/O multiplexing
+#include "receive_buffer.h" // Efficient receive buffer for async I/O
+#include "chained_send_buffer.h" // Zero-copy chained send buffer
 #ifdef RATS_STORAGE
 #include "storage.h" // Distributed storage functionality
 #endif
@@ -25,14 +28,32 @@
 #include <unordered_map>
 #include <memory>
 #include <chrono>
-#include <condition_variable>
-#include <unordered_set> // Added for unordered_set
+#include <unordered_set>
 #include <cstdint>
 #include <cstring>
+#include <optional>
 #include "rats_export.h"
 
 namespace librats {
 
+/**
+ * PeerIOContext - Per-peer async I/O buffers and framing state
+ *
+ * Used by the single-threaded IO loop for non-blocking message framing:
+ *   recv_buffer  – incoming bytes from recv(); frames parsed incrementally
+ *   send_buffer  – outgoing frames queued for non-blocking send()
+ *   noise_hs     – transient Noise XX handshake state (only during NOISE_PENDING)
+ *   noise_step   – current step in the 3-message XX pattern
+ */
+struct PeerIOContext {
+    ReceiveBuffer recv_buffer{8192};
+    ChainedSendBuffer send_buffer;
+
+    // Async Noise handshake (only valid while handshake_state == NOISE_PENDING)
+    std::unique_ptr<rats::NoiseHandshakeState> noise_hs;
+    int noise_step = 0;   // XX pattern: 0→initial, advances per message
+};
+
 /**
  * RatsPeer struct - comprehensive information about a connected rats peer
  */
@@ -65,6 +86,9 @@ struct RatsPeer {
     std::shared_ptr<rats::NoiseCipherState> recv_cipher;   // Cipher for receiving encrypted data
     std::vector<uint8_t> remote_static_key;  // Remote peer's static public key (for identity verification)
     
+    // Async I/O context (per-peer buffers for non-blocking I/O)
+    PeerIOContext io_;
+    
     RatsPeer() : handshake_state(HandshakeState::PENDING), 
                  encryption_enabled(false),
                  noise_handshake_completed(false) {
@@ -83,6 +107,45 @@ struct RatsPeer {
         handshake_start_time = connected_at;
     }
     
+    // Custom copy: copies all fields except io_ (non-copyable due to unique_ptr).
+    // Copies are used for snapshots passed to callbacks – they never need the IO context.
+    RatsPeer(const RatsPeer& o)
+        : peer_id(o.peer_id), ip(o.ip), port(o.port), socket(o.socket),
+          normalized_address(o.normalized_address), connected_at(o.connected_at),
+          is_outgoing(o.is_outgoing), handshake_state(o.handshake_state),
+          version(o.version), handshake_start_time(o.handshake_start_time),
+          encryption_enabled(o.encryption_enabled),
+          noise_handshake_completed(o.noise_handshake_completed),
+          send_cipher(o.send_cipher), recv_cipher(o.recv_cipher),
+          remote_static_key(o.remote_static_key)
+          /* io_ default-constructed (fresh, empty) */ {}
+    
+    RatsPeer& operator=(const RatsPeer& o) {
+        if (this != &o) {
+            peer_id = o.peer_id;
+            ip = o.ip;
+            port = o.port;
+            socket = o.socket;
+            normalized_address = o.normalized_address;
+            connected_at = o.connected_at;
+            is_outgoing = o.is_outgoing;
+            handshake_state = o.handshake_state;
+            version = o.version;
+            handshake_start_time = o.handshake_start_time;
+            encryption_enabled = o.encryption_enabled;
+            noise_handshake_completed = o.noise_handshake_completed;
+            send_cipher = o.send_cipher;
+            recv_cipher = o.recv_cipher;
+            remote_static_key = o.remote_static_key;
+            // io_ left unchanged in the destination (no copy)
+        }
+        return *this;
+    }
+    
+    // Default move operations are fine
+    RatsPeer(RatsPeer&&) = default;
+    RatsPeer& operator=(RatsPeer&&) = default;
+    
     // Check if peer has completed Noise handshake and is ready for encrypted communication
     bool is_noise_encrypted() const { 
         return noise_handshake_completed && send_cipher && recv_cipher; 
@@ -261,7 +324,6 @@ public:
      */
     bool is_running() const;
 
-
     // =========================================================================
     // Utility Methods
     // =========================================================================
@@ -387,7 +449,6 @@ public:
      */
     int get_peer_count() const;
 
-
     /**
      * Get peer_id for a peer by socket (preferred)
      * @param socket Peer socket
@@ -423,16 +484,16 @@ public:
     /**
      * Get peer information by peer ID
      * @param peer_id The peer ID to look up
-     * @return Pointer to RatsPeer object, or nullptr if not found
+     * @return Copy of RatsPeer object, or std::nullopt if not found
      */
-    const RatsPeer* get_peer_by_id(const std::string& peer_id) const;
+    std::optional<RatsPeer> get_peer_by_id(const std::string& peer_id) const;
     
     /**
      * Get peer information by socket
      * @param socket The socket handle to look up
-     * @return Pointer to RatsPeer object, or nullptr if not found  
+     * @return Copy of RatsPeer object, or std::nullopt if not found
      */
-    const RatsPeer* get_peer_by_socket(socket_t socket) const;
+    std::optional<RatsPeer> get_peer_by_socket(socket_t socket) const;
     
     /**
      * Get maximum number of peers
@@ -476,9 +537,9 @@ public:
     
     /**
      * Get current reconnection configuration
-     * @return Current reconnection configuration
+     * @return Copy of current reconnection configuration
      */
-    const ReconnectConfig& get_reconnect_config() const;
+    ReconnectConfig get_reconnect_config() const;
     
     /**
      * Get the number of peers pending reconnection
@@ -636,6 +697,13 @@ public:
      */
     bool is_automatic_discovery_running() const;
     
+    /**
+     * Calculate discovery interval based on current peer count
+     * Uses graduated scaling: more aggressive when fewer peers, less aggressive when nearly full
+     * @return Discovery interval in seconds
+     */
+    std::chrono::seconds calculate_discovery_interval() const;
+    
     /**
      * Get the discovery hash for current protocol configuration
      * @return Discovery hash based on current protocol name and version
@@ -723,14 +791,6 @@ public:
      */
     void send(const std::string& peer_id, const std::string& message_type, const nlohmann::json& data, SendCallback callback = nullptr);
 
-    /**
-     * Parse a JSON message
-     * @param message Raw message string
-     * @param out_json Parsed JSON output
-     * @return true if parsed successfully
-     */
-    bool parse_json_message(const std::string& message, nlohmann::json& out_json);
-
     // =========================================================================
     // Encryption Functionality
     // =========================================================================
@@ -1084,6 +1144,20 @@ public:
      */
     void set_log_retention_count(int count);
     
+    /**
+     * Enable or disable log rotation on application startup
+     * When enabled, the existing log file will be rotated when logging starts,
+     * so each application run gets a fresh log file
+     * @param enabled Whether to rotate logs on startup (default: false)
+     */
+    void set_log_rotate_on_startup(bool enabled);
+    
+    /**
+     * Check if log rotation on startup is enabled
+     * @return true if rotate on startup is enabled
+     */
+    bool is_log_rotate_on_startup_enabled() const;
+    
     /**
      * Clear/reset the current log file
      */
@@ -1233,7 +1307,7 @@ public:
      * Get current file transfer configuration
      * @return Current configuration settings
      */
-    const FileTransferConfig& get_file_transfer_config() const;
+    FileTransferConfig get_file_transfer_config() const;
     
     // Event Handlers
     /**
@@ -1442,7 +1516,7 @@ public:
      * Get current ICE configuration
      * @return Current ICE configuration
      */
-    const IceConfig& get_ice_config() const;
+    IceConfig get_ice_config() const;
     
     // ICE Lifecycle
     /**
@@ -1938,7 +2012,7 @@ private:
     // 3. encryption_mutex_           (Encryption settings and keys)
     // 4. local_addresses_mutex_      (Local interface addresses)
     // 5. peers_mutex_                (Peer management - most frequently locked)
-    // 6. socket_send_mutexes_mutex_ (Socket send mutex management)
+    // 6. io_mutex_                  (I/O poller and send buffer access)
     // 7. message_handlers_mutex_    (Message handler registration)
     // 8. reconnect_mutex_           (Reconnection queue management)
     // =========================================================================
@@ -1964,20 +2038,19 @@ private:
     
     // [4] Local interface address blocking (protected by local_addresses_mutex_)
     mutable std::mutex local_addresses_mutex_;              // [4] Protects local interface addresses
-    std::vector<std::string> local_interface_addresses_;
+    std::unordered_set<std::string> local_interface_addresses_;
     
     // [5] Organized peer management using RatsPeer struct (protected by peers_mutex_)
     mutable std::mutex peers_mutex_;                        // [5] Protects peer data (most frequently locked)
     std::unordered_map<std::string, RatsPeer> peers_;          // keyed by peer_id
     std::unordered_map<socket_t, std::string> socket_to_peer_id_;  // for quick socket->peer_id lookup  
     std::unordered_map<std::string, std::string> address_to_peer_id_;  // for duplicate detection (normalized_address->peer_id)
+    std::atomic<int> validated_peer_count_{0};              // Cached count of peers with COMPLETED handshake
     
-    // [6] Per-socket synchronization for thread-safe message sending (protected by socket_send_mutexes_mutex_)
-    mutable std::mutex socket_send_mutexes_mutex_;          // [6] Protects socket send mutex map
-    std::unordered_map<socket_t, std::shared_ptr<std::mutex>> socket_send_mutexes_;
-    
-    // Server and client management
-    std::thread server_thread_;
+    // [6] Async I/O (poller + io thread)
+    std::unique_ptr<IOPoller> poller_;
+    std::mutex io_mutex_;                                    // Protects poller_ and send-buffer writes from non-IO threads
+    std::thread io_thread_;
     std::thread management_thread_;
     
     ConnectionCallback connection_callback_;
@@ -2015,11 +2088,31 @@ private:
     void initialize_modules();
     void destroy_modules();
 
-    void server_loop();
+    // Async I/O loop (single thread for all sockets)
+    void io_loop();
     void management_loop();
-    void handle_client(socket_t client_socket, const std::string& peer_hash_id);
+    
+    // I/O event handlers (called from io_loop)
+    void accept_incoming();
+    bool handle_readable(socket_t socket);
+    bool handle_writable(socket_t socket);
+    void handle_disconnect(socket_t socket);
+    
+    // Poller registration helpers
+    void poller_add(socket_t fd, uint32_t events);
+    void poller_modify(socket_t fd, uint32_t events);
+    void poller_remove(socket_t fd);
+    
+    // Helpers – post-handshake and message routing
+    void handle_post_handshake_completion(socket_t socket, const RatsPeer& peer_copy);
+    void process_message(socket_t socket, const std::vector<uint8_t>& data, const std::string& peer_id);
+    
+    // Peer lookup helper (assumes peers_mutex_ is already locked)
+    std::unordered_map<std::string, RatsPeer>::iterator find_peer_by_socket_unlocked(socket_t socket);
+    std::unordered_map<std::string, RatsPeer>::const_iterator find_peer_by_socket_unlocked(socket_t socket) const;
+    
     void remove_peer(socket_t socket);
-    std::string generate_peer_hash_id(socket_t socket, const std::string& connection_info);
+    std::string generate_temporary_peer_id(socket_t socket, const std::string& connection_info);
     void handle_dht_peer_discovery(const std::vector<Peer>& peers, const InfoHash& info_hash);
     void handle_mdns_service_discovery(const MdnsService& service, bool is_new);
     
@@ -2028,31 +2121,62 @@ private:
     bool parse_message_with_header(const std::vector<uint8_t>& message, MessageHeader& header, std::vector<uint8_t>& payload) const;
     
     // Peer management methods using RatsPeer
-    void add_peer(const RatsPeer& peer);
     void add_peer_unlocked(const RatsPeer& peer);  // Assumes peers_mutex_ is already locked
-    void remove_peer_by_id(const std::string& peer_id);
     void remove_peer_by_id_unlocked(const std::string& peer_id);  // Assumes peers_mutex_ is already locked
+    void mark_manual_disconnect(const std::string& peer_id);
+    static std::vector<uint8_t> json_to_binary(const nlohmann::json& data);
     bool is_already_connected_to_address(const std::string& normalized_address) const;
     std::string normalize_peer_address(const std::string& ip, int port) const;
     
-    // Data transmission helper - assumes peers_mutex_ is already locked or peer data is cached
+    // Async send – enqueues a framed (length-prefixed) message into the peer's
+    // ChainedSendBuffer and arms PollOut.  Thread-safe (acquires io_mutex_).
+    // Returns false if the peer was not found.
+    bool enqueue_message(socket_t socket, const std::vector<uint8_t>& data);
+    // Unlocked variant – caller must hold peers_mutex_
+    bool enqueue_message_unlocked(RatsPeer& peer, const std::vector<uint8_t>& data);
+
+    // Lightweight snapshot of peer data needed for sending (avoids holding peers_mutex_ during enqueue)
+    struct PeerSendTarget {
+        socket_t socket;
+        std::string peer_id;
+        std::shared_ptr<rats::NoiseCipherState> send_cipher;
+    };
+    
     bool send_binary_to_peer_unlocked(socket_t socket, const std::vector<uint8_t>& data, 
                                        MessageDataType message_type, 
-                                       rats::NoiseCipherState* send_cipher,
+                                       std::shared_ptr<rats::NoiseCipherState> send_cipher,
                                        const std::string& peer_id_for_logging);
+    
+    // Async Noise handshake – processes one Noise XX message received from peer.
+    // Returns false if peer should be disconnected.  Called from handle_readable.
+    bool handle_noise_frame(RatsPeer& peer);
+    // Kick-off: initialises noise_hs and writes first outgoing message if initiator.
+    void start_noise_handshake_async(RatsPeer& peer);
 
     // Local interface address blocking helper functions
     void initialize_local_addresses();
     bool is_blocked_address(const std::string& ip_address) const;
     bool should_ignore_peer(const std::string& ip, int port) const;
+    bool can_connect_to_peer(const std::string& ip, int port) const;
     static bool parse_address_string(const std::string& address_str, std::string& out_ip, int& out_port);
     
     // Helper functions that assume mutex is already locked
     int get_peer_count_unlocked() const;  // Helper that assumes peers_mutex_ is already locked
 
-    // Handshake protocol
+    // Protocol constants
     static constexpr const char* RATS_PROTOCOL_VERSION = "1.0";
     static constexpr int HANDSHAKE_TIMEOUT_SECONDS = 10;
+    static constexpr int TCP_CONNECT_TIMEOUT_MS = 10000;        // 10 second TCP connection timeout
+    static constexpr int IO_POLL_TIMEOUT_MS = 100;              // IO poller tick interval (ms)
+    static constexpr size_t MAX_FRAME_SIZE = 100 * 1024 * 1024; // 100 MB max single frame
+    static constexpr int PEER_RECONNECT_DELAY_MS = 100;         // Delay before reconnecting saved peers
+    static constexpr int HISTORICAL_RECONNECT_DELAY_MS = 500;   // Delay before reconnecting historical peers
+    static constexpr int MANAGEMENT_LOOP_INTERVAL_SECONDS = 2;  // Management loop tick interval
+    static constexpr int THREAD_CLEANUP_INTERVAL_SECONDS = 30;  // Thread cleanup interval
+    static constexpr int INITIAL_DISCOVERY_DELAY_SECONDS = 5;   // DHT bootstrap delay
+    static constexpr int MAX_PEERS_REQUEST_COUNT = 5;            // Max peers to request/respond
+    static constexpr int CONTENT_HASH_HEX_LENGTH = 40;          // 160-bit hash as hex
+    static constexpr int64_t TIMESTAMP_SKEW_TOLERANCE_MS = 10LL * 60LL * 1000LL; // 10 minutes
 
     struct HandshakeMessage {
         std::string protocol;
@@ -2068,9 +2192,8 @@ private:
     bool parse_handshake_message(const std::vector<uint8_t>& data, HandshakeMessage& out_msg) const;
     bool validate_handshake_message(const HandshakeMessage& msg) const;
     bool is_handshake_message(const std::vector<uint8_t>& data) const;
-    bool send_handshake(socket_t socket, const std::string& our_peer_id);
-    bool send_handshake_unlocked(socket_t socket, const std::string& our_peer_id);
-    bool handle_handshake_message(socket_t socket, const std::string& peer_hash_id, const std::vector<uint8_t>& data);
+    bool send_handshake_unlocked(RatsPeer& peer, const std::string& our_peer_id);  // enqueues via send buffer (peers_mutex_ held)
+    bool handle_handshake_message(socket_t socket, const std::string& initial_peer_id, const std::vector<uint8_t>& data);
     void check_handshake_timeouts();
     void log_handshake_completion_unlocked(const RatsPeer& peer);
 
@@ -2082,12 +2205,12 @@ private:
 
     // Message handling system
     nlohmann::json create_rats_message(const std::string& type, const nlohmann::json& payload, const std::string& sender_peer_id);
-    void handle_rats_message(socket_t socket, const std::string& peer_hash_id, const nlohmann::json& message);
+    void handle_rats_message(socket_t socket, const std::string& peer_id, const nlohmann::json& message);
 
     // Specific message handlers
-    void handle_peer_exchange_message(socket_t socket, const std::string& peer_hash_id, const nlohmann::json& payload);
-    void handle_peers_request_message(socket_t socket, const std::string& peer_hash_id, const nlohmann::json& payload);
-    void handle_peers_response_message(socket_t socket, const std::string& peer_hash_id, const nlohmann::json& payload);
+    void handle_peer_exchange_message(socket_t socket, const std::string& peer_id, const nlohmann::json& payload);
+    void handle_peers_request_message(socket_t socket, const std::string& peer_id, const nlohmann::json& payload);
+    void handle_peers_response_message(socket_t socket, const std::string& peer_id, const nlohmann::json& payload);
 
     // Message creation and broadcasting
     nlohmann::json create_peer_exchange_message(const RatsPeer& peer);
@@ -2097,8 +2220,7 @@ private:
     std::vector<RatsPeer> get_random_peers(int max_count, const std::string& exclude_peer_id = "") const;
     void send_peers_request(socket_t socket, const std::string& our_peer_id);
 
-    int broadcast_rats_message(const nlohmann::json& message, const std::string& exclude_peer_id = "");
-    int broadcast_rats_message_to_validated_peers(const nlohmann::json& message, const std::string& exclude_peer_id = "");
+    int broadcast_rats_message(const nlohmann::json& message, const std::string& exclude_peer_id = "", bool validated_only = true);
     
     // [7] Message exchange API implementation (protected by message_handlers_mutex_)
     mutable std::mutex message_handlers_mutex_;             // [7] Protects message handlers
@@ -2124,10 +2246,6 @@ private:
     void remove_from_reconnect_queue(const std::string& peer_id);
     int get_retry_interval_seconds(int attempt, bool is_stable) const;
 
-    // Per-socket synchronization helpers
-    std::shared_ptr<std::mutex> get_socket_send_mutex(socket_t socket);
-    void cleanup_socket_send_mutex(socket_t socket);
-
     // Configuration persistence helpers
     std::string generate_persistent_peer_id() const;
     nlohmann::json serialize_peer_for_persistence(const RatsPeer& peer) const;
@@ -2141,11 +2259,6 @@ private:
     
     // Noise Protocol encryption helpers
     void initialize_noise_keypair();
-    bool perform_noise_handshake(socket_t socket, const std::string& peer_id, bool is_initiator);
-    bool send_noise_message(socket_t socket, const uint8_t* data, size_t len);
-    bool recv_noise_message(socket_t socket, std::vector<uint8_t>& out_data, int timeout_ms = 10000);
-    bool encrypt_and_send(socket_t socket, const std::string& peer_id, const std::vector<uint8_t>& plaintext);
-    bool receive_and_decrypt(socket_t socket, const std::string& peer_id, std::vector<uint8_t>& plaintext);
 };
 
 // Utility functions

package/native-src/src/librats_bittorrent.cpp

@@ -4,11 +4,7 @@
 
 #ifdef RATS_SEARCH_FEATURES
 
-// Logging macros for BitTorrent client
-#define LOG_CLIENT_DEBUG(message) LOG_DEBUG("client", message)
-#define LOG_CLIENT_INFO(message)  LOG_INFO("client", message)
-#define LOG_CLIENT_WARN(message)  LOG_WARN("client", message)
-#define LOG_CLIENT_ERROR(message) LOG_ERROR("client", message)
+#include "librats_log_macros.h"
 
 namespace librats {
 
@@ -248,12 +244,13 @@ void RatsClient::get_torrent_metadata_from_peer(const InfoHash& info_hash,
         return;
     }
     
-    LOG_CLIENT_INFO("Fetching metadata from peer " << peer_ip << ":" << peer_port << " (fast path)");
+    LOG_CLIENT_INFO("Fetching metadata from peer " << peer_ip << ":" << peer_port << " (fast path, no DHT)");
     
     // Add magnet in metadata-only mode (empty save_path) and immediately add the peer
+    // skip_dht_search=true to avoid network-wide DHT queries - we only want this specific peer
     std::string hash_hex = info_hash_to_hex(info_hash);
     std::string magnet = "magnet:?xt=urn:btih:" + hash_hex;
-    auto torrent = bittorrent_client_->add_magnet(magnet, "");
+    auto torrent = bittorrent_client_->add_magnet(magnet, "", true /* skip_dht_search */);
     
     if (!torrent) {
         if (callback) {

package/native-src/src/librats_c.cpp

@@ -237,6 +237,18 @@ void rats_set_log_retention_count(rats_client_t handle, int count) {
     wrap->client->set_log_retention_count(count);
 }
 
+void rats_set_log_rotate_on_startup(rats_client_t handle, int enabled) {
+    if (!handle) return;
+    rats_client_wrapper* wrap = static_cast<rats_client_wrapper*>(handle);
+    wrap->client->set_log_rotate_on_startup(enabled != 0);
+}
+
+int rats_is_log_rotate_on_startup_enabled(rats_client_t handle) {
+    if (!handle) return 0;
+    rats_client_wrapper* wrap = static_cast<rats_client_wrapper*>(handle);
+    return wrap->client->is_log_rotate_on_startup_enabled() ? 1 : 0;
+}
+
 void rats_clear_log_file(rats_client_t handle) {
     if (!handle) return;
     rats_client_wrapper* wrap = static_cast<rats_client_wrapper*>(handle);
@@ -835,7 +847,7 @@ char* rats_get_peer_info_json(rats_client_t handle, const char* peer_id) {
     if (!handle || !peer_id) return nullptr;
     rats_client_wrapper* wrap = static_cast<rats_client_wrapper*>(handle);
     
-    const RatsPeer* peer = wrap->client->get_peer_by_id(std::string(peer_id));
+    auto peer = wrap->client->get_peer_by_id(std::string(peer_id));
     if (!peer) return nullptr;
     
     // Create JSON representation of peer info

package/native-src/src/librats_c.h

@@ -53,6 +53,8 @@ RATS_API void rats_set_log_timestamps_enabled(rats_client_t client, int enabled)
 RATS_API int rats_is_log_timestamps_enabled(rats_client_t client);
 RATS_API void rats_set_log_rotation_size(rats_client_t client, size_t max_size_bytes);
 RATS_API void rats_set_log_retention_count(rats_client_t client, int count);
+RATS_API void rats_set_log_rotate_on_startup(rats_client_t client, int enabled);
+RATS_API int rats_is_log_rotate_on_startup_enabled(rats_client_t client);
 RATS_API void rats_clear_log_file(rats_client_t client);
 
 // Error codes