librats 1.0.2 → 2.2.0

package/native-src/src/{storage.h → librats/storage/storage.h}

@@ -1,7 +1,40 @@
 #pragma once
 
-#include "socket.h"
-#include "json.hpp"
+/**
+ * @file storage.h
+ * @brief Distributed key-value store as a pluggable Node subsystem.
+ *
+ * A replicated key-value database with Last-Write-Wins (LWW) conflict
+ * resolution, typed values, binary on-disk persistence, and peer
+ * synchronization. It is a `Subsystem`: it reaches the mesh only through
+ * `PeerNetwork` (never the Node), exactly like PubSub/FileTransfer.
+ *
+ * Replication model — an epidemic LWW broadcast:
+ *   - A local put/remove builds a StorageEntry (a delete is a tombstone entry
+ *     with `deleted=true`) and broadcasts it to all connected peers.
+ *   - On receiving an entry, a node applies it under LWW. It re-forwards the
+ *     entry to its *other* peers ONLY if the entry actually won (carried new
+ *     information). A duplicate loses LWW and is not forwarded, so flooding
+ *     terminates naturally — no separate dedup table needed.
+ *   - On peer connect, both sides exchange a full snapshot (anti-entropy) so a
+ *     late joiner catches up. LWW makes the merge order-independent.
+ *
+ * Wire format (MessageType::Storage payload, opcode in byte 0):
+ *   ENTRY:         [1][StorageEntry.serialize()]
+ *   SYNC_REQUEST:  [2]
+ *   SYNC_RESPONSE: [3][count:u32][StorageEntry.serialize()] * count
+ *
+ * The class is also usable standalone (no network attached) as a local,
+ * persistent key-value store; all network operations no-op until attach().
+ */
+
+#include "librats/util/rats_export.h"
+#include "librats/node/peer_network.h"
+#include "librats/peer/peer.h"
+#include "librats/peer/peer_id.h"
+#include "librats/core/bytes.h"
+#include "librats/util/json.h"
+
 #include <string>
 #include <vector>
 #include <functional>
@@ -16,9 +49,6 @@
 
 namespace librats {
 
-// Forward declaration
-class RatsClient;
-
 /**
  * Value types supported by the distributed storage
  */
@@ -31,7 +61,7 @@ enum class StorageValueType : uint8_t {
 };
 
 /**
- * Storage operation types for synchronization
+ * Storage operation types for change events
  */
 enum class StorageOperation : uint8_t {
     OP_PUT = 0x01,         // Insert or update
@@ -51,42 +81,42 @@ enum class StorageSyncStatus {
 /**
  * Storage entry - represents a single key-value pair in the database
  */
-struct StorageEntry {
+struct RATS_API StorageEntry {
     std::string key;                    // Key string
     StorageValueType type;              // Value type
     std::vector<uint8_t> data;          // Serialized value data
     uint64_t timestamp_ms;              // Unix timestamp in milliseconds (for LWW)
-    std::string origin_peer_id;         // Peer that created/modified this entry
+    std::string origin_peer_id;         // Peer that created/modified this entry (hex PeerId)
     uint32_t checksum;                  // CRC32 checksum for integrity
     bool deleted;                       // Tombstone marker for deleted entries
-    
-    StorageEntry() 
-        : type(StorageValueType::BINARY), 
-          timestamp_ms(0), 
-          checksum(0), 
+
+    StorageEntry()
+        : type(StorageValueType::BINARY),
+          timestamp_ms(0),
+          checksum(0),
           deleted(false) {}
-    
-    StorageEntry(const std::string& k, StorageValueType t, 
-                 const std::vector<uint8_t>& d, uint64_t ts, 
+
+    StorageEntry(const std::string& k, StorageValueType t,
+                 const std::vector<uint8_t>& d, uint64_t ts,
                  const std::string& peer_id)
-        : key(k), type(t), data(d), timestamp_ms(ts), 
+        : key(k), type(t), data(d), timestamp_ms(ts),
           origin_peer_id(peer_id), checksum(0), deleted(false) {
         calculate_checksum();
     }
-    
+
     // Calculate CRC32 checksum
     void calculate_checksum();
-    
+
     // Verify checksum
     bool verify_checksum() const;
-    
+
     // Serialize entry to binary format
     std::vector<uint8_t> serialize() const;
-    
+
     // Deserialize entry from binary format
-    static bool deserialize(const std::vector<uint8_t>& data, size_t offset, 
+    static bool deserialize(const std::vector<uint8_t>& data, size_t offset,
                            StorageEntry& entry, size_t& bytes_read);
-    
+
     // Compare for LWW resolution (returns true if this entry wins)
     bool wins_over(const StorageEntry& other) const;
 };
@@ -111,19 +141,15 @@ struct StorageChangeEvent {
 struct StorageConfig {
     std::string data_directory;         // Directory for storage files
     std::string database_name;          // Database filename prefix
-    bool enable_compression;            // Enable LZ4 compression for values
     bool enable_sync;                   // Enable network synchronization
-    uint32_t sync_batch_size;           // Number of entries per sync batch
     uint32_t compaction_threshold;      // Number of tombstones before compaction
     uint32_t max_value_size;            // Maximum value size in bytes
     bool persist_to_disk;               // Whether to persist data to disk
-    
+
     StorageConfig()
         : data_directory("./storage"),
           database_name("rats_storage"),
-          enable_compression(false),
           enable_sync(true),
-          sync_batch_size(100),
           compaction_threshold(1000),
           max_value_size(16 * 1024 * 1024),  // 16MB max value size
           persist_to_disk(true) {}
@@ -152,351 +178,161 @@ using StorageChangeCallback = std::function<void(const StorageChangeEvent&)>;
 using StorageSyncCompleteCallback = std::function<void(bool success, const std::string& error_message)>;
 
 /**
- * StorageManager - Distributed key-value storage with peer synchronization
- * 
- * Features:
- * - Binary on-disk storage for efficiency
- * - In-memory index for O(1) key lookups
- * - Real-time synchronization via GossipSub
- * - Last-Write-Wins conflict resolution
- * - Full database sync for new peers
- * - Typed value support (string, int64, double, binary, JSON)
+ * StorageManager - Distributed key-value storage with peer synchronization.
+ *
+ * A Node subsystem: attach()/start()/stop() plug it into a Node, and it reaches
+ * the mesh only via PeerNetwork. It can also be used standalone as a local,
+ * persistent key-value store (network operations no-op until attached).
  */
-class StorageManager {
+class RATS_API StorageManager final : public Subsystem {
 public:
     /**
-     * Constructor
-     * @param client Reference to RatsClient for network communication
-     * @param config Storage configuration settings
+     * Constructor.
+     * @param config Storage configuration settings.
+     *
+     * Loads any existing data from disk and starts the background persistence
+     * thread immediately, so the store is usable for local reads/writes before
+     * (and without) being added to a Node.
      */
-    StorageManager(RatsClient& client, const StorageConfig& config = StorageConfig());
-    
+    explicit StorageManager(const StorageConfig& config = StorageConfig());
+
     /**
-     * Destructor - saves data and cleans up resources
+     * Destructor - saves data and cleans up resources.
      */
-    ~StorageManager();
-    
+    ~StorageManager() override;
+
+    StorageManager(const StorageManager&) = delete;
+    StorageManager& operator=(const StorageManager&) = delete;
+
+    // =========================================================================
+    // Subsystem
+    // =========================================================================
+
+    void attach(NodeContext& ctx) override;
+    void start() override;
+    void stop() override;
+
     // =========================================================================
     // Configuration
     // =========================================================================
-    
-    /**
-     * Update storage configuration
-     * @param config New configuration settings
-     */
+
     void set_config(const StorageConfig& config);
-    
-    /**
-     * Get current configuration
-     * @return Current configuration settings
-     */
     const StorageConfig& get_config() const;
-    
+
     // =========================================================================
     // Put Operations (Write)
     // =========================================================================
-    
-    /**
-     * Store a string value
-     * @param key Key to store under
-     * @param value String value to store
-     * @return true if stored successfully
-     */
+
     bool put(const std::string& key, const std::string& value);
-    
-    /**
-     * Store a 64-bit integer value
-     * @param key Key to store under
-     * @param value Integer value to store
-     * @return true if stored successfully
-     */
     bool put(const std::string& key, int64_t value);
-    
-    /**
-     * Store a double-precision floating point value
-     * @param key Key to store under
-     * @param value Double value to store
-     * @return true if stored successfully
-     */
     bool put(const std::string& key, double value);
-    
-    /**
-     * Store binary data
-     * @param key Key to store under
-     * @param value Binary data to store
-     * @return true if stored successfully
-     */
     bool put(const std::string& key, const std::vector<uint8_t>& value);
-    
-    /**
-     * Store a JSON document
-     * @param key Key to store under
-     * @param value JSON value to store
-     * @return true if stored successfully
-     */
-    bool put_json(const std::string& key, const nlohmann::json& value);
-    
+    bool put_json(const std::string& key, const librats::Json& value);
+
     // =========================================================================
     // Get Operations (Read)
     // =========================================================================
-    
-    /**
-     * Get a string value
-     * @param key Key to retrieve
-     * @return Optional containing value if found and type matches
-     */
+
     std::optional<std::string> get_string(const std::string& key) const;
-    
-    /**
-     * Get a 64-bit integer value
-     * @param key Key to retrieve
-     * @return Optional containing value if found and type matches
-     */
     std::optional<int64_t> get_int(const std::string& key) const;
-    
-    /**
-     * Get a double-precision floating point value
-     * @param key Key to retrieve
-     * @return Optional containing value if found and type matches
-     */
     std::optional<double> get_double(const std::string& key) const;
-    
-    /**
-     * Get binary data
-     * @param key Key to retrieve
-     * @return Optional containing value if found and type matches
-     */
     std::optional<std::vector<uint8_t>> get_binary(const std::string& key) const;
-    
-    /**
-     * Get a JSON document
-     * @param key Key to retrieve
-     * @return Optional containing value if found and type matches
-     */
-    std::optional<nlohmann::json> get_json(const std::string& key) const;
-    
-    /**
-     * Get raw entry (for advanced usage)
-     * @param key Key to retrieve
-     * @return Pointer to entry or nullptr if not found
-     */
-    const StorageEntry* get_entry(const std::string& key) const;
-    
-    /**
-     * Get the type of a stored value
-     * @param key Key to check
-     * @return Optional containing type if key exists
-     */
+    std::optional<librats::Json> get_json(const std::string& key) const;
     std::optional<StorageValueType> get_type(const std::string& key) const;
-    
+
     // =========================================================================
     // Delete and Query Operations
     // =========================================================================
-    
-    /**
-     * Delete a key
-     * @param key Key to delete
-     * @return true if key existed and was deleted
-     */
+
     bool remove(const std::string& key);
-    
-    /**
-     * Check if a key exists
-     * @param key Key to check
-     * @return true if key exists and is not deleted
-     */
     bool has(const std::string& key) const;
-    
-    /**
-     * Get all keys
-     * @return Vector of all keys in storage
-     */
     std::vector<std::string> keys() const;
-    
-    /**
-     * Get keys matching a prefix
-     * @param prefix Prefix to match
-     * @return Vector of matching keys
-     */
     std::vector<std::string> keys_with_prefix(const std::string& prefix) const;
-    
-    /**
-     * Get the number of stored entries (excluding tombstones)
-     * @return Number of active entries
-     */
     size_t size() const;
-    
-    /**
-     * Check if storage is empty
-     * @return true if no active entries exist
-     */
     bool empty() const;
-    
-    /**
-     * Clear all entries
-     */
     void clear();
-    
+
     // =========================================================================
     // Persistence Operations
     // =========================================================================
-    
-    /**
-     * Save storage to disk
-     * @return true if saved successfully
-     */
+
     bool save();
-    
-    /**
-     * Load storage from disk
-     * @return true if loaded successfully
-     */
     bool load();
-    
-    /**
-     * Compact storage (remove tombstones)
-     * @return Number of tombstones removed
-     */
     size_t compact();
-    
+
     // =========================================================================
     // Synchronization Operations
     // =========================================================================
-    
-    /**
-     * Request full sync from connected peers
-     * @return true if sync request was sent
-     */
+
+    /// Request a full snapshot from one connected peer.
     bool request_sync();
-    
-    /**
-     * Get current sync status
-     * @return Current synchronization status
-     */
     StorageSyncStatus get_sync_status() const;
-    
-    /**
-     * Check if initial sync is complete
-     * @return true if at least one full sync has completed
-     */
     bool is_synced() const;
-    
+
     // =========================================================================
     // Event Callbacks
     // =========================================================================
-    
-    /**
-     * Set callback for storage changes
-     * @param callback Function to call when storage changes
-     */
+
     void set_change_callback(StorageChangeCallback callback);
-    
-    /**
-     * Set callback for sync completion
-     * @param callback Function to call when sync completes
-     */
     void set_sync_complete_callback(StorageSyncCompleteCallback callback);
-    
+
     // =========================================================================
-    // Statistics and Debugging
+    // Statistics
     // =========================================================================
-    
-    /**
-     * Get storage statistics
-     * @return Statistics structure
-     */
+
     StorageStatistics get_statistics() const;
-    
-    /**
-     * Get statistics as JSON
-     * @return JSON object with statistics
-     */
-    nlohmann::json get_statistics_json() const;
-    
-    // =========================================================================
-    // Internal Network Message Handlers (called by RatsClient)
-    // =========================================================================
-    
-    /**
-     * Handle storage-related GossipSub message
-     * @param peer_id Source peer ID
-     * @param topic Topic name
-     * @param message Message content
-     */
-    void handle_gossip_message(const std::string& peer_id, const std::string& topic, 
-                               const std::string& message);
-    
-    /**
-     * Handle storage sync request from peer
-     * @param peer_id Requesting peer ID
-     * @param data Request data
-     */
-    void handle_sync_request(const std::string& peer_id, const nlohmann::json& data);
-    
-    /**
-     * Handle storage sync response from peer
-     * @param peer_id Responding peer ID
-     * @param data Response data containing entries
-     */
-    void handle_sync_response(const std::string& peer_id, const nlohmann::json& data);
-    
-    /**
-     * Called when a new peer connects (to trigger sync if needed)
-     * @param peer_id Connected peer ID
-     */
-    void on_peer_connected(const std::string& peer_id);
-    
+    librats::Json get_statistics_json() const;
+
 private:
-    RatsClient& client_;
+    // Network message handlers (run on a reactor thread).
+    void on_storage_message(const PeerId& from, ByteView payload);
+    void on_peer_connected(const PeerId& peer_id);
+
+    PeerNetwork* network_ = nullptr;
     StorageConfig config_;
-    
+
     // In-memory storage
     mutable std::mutex storage_mutex_;
     std::unordered_map<std::string, StorageEntry> entries_;
-    
+
     // Sync state
     mutable std::mutex sync_mutex_;
     StorageSyncStatus sync_status_;
     bool initial_sync_complete_;
-    std::string sync_peer_id_;  // Peer we're syncing from
     std::chrono::steady_clock::time_point last_sync_time_;
-    
+
     // Statistics
     mutable std::mutex stats_mutex_;
     StorageStatistics stats_;
-    
+
     // Callbacks
     StorageChangeCallback change_callback_;
     StorageSyncCompleteCallback sync_complete_callback_;
-    
-    // Background threads
+
+    // Background persistence thread
     std::atomic<bool> running_;
     std::thread persistence_thread_;
     std::condition_variable persistence_cv_;
     std::mutex persistence_mutex_;
     bool dirty_;  // Flag indicating unsaved changes
-    
-    // GossipSub topic for real-time sync
-    static constexpr const char* STORAGE_GOSSIP_TOPIC = "_rats_storage";
-    
-    // Message type identifiers
-    static constexpr const char* MSG_TYPE_PUT = "storage_put";
-    static constexpr const char* MSG_TYPE_DELETE = "storage_delete";
-    static constexpr const char* MSG_TYPE_SYNC_REQUEST = "storage_sync_request";
-    static constexpr const char* MSG_TYPE_SYNC_RESPONSE = "storage_sync_response";
-    
+
+    // Wire opcodes (MessageType::Storage payload, byte 0)
+    static constexpr uint8_t OP_ENTRY         = 1;
+    static constexpr uint8_t OP_SYNC_REQUEST  = 2;
+    static constexpr uint8_t OP_SYNC_RESPONSE = 3;
+
     // Private methods
     void initialize();
     void shutdown();
     void persistence_thread_loop();
-    
+
     // Internal put with full control
-    bool put_internal(const std::string& key, StorageValueType type, 
-                     const std::vector<uint8_t>& data, 
+    bool put_internal(const std::string& key, StorageValueType type,
+                     const std::vector<uint8_t>& data,
                      uint64_t timestamp_ms = 0,
                      const std::string& origin_peer_id = "",
                      bool broadcast = true);
-    
+
     // Serialization helpers
     std::vector<uint8_t> serialize_value(int64_t value) const;
     std::vector<uint8_t> serialize_value(double value) const;
@@ -504,24 +340,24 @@ private:
     int64_t deserialize_int64(const std::vector<uint8_t>& data) const;
     double deserialize_double(const std::vector<uint8_t>& data) const;
     std::string deserialize_string(const std::vector<uint8_t>& data) const;
-    
+
     // Network operations
-    void broadcast_put(const StorageEntry& entry);
-    void broadcast_delete(const std::string& key, uint64_t timestamp_ms);
-    void send_sync_request(const std::string& peer_id);
-    void send_sync_response(const std::string& peer_id);
-    
-    // Apply remote changes with LWW resolution
-    bool apply_remote_entry(const StorageEntry& entry);
-    
+    void broadcast_entry(const StorageEntry& entry);              ///< to all peers
+    void forward_entry(const StorageEntry& entry, const PeerId& except);  ///< re-flood
+    void send_sync_request(const PeerId& peer_id);
+    void send_sync_response(const PeerId& peer_id);
+
+    // Apply a remote entry with LWW; fills `out_event` and returns true if applied.
+    bool apply_remote_entry(const StorageEntry& entry, StorageChangeEvent* out_event);
+
     // File path helpers
     std::string get_data_file_path() const;
     std::string get_index_file_path() const;
-    
+
     // Disk I/O
     bool write_data_file();
     bool read_data_file();
-    
+
     // Utility
     uint64_t get_current_timestamp_ms() const;
     std::string get_our_peer_id() const;