adhd 0.0.1 → 0.1.0

data/lib/adhd/node_manager.rb

@@ -0,0 +1,260 @@
+require 'rubygems'
+require 'couchrest'
+require 'ruby-debug'
+# require File.dirname(__FILE__) + '/models'
+require File.dirname(__FILE__) + '/models/node_doc'
+require File.dirname(__FILE__) + '/models/content_doc'
+require File.dirname(__FILE__) + '/models/shard_range'
+require File.dirname(__FILE__) + '/models/content_shard'
+
+module Adhd
+  class NodeManager
+
+    attr_accessor :our_node, :ndb, :srdb
+
+    def initialize(config)
+      @config = config
+      @couch_server = CouchRest.new("http://#{config.node_url}:#{config.couchdb_server_port}")
+      # @couch_server.default_database = "#{config.node_name}_node_db"
+      @couch_db = @couch_server.database!("#{config.node_name}_node_db") # CouchRest::Database.new(@couch_server, "#{config.node_name}_node_db")
+      sync_with_buddy_node if config.buddy_server_url && config.buddy_server_db_name
+      @our_node = initialize_node
+      build_node_admin_databases
+      set_as_management_node_if_necessary
+      build_shards(10, 2)
+      build_node_content_databases
+      sync_databases
+    end
+
+    def event_handler(ev)
+      puts ev
+    end
+
+    # Sync the db with our buddy
+    #
+    def sync_with_buddy_node
+      begin
+        buddy_server = CouchRest.new("#{@config.buddy_server_url}")
+        buddy_db = buddy_server.database!(@config.buddy_server_db_name + "_node_db")
+        @couch_db.replicate_from(buddy_db)
+      rescue
+        puts "Could not buddy up with node #{@config.buddy_server_db_name}"
+      end
+    end
+
+    # Retrieve our own node record from CouchDB by our name.
+    #
+    # If there are other nodes with the name kill their records!
+    #
+    def initialize_node
+      puts "Initialize node #{@config.node_name}"
+      Node.use_database @couch_db
+      puts "DB #{@couch_db}, node #{@config.node_name}"
+      node_candidates = Node.by_name(:key => @config.node_name)
+      # node_candidates = @couch_db.view("by_name", {:key => @config.node_name})
+      node = node_candidates.pop
+      node = Node.new if node.nil?
+      node_candidates.each do |other_me|
+        other_me.destroy # destroy other records
+      end
+      # Update our very own record
+      node.name = @config.node_name
+      node.url = "http://#{@config.node_url}:#{@config.couchdb_server_port}"
+      node.status = "RUNNING"
+      node.save
+      node # Save our node as instance variable
+    end
+
+    # We check if we are the first node. If we are the first node, we set
+    # ourself up as the management node.
+    #
+    def set_as_management_node_if_necessary
+      all_nodes = Node.by_name
+      if all_nodes.length == 1
+        @our_node.is_management = 300
+        @our_node.save
+      end
+    end
+
+    def build_node_admin_databases
+      @conn_manager = ConnectionBank.new
+
+      # Lets build a nice NodeDB
+      @ndb = NodeDB.new(@our_node)
+      conn_node = UpdateNotifierConnection.new(@config.node_url,
+                                        @config.couchdb_server_port,
+                                        @our_node.name + "_node_db", # NOTE: Sooo ugly!
+                                        Proc.new {|data| handle_node_update data})
+      @conn_manager.add_connection(conn_node)
+
+
+      # Lets build a nice ShardDB
+      @srdb = ShardRangeDB.new(@ndb)
+
+      # Listen to the shard db and in case something changes re-build the DB
+      # Chenges to the shards should be in-frequent and tolerable
+      conn_shard = UpdateNotifierConnection.new(@config.node_url,
+                                        @config.couchdb_server_port,
+                                        @our_node.name + "_shard_db", # NOTE: Sooo ugly!
+                                        Proc.new {|data| build_node_content_databases})
+      @conn_manager.add_connection(conn_shard)
+
+    end
+
+    def handle_node_update update
+      # Added, removed or changed the status of a node
+      # If we are the admin, when a node joins we should allocate to it
+      # some shards.
+
+      # Only the head management node deals with node changes
+      return if @ndb.head_management_node && ! (@ndb.head_management_node.name == @our_node.name)
+
+      # Given the shard_db and the node_db we should work out a new allocation
+      node_list = Node.by_name
+      shard_list = ShardRange.by_range_start
+      if node_list && shard_list
+        assign_nodes_to_shards(node_list, shard_list, 2)
+      end
+    end
+
+
+    def build_node_content_databases
+      # Get all content shard databases
+      # NOTE: we will have to refresh those then we are re-assigned shards
+      @contentdbs = {} if !@contentdbs
+      current_shards = @srdb.get_content_shards
+
+      # Add the new shards
+      current_shards.each_key do |cs|
+        if !(@contentdbs.has_key?(cs)) # Make sure we do not know of this shard
+          shard_db = current_shards[cs]
+          conn = UpdateNotifierConnection.new(@config.node_url,
+                                          @config.couchdb_server_port,
+                                          @our_node.name + "_" + shard_db.this_shard.shard_db_name + "_content_db", # NOTE: Sooo ugly!
+                                          Proc.new { |data| shard_db.sync })
+          @conn_manager.add_connection(conn)
+
+          # Store both the shard object and the update notifier
+          @contentdbs[cs] = [shard_db, conn]
+        end
+      end
+
+      # Delete what we do not need
+      @contentdbs.each_key do |cs|
+        if !(current_shards.has_key?(cs))
+          # Delete this shard from our DB
+          remove_content_shard @contentdbs[cs][0], @contentdbs[cs][1]
+          # Remove that key
+          @contentdbs.delete cs
+        end
+      end
+    end
+
+    def remove_content_shard content_shard, connection
+      # Kill the connection listening for updates on this shard
+      connection.kill
+      content_shard.sync
+      # TODO: test if the sync happened
+      # content_shard.this_shard_db.delete!
+      # TODO: run a sync with the current master to ensure that
+      #       any changes have been pushed. The DELETE the database
+      #       to save space
+    end
+
+    def run
+      # Enters the event machine loop
+      @conn_manager.run_all
+    end
+
+    def build_shards(number_of_shards, number_of_replicators)
+      if @our_node.is_management
+
+        if ShardRange.by_range_start.length == 0
+          puts "Creating new ranges"
+          @srdb.build_shards(number_of_shards)
+        end
+
+        # Populate the shards with some nodes at random
+        node_names = []
+        all_nodes = Node.by_name
+        all_nodes.each do |anode|
+          node_names << anode.name
+        end
+
+        ShardRange.by_range_start.each do |s|
+          if !s.node_list or s.node_list.length == 0
+            node_names.shuffle!
+            s.node_list = node_names[0..(number_of_replicators-1)]
+            s.master_node = node_names[0]
+            s.save
+          end
+        end
+      end
+    end
+
+    def sync_databases
+      @ndb.sync # SYNC
+      @srdb.sync # SYNC
+
+      @contentdbs.each_key do |cs|
+        @contentdbs[cs][0].sync
+      end
+    end
+
+  def sync_admin
+      @ndb.sync # SYNC
+      @srdb.sync # SYNC
+  end
+
+  end
+end
+
+# -------- Management node logic -------------------
+
+require 'md5'
+
+def assign_nodes_to_shards(node_list, shard_range_list, replication_factor)
+  # This is an automatic way to allocate shards to nodes that just
+  # arrive in the networks, as well as re-allocate shards if nodes
+  # become unavailable or leave the network.
+
+  # NOTE: How to build skynet (Part III)
+  #
+  #       The invarient we try to impost on the list of nodes part of a shard
+  #       is that there should be at least replication_factor available nodes
+  #       in it. At the same time we try to keep the list stable over nodes
+  #       joining and leaving. To achieve this we hash in sequence the name of
+  #       each node with the name of the shard. We sort this list by hash, and
+  #       choose the first n nodes such that at least replication_factor nodes
+  #       are available. Then we chose the first available node as the master
+  #       for that shard.
+
+  shard_range_list.each do |shard_range|
+    # Sort all nodes using consistent hashing
+    sorted_nodes = node_list.sort_by {|node| MD5.new("#{node.name}||#{shard_range.shard_db_name}").to_s}
+    avail = 0
+    master = nil
+    shard_node_list = []
+    sorted_nodes.each do |node|
+      shard_node_list << node
+      if node.status == "RUNNING"
+        master = node if !master # Chose the first available to be the master
+        avail += 1
+        break if avail == replication_factor # We have enough available nodes
+      end
+    end
+
+    # Now put this list in the shard_range and save it
+    # but only if there were changes
+    new_master = master.name if master
+    new_node_list = shard_node_list.map {|node| node.name}
+
+    if !(new_master == shard_range.master_node) or !(new_node_list == shard_range.node_list)
+      shard_range.master_node = master.name if master
+      shard_range.node_list = shard_node_list.map {|node| node.name}
+      shard_range.save
+    end
+  end
+
+end
+

data/lib/adhd/reactor.rb

@@ -4,7 +4,7 @@ require 'eventmachine'
 module  Adhd
 
   # This module gets mixed into the EventMachine reactor loop, and sends events
-  # to our Adhd::Node which is hanging around waiting for shit to happen.
+  # to our Adhd::Node which is hanging around waiting for stuff to happen.
   #
   # Currently it just illustrates the fact that we can fire up an EventMachine
   # and have this reactor get mixed in as glue code to our Node.
@@ -29,29 +29,211 @@ module  Adhd
 
   end
 
-  # A reactor that makes a long-running request to a CouchDB instance (using
-  # Comet technology) and continually sends any update notifications that it
-  # gets back to its @node.
+  # A notifier client that makes a long-running request to a CouchDB instance
+  # on a socket and continually sends any update notifications that it
+  # receives back to its @node.
   #
-  module DbUpdateReactor
+  module DbUpdateNotifier
 
-    def initialize(node)
-      puts "Db update reactor start..."
-      @node = node
+    def initialize(db_name, conn_obj)
+      puts "Db update notifier start..."
+      @db_name = db_name
+      @conn_obj = conn_obj
+      @buffer = ""
+      conn_obj.connection_inside = self # We tell the outer object who we are
     end
 
     # Makes a long-running request to a CouchDB instance's _changes URL.
     #
     def post_init
-      send_data "GET http://192.168.1.104:5984/bar_node_db/_changes?feed=continuous&heartbeat=5000\r\n\r\n"
+      # NOTE: full domain name needed for virtual hosting
+      req = "GET #{@conn_obj.base_url}/#{@db_name}/_changes?feed=continuous&heartbeat=5000\r\n\r\n"
+      puts req
+      send_data req
     end
 
-    # Shoots update notifications from CouchDB to the @node.
+    # Shoots update notifications from CouchDB to the @conn.
     #
-    def receive_data data
-      @node.event_handler(data)
+    def receive_data data      
+      # puts "received_data: #{data}"
+      # puts "||#{data}||length=#{data.length}||#{data.dump}||"
+      
+      @buffer += data # Add the data to the current buffer
+      updates = []
+      if @buffer =~ /(\{[^\n]+\}\n)/
+        updates += $~.to_a
+        # Trim the buffer to $_.end(0)
+        @buffer = @buffer[$~.end(0)..-1]
+      end
+      
+      # Regexp for JSON updates is /\{[\n]\}+/
+      updates.each do |json_event|
+        @conn_obj.event_handler(json_event) unless data == "\n"
+      end
+    end
+
+    #def close_connection
+    #  @conn_obj.close_handler(data)
+    #end
+
+  end
+
+
+  # Note: Some of manos's thoughts on how to manage our connections and events.
+  #       We should build a class called connection_manager that we ask to build
+  #       and listen to connections, as well as route events. Its job is to
+  #       re-open them if they are closed or times out, mark nodes as UNAVAILABLE
+  #       and notify us back when data (i.e. an update) arrives. It would also be
+  #       nifty if each connection was associated with a predicate: once this
+  #       predicate is false we can simply close the connection. For example upon
+  #       being given control of a different content shard, or a different master
+  #       for the shard.
+
+  # In practice we will have two types of connections: Replicate and Notify.
+
+  class UpdateNotifierConnection
+    attr_accessor :db_name, :base_url, :connection_inside, :name
+
+    def initialize(node_url, couchdb_server_port, db_name, sync_block)
+      @node_url = node_url
+      @couchdb_server_port = couchdb_server_port
+      @db_name = db_name
+      @sync_block = sync_block
+      @status = "NOTRUNNING"
+      @base_url = "http://#{@node_url}:#{@couchdb_server_port}"
+      @name = @base_url +"/"+ @db_name
+      @keep_alive = true      
+    end
+    
+    def kill
+      @keep_alive = false
+    end
+
+    def start
+      puts "Register the connection for #{@db_name}"
+      EM.connect @node_url, @couchdb_server_port, Adhd::DbUpdateNotifier, @db_name, self
+      @status = "RUNNING"
+    end
+
+    def event_handler data
+      # puts "||#{data}||nn"
+      puts "Run a crazy sync on db #{@db_name}"
+      #@db_obj_for_sync.sync
+      @sync_block.call(data)
+    end
+
+    def close_handler
+      puts "Closed abnormally #{reason}"
+      @status = "NOTRUNNING"
+    end
+
+    def down_for_good(reason)
+      if reason
+        puts "Closed for good #{reason}"
+      end
+    end
+
+    def keep_alive?
+      # Returns the truth value of the predicate
+      @keep_alive
+    end
+    
+    def keep_alive_or_kill!
+      if ! keep_alive?
+        # Schedule this connection for close
+        connection_inside.close_connection_after_writing
+        @status = "NOTRUNNING"
+      end
+    end
+
+    def should_start?
+      !(@status == "RUNNING")
+    end
+
+    def is_closed?
+      (@status == "NOTRUNNING")
     end
 
+
   end
+
+  class Connection
+    #def on_teardown(&block)
+    #  # Set the handler to be called then a connection is dead
+    #  block(self) # Run the teardown handler
+    #end
+    
+    def initialize
+    
+    end
+
+    def should_start?
+      !(@status == "RUNNING")
+    end
+
+    def is_closed?
+      (@status == "NOTRUNNING")
+    end
+
+  end
+
+  class ConnectionBank
+    # Manage a bunch of connections for us
+    def initialize
+      @our_connections = []
+    end
+
+    def add_connection(conn)
+      # Make sure we have no duplicates
+      @our_connections.each do |c|
+        if conn.name == c.name
+          return
+        end
+      end      
+    
+      # If it is happy to run, add it to the list and start it!      
+      if conn.keep_alive?
+          @our_connections << conn
+          # Register the teardown handler for when the end comes...
+          # conn.on_teardown(|c| { rerun(c) })
+      end
+    end
+
+    def rerun(conn)
+      # When a connection is down, we check to see if it wants to be kept
+      # alive, and restart it otherwise we remove it from the list.
+      if conn.keep_alive?
+        begin
+          conn.start
+        rescue Exception => e
+          conn.down_for_good(e)
+        end
+      else
+        # It seems we have died of natural causes
+        # XXX: is it true that Ruby does not throw and exception for EOF?
+        #      Otherwise we will never see this
+        conn.keep_alive_or_kill!
+        @our_connections.delete(conn)
+        conn.down_for_good(nil)
+      end
+
+    end
+
+    def run_all
+      # Go through all connections and run them all
+      # Run within EM.run loop
+      # puts "Connection bank runs all... (#{@our_connections.length} connections)"
+      @our_connections.each do |c|
+        if c.is_closed? or !c.keep_alive?
+          puts "Actually rerun #{c.db_name}..."
+          
+          rerun(c)
+        end
+      end
+
+    end
+
+  end
+
 end