adhd 0.0.1 → 0.1.0

data/lib/adhd/config.yml

@@ -3,5 +3,5 @@
     node_url: "192.168.1.104"
     couchdb_server_port: 5984
     buddy_server_url: "http://192.168.1.104:5984" # It is a good idea if this is a management node
-    buddy_server_name: foo
+    buddy_server_db_name: foo
 

data/lib/adhd/models/content_doc.rb

@@ -0,0 +1,17 @@
+class ContentDoc < CouchRest::ExtendedDocument
+  # NOTE: NO DEFAULT DATABASE IN THE OBJECT -- WE WILL BE STORING A LOT OF
+  # DATABASES OF THIS TYPE.
+
+
+  property :_id
+  property :internal_id
+  property :size_bytes
+  property :filename
+  property :mime_type
+
+  view_by :internal_id
+
+  # A special attachment "File" is expected to exist
+
+end
+

data/lib/adhd/models/content_shard.rb

@@ -0,0 +1,97 @@
+
+
+class ContentShard
+  attr_accessor :nodes, :this_shard, :our_node, :this_shard_db
+
+  def initialize(nodesv, this_shardv)
+    @nodes = nodesv
+    @this_shard = this_shardv
+
+    # Work out the rest
+    @our_node = nodesv.our_node
+    @this_shard_db = nodesv.our_node.get_content_db(this_shardv.shard_db_name)
+
+    @last_sync_seq = 0 # @this_shard_db.info['update_seq']
+  end
+
+  def in_shard?(internal_id)
+     internal_id >= this_shard.range_start && internal_id < this_shard.range_end
+  end
+
+  def write_doc(content_doc)
+    # Write a content document to this shard
+    # Make sure it is in this shard
+    if in_shard? content_doc.internal_id
+      this_shard_db.save_doc(content_doc)
+    end
+  end
+
+  def sync
+    # A Shard only pushes with the master of the shard
+    # or the node with the highest is_storage value alive
+    # Shard masters ensure changes are pushed to all
+
+    # NOTE: This method needs serious refactoring
+    # No need to update
+    return false if @this_shard_db.info['update_seq'] == @last_sync_seq
+
+    # Are we the shard master?
+    am_master = (our_node.name == this_shard.master_node)
+
+    if !am_master
+      master_node = Node.by_name(:key => this_shard.master_node).first
+      remote_db = master_node.get_content_db(this_shard.shard_db_name)
+      bool_to = @our_node.replicate_to(this_shard_db, master_node, remote_db)
+      if bool_to
+        @last_sync_seq = @this_shard_db.info['update_seq']
+        return true
+      end
+    end
+
+    # Either we are the master or the master has failed -- we replicate with
+    # all nodes or the first available aside us and master
+    all_good = true
+    this_shard.node_list.each do |node_name|
+       # Push all changes to the other nodes
+       remote_node = Node.by_name(:key =>  node_name).first
+       remote_db = remote_node.get_content_db(this_shard.shard_db_name)
+       all_good &= @our_node.replicate_to(this_shard_db, remote_node, remote_db)
+
+       if !am_master && bool_to
+         # NOTE: How to build skynet, Note 2
+         #       We are doing some "gonzo" replication, here. Our master is
+         #       clearly down so we find the second best node; we push our
+         #       changes to this node, and now also *replicate from*
+         #       that node.
+         @our_node.replicate_from(this_shard_db, remote_node, remote_db)
+         @last_sync_seq = @this_shard_db.info['update_seq']
+         break
+       end
+    end
+    if all_good
+      @last_sync_seq = @this_shard_db.info['update_seq']
+      return true
+    else
+      return false
+    end
+  end
+
+end
+
+class ContentDoc < CouchRest::ExtendedDocument
+  # NOTE: NO DEFAULT DATABASE IN THE OBJECT -- WE WILL BE STORING A LOT OF
+  # DATABASES OF THIS TYPE.
+
+
+  property :_id
+  property :internal_id
+  property :size_bytes
+  property :filename
+  property :mime_type
+
+  view_by :internal_id
+
+  # A special attachment "File" is expected to exist
+
+end
+

data/lib/adhd/models/node_doc.rb

@@ -0,0 +1,139 @@
+## Key Restrictions ok internal_IDs: must only contain [a-z0-9-]
+
+class NodeDB
+  
+  attr_accessor :local_node_db, :our_node
+  
+  def initialize(our_nodev)
+    @our_node = our_nodev
+    
+    # Get the address of the CDB from the node
+    @local_node_db = our_nodev.get_node_db
+  end
+  
+  def sync
+    # We replicate our state to the management node(s)
+    management_nodes = Node.by_is_management.reverse
+    # NOTE: randomize the order for load balancing here
+
+    # NOTE2: How to build skynet (TODO)
+    #        -------------------
+    #        If length of management is zero, then chose 3 different random
+    #        nodes at each sync, and sync with them in node_name order.
+    #        This guarantees that any updates on nodes are communicated in
+    #        O(log N) ephocs, at the cost of O(3 * N) connections per epoch.
+    #        It also guarantees any new management servers are discovered in
+    #        this O(log N) time, creating "jelly fish" or "partition proof"
+    #        availability. 
+    
+    management_nodes.each do |mng_node|
+      remote_db = mng_node.get_node_db
+      bool_from = @our_node.replicate_from(local_node_db, mng_node, remote_db)
+      bool_to = @our_node.replicate_to(local_node_db, mng_node, remote_db)
+      if bool_from && bool_to && !our_node.is_management
+         #puts "Pushed to management"
+         break
+      end       
+      #puts "Did not push to management"
+    end
+  end 
+
+  def available_node_list
+    # Returns all nodes marked as available
+    all_nodes = Node.by_name
+    return all_nodes.select {|node| node.status == "RUNNING"}
+  end
+
+  def head_management_node
+    management_nodes = Node.by_is_management.reverse
+    hmn = management_nodes.find {|node| node.status == "RUNNING"}
+    return hmn
+  end
+    
+end
+
+class Node  < CouchRest::ExtendedDocument
+  #NODESERVER = CouchRest.new("#{ARGV[1]}")
+  #NODESERVER.default_database = "#{ARGV[0]}_node_db"
+
+  #use_database NODESERVER.default_database
+
+  unique_id :name
+
+  property :name
+  property :url
+  property :is_store
+  property :is_management
+  property :is_directory
+  property :status
+
+  timestamps!
+
+  view_by :name
+  view_by :is_management
+
+  def get_node_db
+    server = CouchRest.new("#{url}")
+    db = server.database!("#{name}_node_db")
+    # puts "Open db #{db}"
+    db
+  end
+
+  def get_shard_db
+    server = CouchRest.new("#{url}")
+    db = server.database!("#{name}_shard_db")
+    # puts "Open db #{db}"
+    db
+  end  
+  
+  def get_content_db(shard_db_name)
+    server = CouchRest.new("#{url}")
+    db = server.database!("#{name}_#{shard_db_name}_content_db")
+    # puts "Open db #{db}"
+    db
+  end  
+
+  # Replicating databases and marking nodes as unavailable
+  # In the future we should hook these into a "replication manager"
+  # for databases. The manager should set up continuous replication across
+  # databases, and only do a replication after some time lapses.
+
+  def replicate_to(local_db, other_node, remote_db)
+    # Do not try to contact unavailable nodes
+    return false if other_node.status == "UNAVAILABLE"
+    # No point replicating to ourselves    
+    return false if (name == other_node.name)
+    
+    begin  
+      # Replicate to other node is possible
+      local_db.replicate_to(remote_db)
+      return true
+    rescue Exception => e
+      # Other node turns out to be unavailable
+      other_node.status = "UNAVAILABLE"
+      other_node.save
+      return false
+    end
+  end   
+  
+  def replicate_from(local_db, other_node, remote_db)
+    # Do not try to contact unavailable nodes
+    return false if other_node.status == "UNAVAILABLE"
+    # No point replicating to ourselves    
+    return false if (name == other_node.name)
+    
+    begin  
+      # Replicate to other node is possible
+      local_db.replicate_from(remote_db)
+      return true
+    rescue Exception => e
+      # Other node turns out to be unavailable
+      other_node.status = "UNAVAILABLE"
+      other_node.save
+      return false
+    end
+
+  end
+
+end
+

data/lib/adhd/models/shard_range.rb

@@ -0,0 +1,202 @@
+class ShardRangeDB
+
+  attr_accessor :nodes, :local_shard_db, :our_node  
+
+  def initialize(nodesv)
+    @nodes = nodesv
+    
+    # Automatically get our shard_db address from our own node name
+    @our_node = nodesv.our_node
+    @local_shard_db = nodesv.our_node.get_shard_db
+    
+    puts "Assign default database for shard ranges (#{@local_shard_db})"
+    ShardRange.use_database @local_shard_db
+  end
+  
+  def sync
+    # We replicate our state from the management node(s)
+    # We never push content if we are only storage
+    management_nodes = Node.by_is_management.reverse
+    
+    # NOTE: randomize the order for load balancing here    
+    
+    management_nodes.each do |mng_node|      
+      remote_db = mng_node.get_shard_db      
+      bool_from = @our_node.replicate_from(local_shard_db, mng_node, remote_db)
+      if our_node.is_management 
+        # Push any changes to other management nodes
+        bool_to = @our_node.replicate_to(local_shard_db, mng_node, remote_db)
+      end      
+      break if bool_from && !our_node.is_management
+    end
+  end 
+
+  def build_shards(number)
+    # Make a large list of possible id boundaries
+    characters = []
+    ("0".."9").each do |c|
+      characters << c
+    end
+    ("a".."f").each do |c|
+      characters << c
+    end
+    
+    # Generate 36 x 36 keys to choose boundaries from
+    all_keys = []
+    characters.each do |c1|
+      characters.each do |c2|
+        characters.each do |c3|
+          all_keys << (c1+c2+c3)
+        end
+      end
+    end
+    
+    # Now chose our boundaries
+    num_range_keys = all_keys.length
+    approx_shard_size = (num_range_keys * 1.0) / number
+    
+    shard_starts = []
+    (0...number).each do |n|
+      shard_starts << (all_keys[(n * approx_shard_size).floor])
+    end
+    
+    shard_ends = shard_starts.clone
+    shard_ends << ("z" * 3)
+    shard_ends.delete_at(0)
+    
+    # Finally build them!
+    puts "Build Shards"
+    (0...number).each do |n|
+      puts "Shard #{n}: from #{shard_starts[n]} to #{shard_ends[n]}"
+      shard_name = "sh_#{shard_starts[n]}_to_#{shard_ends[n]}"
+      sr = ShardRange.new
+      sr.range_start = shard_starts[n]
+      sr.range_end = shard_ends[n]
+      sr.shard_db_name = shard_name
+      sr.save
+    end    
+  end
+
+  def get_shard(internal_id)
+    # Finds the list of shards within which this ID lives
+    all_shards = ShardRange.by_range_start 
+    selected_shards = []
+    all_shards.each do |a_shard| # TODO: linear search is inefficient -- create a view 
+      if a_shard.range_start <= internal_id && a_shard.range_end > internal_id
+        selected_shards << a_shard
+      end
+    end
+    selected_shards
+  end
+  
+  def get_content_shards
+    # Return the content_shards of our node
+    content_shards = {}
+    ShardRange.by_node(:key => our_node.name).each do |s|
+      
+      # Build a content shard object
+      content_shards[s.shard_db_name] = ContentShard.new(nodes, s)
+    end        
+    puts "Content shards #{content_shards.length}"
+    content_shards
+  end
+  
+  def write_doc_directly(content_doc)
+    # Write a document directly to a nodes content repository
+    success = {:ok => false , :reason => "No available node found"}
+    doc_shard = get_shard(content_doc.internal_id).first
+    doc_shard.get_nodes.each do |node|
+      # Try to write the doc to this node
+      begin
+        remote_node = Node.by_name(:key => node).first
+        remote_ndb = NodeDB.new(remote_node)
+        remote_content_shard = ContentShard.new(remote_ndb, doc_shard)
+        remote_content_shard.this_shard_db.save_doc(content_doc)
+        success = {:ok => true, :doc => content_doc, :db => remote_content_shard.this_shard_db}
+        break
+      rescue RestClient::RequestFailed => rf
+        if rf.http_code == 409
+          puts "Document already there"
+          return {:ok => false , :reason => "Document already in database"}
+        end
+      rescue Exception =>e
+        puts "Could not put doc in node #{node} because of #{rf}"
+        # TODO: change status or chose another management server
+        remote_node.status = "UNAVAILABLE"
+        remote_node.save
+      end
+    end
+    return success
+  end
+  
+  def get_doc_directly(internal_id)
+    # Write a document directly to a nodes content repository
+    doc_shard = get_shard(internal_id).first
+    
+    # TODO: Randomize the order of nodes for load balancing in retrieval!
+    docx = []  
+    doc_shard.get_nodes.each do |node|
+      # Try to write the doc to this node
+      begin
+        remote_node = Node.by_name(:key => node).first
+        remote_ndb = NodeDB.new(remote_node)
+        remote_content_shard = ContentShard.new(remote_ndb, doc_shard)
+        
+        docx = ContentDoc.by_internal_id(:key => internal_id, :database => remote_content_shard.this_shard_db)
+        if docx.length > 0
+          return {:ok => true, :doc => docx.first, :db => remote_content_shard.this_shard_db }
+        end
+      rescue
+        puts "Could not put doc in node #{node.name}"
+        # TODO: change status or chose another management server
+        remote_node.status = "UNAVAILABLE"
+        remote_node.save      
+      end
+    end    
+  return {:ok => false }
+  end
+  
+end
+
+class ShardRange < CouchRest::ExtendedDocument
+  # SHARDSERVER = CouchRest.new("#{ARGV[1]}")
+  # SHARDSERVER.default_database = "#{ARGV[0]}_shard_db"
+  # use_database SHARDSERVER.default_database
+
+  property :range_start
+  property :range_end
+  property :node_list
+  property :master_node
+  property :shard_db_name
+
+  view_by :range_start 
+  
+  # View "node" - given a node returns the shards watched
+  # How to use this new
+  # 
+  # puts "Which Shards does 'node1' watch?"
+  # ShardRange.by_node(:key => "node1").each do |s|
+  #   puts "Shard: #{s.shard_db_name}"
+  # end
+
+   
+  view_by :node,
+          :map => 
+          "function(doc) {
+          if (doc['couchrest-type'] == 'ShardRange' && doc.node_list) {
+            doc.node_list.forEach(function(node){
+              emit(node, 1);
+            });
+          }
+        }"  
+
+  def get_nodes
+    # Return all nodes, with the master being first
+    all_nodes = node_list.clone
+    all_nodes.delete(master_node)
+    all_nodes = [master_node] + all_nodes
+    all_nodes
+  end
+
+end
+