adhd 0.0.0 → 0.0.1

data/Rakefile

@@ -14,7 +14,7 @@ begin
     gem.add_development_dependency "ruby-debug", ">= 0.10.3"
     gem.add_dependency "sinatra", ">= 0.9.4"
     gem.add_dependency "couchrest", ">= 0.33"
-
+    gem.add_dependency "thin", ">= 1.2.4"
     # gem is a Gem::Specification... see http://www.rubygems.org/read/chapter/20 for additional settings
   end
   Jeweler::GemcutterTasks.new

data/VERSION

@@ -1 +1 @@
-0.0.0
+0.0.1

data/adhd.gemspec

@@ -5,13 +5,15 @@
 
 Gem::Specification.new do |s|
   s.name = %q{adhd}
-  s.version = "0.0.0"
+  s.version = "0.0.1"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["dave.hrycyszyn@headlondon.com"]
-  s.date = %q{2009-12-18}
+  s.date = %q{2009-12-19}
+  s.default_executable = %q{adhd}
   s.description = %q{More to say when something works! Do not bother installing this! }
   s.email = %q{dave.hrycyszyn@headlondon.com}
+  s.executables = ["adhd"]
   s.extra_rdoc_files = [
     "LICENSE",
      "README.rdoc"
@@ -24,9 +26,33 @@ Gem::Specification.new do |s|
      "Rakefile",
      "VERSION",
      "adhd.gemspec",
+     "bin/adhd",
      "doc/adhd.xmi",
      "lib/adhd.rb",
+     "lib/adhd/config.yml",
      "lib/adhd/models.rb",
+     "lib/adhd/node.rb",
+     "lib/adhd/reactor.rb",
+     "lib/ext/hash_to_openstruct.rb",
+     "lib/public/images/img01.jpg",
+     "lib/public/images/img02.jpg",
+     "lib/public/images/img03.jpg",
+     "lib/public/images/img04.jpg",
+     "lib/public/images/img05.jpg",
+     "lib/public/images/img06.jpg",
+     "lib/public/images/img07.jpg",
+     "lib/public/images/img08.jpg",
+     "lib/public/images/img09.jpg",
+     "lib/public/images/img10.gif",
+     "lib/public/images/img11.gif",
+     "lib/public/images/img12.jpg",
+     "lib/public/images/img13.jpg",
+     "lib/public/images/img14.jpg",
+     "lib/public/images/img15.jpg",
+     "lib/public/images/spacer.gif",
+     "lib/public/style.css",
+     "lib/views/index.erb",
+     "lib/views/layout.erb",
      "models.rb",
      "test/helper.rb",
      "test/test_adhd.rb"
@@ -50,17 +76,20 @@ Gem::Specification.new do |s|
       s.add_development_dependency(%q<ruby-debug>, [">= 0.10.3"])
       s.add_runtime_dependency(%q<sinatra>, [">= 0.9.4"])
       s.add_runtime_dependency(%q<couchrest>, [">= 0.33"])
+      s.add_runtime_dependency(%q<thin>, [">= 1.2.4"])
     else
       s.add_dependency(%q<thoughtbot-shoulda>, [">= 0"])
       s.add_dependency(%q<ruby-debug>, [">= 0.10.3"])
       s.add_dependency(%q<sinatra>, [">= 0.9.4"])
       s.add_dependency(%q<couchrest>, [">= 0.33"])
+      s.add_dependency(%q<thin>, [">= 1.2.4"])
     end
   else
     s.add_dependency(%q<thoughtbot-shoulda>, [">= 0"])
     s.add_dependency(%q<ruby-debug>, [">= 0.10.3"])
     s.add_dependency(%q<sinatra>, [">= 0.9.4"])
     s.add_dependency(%q<couchrest>, [">= 0.33"])
+    s.add_dependency(%q<thin>, [">= 1.2.4"])
   end
 end
 

data/bin/adhd

@@ -0,0 +1,33 @@
+#!/usr/bin/env ruby
+require File.dirname(__FILE__) + '/../lib/ext/hash_to_openstruct'
+require File.dirname(__FILE__) + '/../lib/adhd/node'
+require File.dirname(__FILE__) + '/../lib/adhd/reactor'
+
+require 'optparse'
+require 'ftools'
+require 'yaml'
+require 'socket'
+
+def parse_config(file)
+  @config = YAML.load_openstruct(File.read(file))
+end
+
+@command = ARGV.shift
+
+options = {}
+
+opts = OptionParser.new do |opts|
+  opts.on("-C", "--config C", "YAML config file") do |n|
+    parse_config(n)
+  end
+end
+
+opts.parse! ARGV
+
+@node = Adhd::Node.new(@config)
+
+EM.run {
+  puts "Starting EventMachine reactor loop..."
+  EM.connect @config.node_url, @config.couchdb_server_port, Adhd::DbUpdateReactor, @node
+}
+

data/lib/adhd.rb

@@ -5,10 +5,28 @@ require 'erb'
 require 'ruby-debug'
 require File.dirname(__FILE__) + '/adhd/models'
 
-node_name = ARGV[0]
-node_url = ARGV[1]
-buddy_server_url = ARGV[2]
-buddy_db =  ARGV[3]
+# Start the server for now by cd'ing into the /lib directory and running the
+# following command:
+#
+# (first node):
+# ruby adhd.rb <node_name> <couchdb_server_url>
+#
+# (second or later node)
+# ruby adhd.rb <node_name> <couchdb_server_url> <management_node_url> <management_node_db> -p <port_number>
+#
+# <node_name> is just a string, e.g. "foo".
+# <couchdb_server_url>: the url (including port) for this node's couchdb server
+#   instance, e.g, http://192.168.1.104:5984
+# <management_node_url>: the url of the management node where this node should
+#   initially replicate from, e.g. http://192.168.1.104:5984
+# <management_node_db>: the couchdb management node database, e.g. "bar_node_db"
+# <port_number>: a port number to run on. If you're running more than one node locally
+#   for development purposes you'll need to pick a non-default port higher than 1024.
+
+node_name = ARGV[1]
+node_url = ARGV[2]
+buddy_server_url = ARGV[3]
+buddy_db =  ARGV[4]
 
 NODESERVER = CouchRest.new("#{node_url}")
 NODESERVER.default_database = "#{node_name}_node_db"
@@ -17,26 +35,86 @@ node_db = CouchRest::Database.new(NODESERVER, "#{node_name}_node_db")
 # sync the db with our buddy
 if buddy_server_url && buddy_db
   buddy_server = CouchRest.new("#{buddy_server_url}")
-  buddy_db = CouchRest::Database.new(buddy_server, buddy_db)
+  buddy_db = CouchRest::Database.new(buddy_server, buddy_db + "_node_db")
   node_db.replicate_from(buddy_db)
 end
 
-node = Node.by_name(:key => node_name).first
+# Retrieve our own node by our name
+# If there are other nodes with the name kill their records!
+node_candidates = Node.by_name(:key => node_name)
+node = node_candidates.pop
 node = Node.new if node.nil?
+node_candidates.each do |other_me|
+  other_me.destroy # destroy other records
+end
 
+# Update our very own record
 node.name = node_name
 node.url = node_url
+node.status = "RUNNING"
 node.save
 
 # We check if we are the first node. If we are the first node, we set ourself up
-# as the management node. If not, we find out where the management node is and
-# we replicate to the administrative node.
-if management_node = Node.by_is_management.last
-  management_node_server = CouchRest.new(management_node.url)
-  management_node_db = CouchRest::Database.new(management_node_server, management_node.name + "_node_db")
-  node_db.replicate_to(management_node_db)
-else
+# as the management node.
+all_nodes = Node.by_name()
+if all_nodes.length == 1
+  # puts "Setup #{node.name} as management node"
   node.is_management = 3
   node.save
 end
 
+# Lets build a nice NodeDB
+ndb = NodeDB.new(node)
+
+# Lets build a nice ShardDB
+srdb = ShardRangeDB.new(ndb)
+
+# If there are no shards make a few, if we are managers
+#puts "Create new ranges?"
+#puts "How many shards: #{ShardRange.by_range_start.length}"
+#puts "in #{ShardRange::SHARDSERVER.default_database}"
+if ShardRange.by_range_start.length == 0 && node.is_management
+  puts "Creating new ranges"
+  srdb.build_shards(100)
+end
+
+# Polulate the shards with some nodes at random
+node_names = []
+all_nodes.each do |anode|
+  node_names << anode.name
+end
+
+ShardRange.by_range_start.each do |s|
+  if !s.node_list or s.node_list.length == 0
+    node_names.shuffle!
+    s.node_list = node_names[0..2]
+    s.master_node = node_names[0]
+    s.save
+  end
+
+end
+# Sync all the node databases
+
+ndb.sync # SYNC
+srdb.sync # SYNC
+
+srdb.get_content_shards.each do |content_shard_db|
+  content_shard_db.sync
+end
+
+get "/" do
+  @all_nodes = Node.by_name
+  erb :index
+end 
+
+get "/sync" do
+  # Sync the node database
+  ndb.sync
+  # Sync the shard database
+  srdb.sync
+
+  srdb.get_content_shards.each do |content_shard_db|
+    content_shard_db.sync
+  end
+end
+

data/lib/adhd/config.yml

@@ -0,0 +1,7 @@
+---
+    node_name: superfoo
+    node_url: "192.168.1.104"
+    couchdb_server_port: 5984
+    buddy_server_url: "http://192.168.1.104:5984" # It is a good idea if this is a management node
+    buddy_server_name: foo
+

data/lib/adhd/models.rb

@@ -1,3 +1,59 @@
+# Key Restrictions ok internal_IDs: must only contain [a-z0-9-]
+
+class Array
+  def shuffle!
+    size.downto(1) { |n| push delete_at(rand(n)) }
+    self
+  end
+end
+
+class NodeDB
+  
+  attr_accessor :local_node_db, :our_node
+  
+  def initialize(our_nodev)
+    @our_node = our_nodev
+    
+    # Get the address of the CDB from the node
+    @local_node_db = our_nodev.get_node_db
+  end
+  
+  def sync
+    # We replicate our state to the management node(s)
+    management_nodes = Node.by_is_management.reverse
+    # NOTE: randomize the order for load balancing here
+
+    # NOTE2: How to build skynet (TODO)
+    #        -------------------
+    #        If length of management is zero, then chose 3 different random
+    #        nodes at each sync, and sync with them in node_name order.
+    #        This guarantees that any updates on nodes are communicated in
+    #        O(log N) ephocs, at the cost of O(3 * N) connections per epoch.
+    #        It also guarantees any new management servers are discovered in
+    #        this O(log N) time, creating "jelly fish" or "partition proof"
+    #        availability. 
+    
+    management_nodes.each do |mng_node|
+      remote_db = mng_node.get_node_db
+      if !(mng_node.name == our_node.name)
+        begin 
+          puts "Sync NodeDB with #{mng_node.name}"
+          local_node_db.replicate_from(remote_db)
+          # TODO: Manage conflicts here
+          local_node_db.replicate_to(remote_db)
+          break if !our_node.is_management # Only need to contact one node
+        rescue
+          puts "Could not connect to DB node #{mng_node.name}"
+          # TODO: change status or chose another management server
+          mng_node.status = "UNAVAILABLE"
+          mng_node.save
+        end     
+      end
+    end
+  end 
+
+end
+
 class Node  < CouchRest::ExtendedDocument
   NODESERVER = CouchRest.new("#{ARGV[1]}")
   NODESERVER.default_database = "#{ARGV[0]}_node_db"
@@ -15,5 +71,318 @@ class Node  < CouchRest::ExtendedDocument
 
   view_by :name
   view_by :is_management
+
+  def get_node_db
+    server = CouchRest.new("#{url}")
+    server.database!("#{name}_node_db")
+  end
+
+  def get_shard_db
+    server = CouchRest.new("#{url}")
+    server.database!("#{name}_shard_db")
+  end  
+  
+  def get_content_db(shard_db_name)
+    server = CouchRest.new("#{url}")
+    server.database!("#{name}_#{shard_db_name}_content_db")
+  end  
+end
+
+class ShardRangeDB
+
+  attr_accessor :nodes, :local_shard_db, :our_node  
+
+  def initialize(nodesv)
+    @nodes = nodesv
+    
+    # Automatically get our shard_db address from our own node name
+    @our_node = nodesv.our_node
+    @local_shard_db = nodesv.our_node.get_shard_db
+  end
+  
+  def sync
+    # We replicate our state from the management node(s)
+    # We never push content if we are only storage
+    management_nodes = Node.by_is_management.reverse
+    # NOTE: randomize the order for load balancing here
+    
+    
+    management_nodes.each do |mng_node|      
+      remote_db = mng_node.get_shard_db
+      if !(mng_node.name == our_node.name)
+        begin 
+          puts "Sync ShardRange DB pull from #{mng_node.name}"
+          local_shard_db.replicate_from(remote_db)
+          # TODO: Manage conflicts here          
+          if our_node.is_management 
+            # Push any changes to other management nodes
+            puts "Sync ShardRange DB pushto #{mng_node.name}"
+            local_shard_db.replicate_to(remote_db)
+          else
+            break # sync with only one management server
+        end
+        rescue
+          puts "Could not connect to DB node #{mng_node.name}"
+          # TODO: change status or chose another management server
+          mng_node.status = "UNAVAILABLE"
+          mng_node.save
+        end     
+      end
+    end
+  end 
+
+  def build_shards(number)
+    # Make a large list of possible id boundaries
+    characters = []
+    ("0".."9").each do |c|
+      characters << c
+    end
+    ("a".."z").each do |c|
+      characters << c
+    end
+    
+    # Generate 36 x 36 keys to choose boundaries from
+    all_keys = []
+    characters.each do |c1|
+      characters.each do |c2|
+        all_keys << (c1+c2)
+      end
+    end
+    
+    # Now chose our boundaries
+    num_range_keys = all_keys.length
+    approx_shard_size = (num_range_keys * 1.0) / number
+    
+    shard_starts = []
+    (0...number).each do |n|
+      shard_starts << (all_keys[(n * approx_shard_size).floor])
+    end
+    
+    shard_ends = shard_starts.clone
+    shard_ends << ("z" * 100)
+    shard_ends.delete_at(0)
+    
+    # Finally build them!
+    puts "Build Shards"
+    (0...number).each do |n|
+      puts "Shard #{n}: from #{shard_starts[n]} to #{shard_ends[n]}"
+      shard_name = "sh_#{shard_starts[n]}_to_#{shard_ends[n]}"
+      sr = ShardRange.new
+      sr.range_start = shard_starts[n]
+      sr.range_end = shard_ends[n]
+      sr.shard_db_name = shard_name
+      sr.save
+    end    
+  end
+
+  def get_shard(internal_id)
+    # Finds the list of shards within which this ID lives
+    all_shards = ShardRange.by_range_start 
+    selected_shards = []
+    all_shards.each do |a_shard| # TODO: linear search is inefficient -- create a view 
+      if a_shard.range_start <= internal_id && a_shard.range_end > internal_id
+        selected_shards << a_shard
+      end
+    end
+    selected_shards
+  end
+  
+  def get_content_shards
+    # Return the content_shards of our node
+    content_shards = []
+    ShardRange.by_node(:key => "node1").each do |s|
+      
+      # Build a content shard object
+      cs = ContentShard.new
+      cs.our_node = our_node
+      cs.this_shard = s
+      cs.nodes = nodes
+      cs.this_shard_db = our_node.get_content_db(s.shard_db_name)
+      
+      # add it to the list
+      content_shards << cs
+    end
+    content_shards
+  end
+  
+  def write_doc_directly(content_doc)
+    # Write a document directly to a nodes content repository
+    doc_shard = get_shard(content_doc.internal_id).first
+    doc_shard.get_nodes.each do |node|
+      # Try to write the doc to this node
+      begin
+        remote_ndb = NodeDB.new(node)
+        remote_content_shard = ContentShard.new(remote_ndb, doc_shard)
+        remote_content_shard.this_shard_db.save_doc(content_doc)
+        break
+      rescue
+        puts "Could not put doc in node #{node.name}"
+        # TODO: change status or chose another management server
+        node.status = "UNAVAILABLE"
+        node.save
+      
+      end
+    end
+    
+  end
+  
+  def get_doc_directly(internal_id)
+    # Write a document directly to a nodes content repository
+    doc_shard = get_shard(internal_id).first
+    
+    # TODO: Randomize the order of nodes for load balancing in retrieval!
+    docx = []  
+    doc_shard.get_nodes.each do |node|
+      # Try to write the doc to this node
+      begin
+        remote_ndb = NodeDB.new(node)
+        remote_content_shard = ContentShard.new(remote_ndb, doc_shard)
+        
+        docx = ContentDoc.by_internal_id(:key => internal_id, :database => remote_content_shard.this_shard_db)
+        if docx.length > 0
+          break
+        end
+      rescue
+        puts "Could not put doc in node #{node.name}"
+        # TODO: change status or chose another management server
+        node.status = "UNAVAILABLE"
+        node.save      
+      end
+    end    
+  docx
+  end
+  
+end
+
+class ShardRange < CouchRest::ExtendedDocument
+  SHARDSERVER = CouchRest.new("#{ARGV[1]}")
+  SHARDSERVER.default_database = "#{ARGV[0]}_shard_db"
+
+  use_database SHARDSERVER.default_database
+
+  property :range_start
+  property :range_end
+  property :node_list
+  property :master_node
+  property :shard_db_name
+
+  view_by :range_start 
+  
+  # View "node" - given a node returns the shards watched
+  # How to use this new
+  # 
+  # puts "Which Shards does 'node1' watch?"
+  # ShardRange.by_node(:key => "node1").each do |s|
+  #   puts "Shard: #{s.shard_db_name}"
+  # end
+
+   
+  view_by :node,
+          :map => 
+          "function(doc) {
+          if (doc['couchrest-type'] == 'ShardRange' && doc.node_list) {
+            doc.node_list.forEach(function(node){
+              emit(node, 1);
+            });
+          }
+        }"  
+
+  def get_nodes
+    # Return all nodes, with the master being first
+    all_nodes = node_list
+    all_nodes.delete(master_node)
+    all_nodes = [master_node] + all_nodes
+    allnodes
+  end
+
 end
 
+
+class ContentShard
+  attr_accessor :nodes, :this_shard, :our_node, :this_shard_db  
+
+  def initialize(nodesv, this_shardv)
+    @nodes = nodesv
+    @this_shard = this_shardv
+    
+    # Work out the rest
+    @our_node = nodesv.our_node
+    @this_shard_db = nodesv.our_node.get_content_db(this_shardv.shard_db_name)
+  end
+
+  def in_shard?(internal_id)
+     internal_id >= this_shard.range_start && internal_id < this_shard.range_end
+  end
+
+  def write_doc(content_doc)
+    # Write a content document to this shard    
+    # Make sure it is in this shard
+    if in_shard? content_doc.internal_id
+      this_shard_db.save_doc(content_doc)
+    end
+  end
+  
+  def sync
+    # A Shard only pushes with the master of the shard
+    # or the node with the highest is_storage value alive
+    # Shard masters ensure changes are pushed to all    
+
+    # NOTE: This method needs serious refactoring
+
+    # Are we the shard master?
+    am_master = false
+    if our_node.name == this_shard.master_node
+      am_master = true      
+    end
+
+    if !am_master
+      begin
+        master_node = Nodes.by_name(this_shard.master_node).first
+        remotedb = MASTER_node.get_content_db(this_shard.shard_db_name)
+        this_shard_db.replicate_to(remote_db)
+        return # We sync-ed so job is done
+      rescue
+        # We flag the master as unavailable
+        if remote_node
+          master_node.status = "UNAVAILABLE"
+          master_node.save
+        end         
+      end
+    end
+    
+    # Either we are the master or the master has failed -- we replicate with 
+    # all nodes or the first available aside us and master
+    this_shard.node_list.each do |node_name|
+       if !(our_node.name == node_name) && !(this_shard.master_node == node_name)
+         begin
+           # Push all changes to the other nodes
+           remote_node = Nodes.by_name(node_name).first
+           remotedb = remote_node.get_content_db(this_shard.shard_db_name)
+           this_shard_db.replicate_to(remote_db)
+           break if !am_master
+         rescue
+           # Make sure that the node exist in the DB and flag it as unresponsive
+           if remote_node
+             remote_node.status = "UNAVAILABLE"
+             remote_node.save
+           end         
+         end
+       end
+
+    end    
+  end
+end
+
+class ContentDoc < CouchRest::ExtendedDocument
+  # NOTE: NO DEFAULT DATABASE IN THE OBJECT -- WE WILL BE STORING A LOT OF 
+  # DATABASES OF THIS TYPE.
+
+  property :internal_id
+  property :size_bytes
+  property :filenane
+  property :mime_type
+  
+  view_by :internal_id
+  
+  # A special attachment "File" is expected to exist
+end