adhd 0.0.1 → 0.1.0

data/test/test_adhd.rb

@@ -1,12 +1,3 @@
-<<<<<<< HEAD:test/test_adhd.rb
-require 'helper'
-
-class TestAdhd < Test::Unit::TestCase
-  should "probably rename this file and start testing for real" do
-    flunk "hey buddy, you should probably rename this file and start testing for real"
-  end
-end
-=======
 require 'rubygems'
 require 'test/unit'
 require 'shoulda'
@@ -80,5 +71,3 @@ class TestAdhd <  Test::Unit::TestCase
   private
 
 end
-
->>>>>>> 15388c36a078b8eee9bc17c79501985e54be519b:test/test_adhd.rb

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: adhd
 version: !ruby/object:Gem::Version 
-  version: 0.0.1
+  version: 0.1.0
 platform: ruby
 authors: 
 - dave.hrycyszyn@headlondon.com
@@ -9,8 +9,8 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2009-12-19 00:00:00 +00:00
-default_executable: adhd
+date: 2009-12-27 00:00:00 +00:00
+default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
   name: thoughtbot-shoulda
@@ -66,6 +66,7 @@ description: "More to say when something works! Do not bother installing this! "
 email: dave.hrycyszyn@headlondon.com
 executables: 
 - adhd
+- adhd_cleanup
 extensions: []
 
 extra_rdoc_files: 
@@ -80,11 +81,15 @@ files:
 - VERSION
 - adhd.gemspec
 - bin/adhd
+- bin/adhd_cleanup
 - doc/adhd.xmi
-- lib/adhd.rb
+- lib/adhd/adhd_rest_server.rb
 - lib/adhd/config.yml
-- lib/adhd/models.rb
-- lib/adhd/node.rb
+- lib/adhd/models/content_doc.rb
+- lib/adhd/models/content_shard.rb
+- lib/adhd/models/node_doc.rb
+- lib/adhd/models/shard_range.rb
+- lib/adhd/node_manager.rb
 - lib/adhd/reactor.rb
 - lib/ext/hash_to_openstruct.rb
 - lib/public/images/img01.jpg
@@ -106,7 +111,6 @@ files:
 - lib/public/style.css
 - lib/views/index.erb
 - lib/views/layout.erb
-- models.rb
 - test/helper.rb
 - test/test_adhd.rb
 has_rdoc: true

data/lib/adhd.rb

@@ -1,120 +0,0 @@
-require 'rubygems'
-require 'sinatra'
-require 'couchrest'
-require 'erb'
-require 'ruby-debug'
-require File.dirname(__FILE__) + '/adhd/models'
-
-# Start the server for now by cd'ing into the /lib directory and running the
-# following command:
-#
-# (first node):
-# ruby adhd.rb <node_name> <couchdb_server_url>
-#
-# (second or later node)
-# ruby adhd.rb <node_name> <couchdb_server_url> <management_node_url> <management_node_db> -p <port_number>
-#
-# <node_name> is just a string, e.g. "foo".
-# <couchdb_server_url>: the url (including port) for this node's couchdb server
-#   instance, e.g, http://192.168.1.104:5984
-# <management_node_url>: the url of the management node where this node should
-#   initially replicate from, e.g. http://192.168.1.104:5984
-# <management_node_db>: the couchdb management node database, e.g. "bar_node_db"
-# <port_number>: a port number to run on. If you're running more than one node locally
-#   for development purposes you'll need to pick a non-default port higher than 1024.
-
-node_name = ARGV[1]
-node_url = ARGV[2]
-buddy_server_url = ARGV[3]
-buddy_db =  ARGV[4]
-
-NODESERVER = CouchRest.new("#{node_url}")
-NODESERVER.default_database = "#{node_name}_node_db"
-node_db = CouchRest::Database.new(NODESERVER, "#{node_name}_node_db")
-
-# sync the db with our buddy
-if buddy_server_url && buddy_db
-  buddy_server = CouchRest.new("#{buddy_server_url}")
-  buddy_db = CouchRest::Database.new(buddy_server, buddy_db + "_node_db")
-  node_db.replicate_from(buddy_db)
-end
-
-# Retrieve our own node by our name
-# If there are other nodes with the name kill their records!
-node_candidates = Node.by_name(:key => node_name)
-node = node_candidates.pop
-node = Node.new if node.nil?
-node_candidates.each do |other_me|
-  other_me.destroy # destroy other records
-end
-
-# Update our very own record
-node.name = node_name
-node.url = node_url
-node.status = "RUNNING"
-node.save
-
-# We check if we are the first node. If we are the first node, we set ourself up
-# as the management node.
-all_nodes = Node.by_name()
-if all_nodes.length == 1
-  # puts "Setup #{node.name} as management node"
-  node.is_management = 3
-  node.save
-end
-
-# Lets build a nice NodeDB
-ndb = NodeDB.new(node)
-
-# Lets build a nice ShardDB
-srdb = ShardRangeDB.new(ndb)
-
-# If there are no shards make a few, if we are managers
-#puts "Create new ranges?"
-#puts "How many shards: #{ShardRange.by_range_start.length}"
-#puts "in #{ShardRange::SHARDSERVER.default_database}"
-if ShardRange.by_range_start.length == 0 && node.is_management
-  puts "Creating new ranges"
-  srdb.build_shards(100)
-end
-
-# Polulate the shards with some nodes at random
-node_names = []
-all_nodes.each do |anode|
-  node_names << anode.name
-end
-
-ShardRange.by_range_start.each do |s|
-  if !s.node_list or s.node_list.length == 0
-    node_names.shuffle!
-    s.node_list = node_names[0..2]
-    s.master_node = node_names[0]
-    s.save
-  end
-
-end
-# Sync all the node databases
-
-ndb.sync # SYNC
-srdb.sync # SYNC
-
-srdb.get_content_shards.each do |content_shard_db|
-  content_shard_db.sync
-end
-
-get "/" do
-  @all_nodes = Node.by_name
-  erb :index
-end 
-
-get "/sync" do
-  # Sync the node database
-  ndb.sync
-  # Sync the shard database
-  srdb.sync
-
-  srdb.get_content_shards.each do |content_shard_db|
-    content_shard_db.sync
-  end
-end
-

data/lib/adhd/models.rb

@@ -1,388 +0,0 @@
-# Key Restrictions ok internal_IDs: must only contain [a-z0-9-]
-
-class Array
-  def shuffle!
-    size.downto(1) { |n| push delete_at(rand(n)) }
-    self
-  end
-end
-
-class NodeDB
-  
-  attr_accessor :local_node_db, :our_node
-  
-  def initialize(our_nodev)
-    @our_node = our_nodev
-    
-    # Get the address of the CDB from the node
-    @local_node_db = our_nodev.get_node_db
-  end
-  
-  def sync
-    # We replicate our state to the management node(s)
-    management_nodes = Node.by_is_management.reverse
-    # NOTE: randomize the order for load balancing here
-
-    # NOTE2: How to build skynet (TODO)
-    #        -------------------
-    #        If length of management is zero, then chose 3 different random
-    #        nodes at each sync, and sync with them in node_name order.
-    #        This guarantees that any updates on nodes are communicated in
-    #        O(log N) ephocs, at the cost of O(3 * N) connections per epoch.
-    #        It also guarantees any new management servers are discovered in
-    #        this O(log N) time, creating "jelly fish" or "partition proof"
-    #        availability. 
-    
-    management_nodes.each do |mng_node|
-      remote_db = mng_node.get_node_db
-      if !(mng_node.name == our_node.name)
-        begin 
-          puts "Sync NodeDB with #{mng_node.name}"
-          local_node_db.replicate_from(remote_db)
-          # TODO: Manage conflicts here
-          local_node_db.replicate_to(remote_db)
-          break if !our_node.is_management # Only need to contact one node
-        rescue
-          puts "Could not connect to DB node #{mng_node.name}"
-          # TODO: change status or chose another management server
-          mng_node.status = "UNAVAILABLE"
-          mng_node.save
-        end     
-      end
-    end
-  end 
-
-end
-
-class Node  < CouchRest::ExtendedDocument
-  NODESERVER = CouchRest.new("#{ARGV[1]}")
-  NODESERVER.default_database = "#{ARGV[0]}_node_db"
-
-  use_database NODESERVER.default_database
-
-  property :name
-  property :url
-  property :is_store
-  property :is_management
-  property :is_directory
-  property :status
-
-  timestamps!
-
-  view_by :name
-  view_by :is_management
-
-  def get_node_db
-    server = CouchRest.new("#{url}")
-    server.database!("#{name}_node_db")
-  end
-
-  def get_shard_db
-    server = CouchRest.new("#{url}")
-    server.database!("#{name}_shard_db")
-  end  
-  
-  def get_content_db(shard_db_name)
-    server = CouchRest.new("#{url}")
-    server.database!("#{name}_#{shard_db_name}_content_db")
-  end  
-end
-
-class ShardRangeDB
-
-  attr_accessor :nodes, :local_shard_db, :our_node  
-
-  def initialize(nodesv)
-    @nodes = nodesv
-    
-    # Automatically get our shard_db address from our own node name
-    @our_node = nodesv.our_node
-    @local_shard_db = nodesv.our_node.get_shard_db
-  end
-  
-  def sync
-    # We replicate our state from the management node(s)
-    # We never push content if we are only storage
-    management_nodes = Node.by_is_management.reverse
-    # NOTE: randomize the order for load balancing here
-    
-    
-    management_nodes.each do |mng_node|      
-      remote_db = mng_node.get_shard_db
-      if !(mng_node.name == our_node.name)
-        begin 
-          puts "Sync ShardRange DB pull from #{mng_node.name}"
-          local_shard_db.replicate_from(remote_db)
-          # TODO: Manage conflicts here          
-          if our_node.is_management 
-            # Push any changes to other management nodes
-            puts "Sync ShardRange DB pushto #{mng_node.name}"
-            local_shard_db.replicate_to(remote_db)
-          else
-            break # sync with only one management server
-        end
-        rescue
-          puts "Could not connect to DB node #{mng_node.name}"
-          # TODO: change status or chose another management server
-          mng_node.status = "UNAVAILABLE"
-          mng_node.save
-        end     
-      end
-    end
-  end 
-
-  def build_shards(number)
-    # Make a large list of possible id boundaries
-    characters = []
-    ("0".."9").each do |c|
-      characters << c
-    end
-    ("a".."z").each do |c|
-      characters << c
-    end
-    
-    # Generate 36 x 36 keys to choose boundaries from
-    all_keys = []
-    characters.each do |c1|
-      characters.each do |c2|
-        all_keys << (c1+c2)
-      end
-    end
-    
-    # Now chose our boundaries
-    num_range_keys = all_keys.length
-    approx_shard_size = (num_range_keys * 1.0) / number
-    
-    shard_starts = []
-    (0...number).each do |n|
-      shard_starts << (all_keys[(n * approx_shard_size).floor])
-    end
-    
-    shard_ends = shard_starts.clone
-    shard_ends << ("z" * 100)
-    shard_ends.delete_at(0)
-    
-    # Finally build them!
-    puts "Build Shards"
-    (0...number).each do |n|
-      puts "Shard #{n}: from #{shard_starts[n]} to #{shard_ends[n]}"
-      shard_name = "sh_#{shard_starts[n]}_to_#{shard_ends[n]}"
-      sr = ShardRange.new
-      sr.range_start = shard_starts[n]
-      sr.range_end = shard_ends[n]
-      sr.shard_db_name = shard_name
-      sr.save
-    end    
-  end
-
-  def get_shard(internal_id)
-    # Finds the list of shards within which this ID lives
-    all_shards = ShardRange.by_range_start 
-    selected_shards = []
-    all_shards.each do |a_shard| # TODO: linear search is inefficient -- create a view 
-      if a_shard.range_start <= internal_id && a_shard.range_end > internal_id
-        selected_shards << a_shard
-      end
-    end
-    selected_shards
-  end
-  
-  def get_content_shards
-    # Return the content_shards of our node
-    content_shards = []
-    ShardRange.by_node(:key => "node1").each do |s|
-      
-      # Build a content shard object
-      cs = ContentShard.new
-      cs.our_node = our_node
-      cs.this_shard = s
-      cs.nodes = nodes
-      cs.this_shard_db = our_node.get_content_db(s.shard_db_name)
-      
-      # add it to the list
-      content_shards << cs
-    end
-    content_shards
-  end
-  
-  def write_doc_directly(content_doc)
-    # Write a document directly to a nodes content repository
-    doc_shard = get_shard(content_doc.internal_id).first
-    doc_shard.get_nodes.each do |node|
-      # Try to write the doc to this node
-      begin
-        remote_ndb = NodeDB.new(node)
-        remote_content_shard = ContentShard.new(remote_ndb, doc_shard)
-        remote_content_shard.this_shard_db.save_doc(content_doc)
-        break
-      rescue
-        puts "Could not put doc in node #{node.name}"
-        # TODO: change status or chose another management server
-        node.status = "UNAVAILABLE"
-        node.save
-      
-      end
-    end
-    
-  end
-  
-  def get_doc_directly(internal_id)
-    # Write a document directly to a nodes content repository
-    doc_shard = get_shard(internal_id).first
-    
-    # TODO: Randomize the order of nodes for load balancing in retrieval!
-    docx = []  
-    doc_shard.get_nodes.each do |node|
-      # Try to write the doc to this node
-      begin
-        remote_ndb = NodeDB.new(node)
-        remote_content_shard = ContentShard.new(remote_ndb, doc_shard)
-        
-        docx = ContentDoc.by_internal_id(:key => internal_id, :database => remote_content_shard.this_shard_db)
-        if docx.length > 0
-          break
-        end
-      rescue
-        puts "Could not put doc in node #{node.name}"
-        # TODO: change status or chose another management server
-        node.status = "UNAVAILABLE"
-        node.save      
-      end
-    end    
-  docx
-  end
-  
-end
-
-class ShardRange < CouchRest::ExtendedDocument
-  SHARDSERVER = CouchRest.new("#{ARGV[1]}")
-  SHARDSERVER.default_database = "#{ARGV[0]}_shard_db"
-
-  use_database SHARDSERVER.default_database
-
-  property :range_start
-  property :range_end
-  property :node_list
-  property :master_node
-  property :shard_db_name
-
-  view_by :range_start 
-  
-  # View "node" - given a node returns the shards watched
-  # How to use this new
-  # 
-  # puts "Which Shards does 'node1' watch?"
-  # ShardRange.by_node(:key => "node1").each do |s|
-  #   puts "Shard: #{s.shard_db_name}"
-  # end
-
-   
-  view_by :node,
-          :map => 
-          "function(doc) {
-          if (doc['couchrest-type'] == 'ShardRange' && doc.node_list) {
-            doc.node_list.forEach(function(node){
-              emit(node, 1);
-            });
-          }
-        }"  
-
-  def get_nodes
-    # Return all nodes, with the master being first
-    all_nodes = node_list
-    all_nodes.delete(master_node)
-    all_nodes = [master_node] + all_nodes
-    allnodes
-  end
-
-end
-
-
-class ContentShard
-  attr_accessor :nodes, :this_shard, :our_node, :this_shard_db  
-
-  def initialize(nodesv, this_shardv)
-    @nodes = nodesv
-    @this_shard = this_shardv
-    
-    # Work out the rest
-    @our_node = nodesv.our_node
-    @this_shard_db = nodesv.our_node.get_content_db(this_shardv.shard_db_name)
-  end
-
-  def in_shard?(internal_id)
-     internal_id >= this_shard.range_start && internal_id < this_shard.range_end
-  end
-
-  def write_doc(content_doc)
-    # Write a content document to this shard    
-    # Make sure it is in this shard
-    if in_shard? content_doc.internal_id
-      this_shard_db.save_doc(content_doc)
-    end
-  end
-  
-  def sync
-    # A Shard only pushes with the master of the shard
-    # or the node with the highest is_storage value alive
-    # Shard masters ensure changes are pushed to all    
-
-    # NOTE: This method needs serious refactoring
-
-    # Are we the shard master?
-    am_master = false
-    if our_node.name == this_shard.master_node
-      am_master = true      
-    end
-
-    if !am_master
-      begin
-        master_node = Nodes.by_name(this_shard.master_node).first
-        remotedb = MASTER_node.get_content_db(this_shard.shard_db_name)
-        this_shard_db.replicate_to(remote_db)
-        return # We sync-ed so job is done
-      rescue
-        # We flag the master as unavailable
-        if remote_node
-          master_node.status = "UNAVAILABLE"
-          master_node.save
-        end         
-      end
-    end
-    
-    # Either we are the master or the master has failed -- we replicate with 
-    # all nodes or the first available aside us and master
-    this_shard.node_list.each do |node_name|
-       if !(our_node.name == node_name) && !(this_shard.master_node == node_name)
-         begin
-           # Push all changes to the other nodes
-           remote_node = Nodes.by_name(node_name).first
-           remotedb = remote_node.get_content_db(this_shard.shard_db_name)
-           this_shard_db.replicate_to(remote_db)
-           break if !am_master
-         rescue
-           # Make sure that the node exist in the DB and flag it as unresponsive
-           if remote_node
-             remote_node.status = "UNAVAILABLE"
-             remote_node.save
-           end         
-         end
-       end
-
-    end    
-  end
-end
-
-class ContentDoc < CouchRest::ExtendedDocument
-  # NOTE: NO DEFAULT DATABASE IN THE OBJECT -- WE WILL BE STORING A LOT OF 
-  # DATABASES OF THIS TYPE.
-
-  property :internal_id
-  property :size_bytes
-  property :filenane
-  property :mime_type
-  
-  view_by :internal_id
-  
-  # A special attachment "File" is expected to exist
-end