mitchellh-lightcloud 0.1

data/CREDITS

@@ -0,0 +1,9 @@
+======================================
+= lightcloud credits                 =
+======================================
+
+Original Author (Python Implementation):
+  Amir Salihefendic http://amix.dk
+
+Ported to Ruby by:
+  Mitchell Hashimoto http://mitchellhashimoto.com

data/LICENSE

@@ -0,0 +1,10 @@
+Copyright (c) 2009, Mitchell Hashimoto
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
+
+    * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
+    * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
+    * Neither the name of the owner nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

data/README.rdoc

@@ -0,0 +1,34 @@
+= LightCloud Library for Ruby
+
+This is a library for accessing LightCloud systems through Ruby.
+
+== Background
+
+LightCloud is a distributed key-value stored open-sourced by Plurk.
+The official website which includes benchmarks, design specs, and
+more can be viewed at the following URL:
+
+http://opensource.plurk.com/LightCloud/
+
+== Usage
+
+  require 'rubygems'
+  require 'lightcloud'
+
+  LIGHT_CLOUD = {
+    'lookup1_A' => ['127.0.0.1:41401', '127.0.0.1:41402'],
+    'storage1_A' => ['192.168.0.2:51401', '192.168.0.2:51402']
+  }
+
+  lookup_nodes, storage_nodes = LightCloud.generate_nodes(LIGHT_CLOUD)
+  LightCloud.init(lookup_nodes, storage_nodes)
+
+  LightCloud.set("hello", "world")
+  print LightCloud.get("hello") # => world
+  LightCloud.delete("hello")
+
+  print LightCloud.get("hello") # => nil
+
+== Installation
+
+  sudo gem install mitchellh-lightcloud

data/Rakefile

@@ -0,0 +1,90 @@
+require 'rake'
+require 'rake/clean'
+require 'rake/packagetask'
+require 'rake/gempackagetask'
+require 'rake/rdoctask'
+require 'spec/rake/spectask'
+require 'fileutils'
+
+###################################
+# Clean & Defaut Task
+###################################
+CLEAN.include('dist','tmp','rdoc')
+task :default => [:clean, :repackage]
+
+###################################
+# Specs
+################################### 
+desc "Run all specs for hash_ring"
+Spec::Rake::SpecTask.new('spec') do |t|
+  t.spec_files = FileList['spec/**/*.rb']
+end
+
+###################################
+# Docs
+###################################
+Rake::RDocTask.new do |rd|
+  rd.main = 'README.rdoc'
+ 
+  rd.rdoc_dir = 'doc'
+ 
+  rd.rdoc_files.include(
+    'README.rdoc',
+    'LICENSE',
+    'CREDITS',
+    'lib/**/*.rb')
+ 
+  rd.title = 'lightcloud'
+ 
+  rd.options << '-N' # line numbers
+  rd.options << '-S' # inline source
+end
+
+###################################
+# Packaging - Thank you Sinatra
+################################### 
+# Load the gemspec using the same limitations as github
+def spec
+  @spec ||=
+    begin
+      require 'rubygems/specification'
+      data = File.read('lightcloud.gemspec')
+      spec = nil
+      Thread.new { spec = eval("$SAFE = 3\n#{data}") }.join
+      spec
+    end
+end
+ 
+def package(ext='')
+  "dist/lightcloud-#{spec.version}" + ext
+end
+ 
+desc 'Build packages'
+task :package => %w[.gem .tar.gz].map {|e| package(e)}
+ 
+desc 'Build and install as local gem'
+task :install => package('.gem') do
+  sh "gem install #{package('.gem')}"
+end
+ 
+directory 'dist/'
+CLOBBER.include('dist')
+ 
+file package('.gem') => %w[dist/ lightcloud.gemspec] + spec.files do |f|
+  sh "gem build hash_ring.gemspec"
+  mv File.basename(f.name), f.name
+end
+ 
+file package('.tar.gz') => %w[dist/] + spec.files do |f|
+  sh <<-SH
+git archive \
+--prefix=lightcloud-#{source_version}/ \
+--format=tar \
+HEAD | gzip > #{f.name}
+SH
+end
+
+def source_version
+  line = File.read('lib/lightcloud.rb')[/^\s*VERSION = .*/]
+  line.match(/.*VERSION = '(.*)'/)[1]
+end

data/lib/lightcloud.rb

@@ -0,0 +1,215 @@
+######################################
+# LightCloud
+# Code ported from Python version written by Amir Salihefendic
+######################################
+# Copyright (c) 2009, Mitchell Hashimoto, mitchell.hashimoto@gmail.com
+#
+
+require 'rubygems'
+require 'hash_ring'
+require 'rufus/tokyo'
+
+require File.join(File.dirname(__FILE__), 'tyrant_client')
+
+# = LightCloud Library
+#
+# == Background
+#
+# == Usage
+#
+#  require 'rubygems'
+#  require 'lightcloud'
+#
+#  LIGHT_CLOUD = {
+#    'lookup1_A' => ['127.0.0.1:41401', '127.0.0.1:41402'],
+#    'storage1_A' => ['192.168.0.2:51401', '192.168.0.2:51402']
+#  }
+#
+#  lookup_nodes, storage_nodes = LightCloud.generate_nodes(LIGHT_CLOUD)
+#  LightCloud.init(lookup_nodes, storage_nodes)
+#
+#  LightCloud.set("hello", "world")
+#  print LightCloud.get("hello") # => world
+#  LightCloud.delete("hello")
+#
+#  print LightCloud.get("hello") # => nil
+#
+class LightCloud
+  VERSION = '0.1'
+  DEFAULT_SYSTEM = 'default'
+
+  @@systems = {}
+
+  def self.init(lookup_nodes, storage_nodes, system=DEFAULT_SYSTEM)
+    lookup_ring, name_to_l_nodes = self.generate_ring(lookup_nodes)
+    storage_ring, name_to_s_nodes = self.generate_ring(storage_nodes)
+
+    @@systems[system] = [lookup_ring, storage_ring, name_to_l_nodes, name_to_s_nodes]
+  end
+
+  #--
+  # Get/Set/Delete
+  #++
+  #
+  # Sets a value to a key in the LightCloud system.
+  #
+  # Set first checks to see if the key is already stored. If it is
+  # it uses that same node to store the new value. Otherwise, it
+  # determines where to store the value based on the hash_ring
+  def self.set(key, value, system=DEFAULT_SYSTEM)
+    storage_node = self.locate_node_or_init(key, system)
+    return storage_node.set(key, value)
+  end
+
+  #
+  # Gets a value based on a key. 
+  def self.get(key, system=DEFAULT_SYSTEM)
+    result = nil
+
+    # Try to lookup key directly
+    storage_node = self.get_storage_ring(system).get_node(key)
+    value = storage_node.get(key)
+
+    result = value unless value.nil?
+    
+    # Else use the lookup ring
+    if result.nil?
+      storage_node = self.locate_node(key, system)
+      
+      result = storage_node.get(key) unless storage_node.nil?
+    end
+
+    result
+  end
+
+  #
+  # Lookup the key and delete it from both the storage ring
+  # and lookup ring
+  def self.delete(key, system=DEFAULT_SYSTEM)
+    storage_node = self.locate_node(key, system)
+    
+    storage_node = get_storage_ring(system).get_node(key) if storage_node.nil?
+    lookup_nodes = get_lookup_ring(system).iterate_nodes(key)
+    lookup_nodes.each_index do |i|
+      break if i > 1
+      
+      lookup_nodes[i].delete(key)
+    end
+    
+    storage_node.delete(key) unless storage_node.nil?
+    true
+  end
+
+  #--
+  # Lookup Cloud
+  #++
+  def self.locate_node_or_init(key, system)
+    storage_node = self.locate_node(key, system)
+
+    if storage_node.nil?
+      storage_node = self.get_storage_ring(system).get_node(key)
+
+      lookup_node = self.get_lookup_ring(system).get_node(key)
+      lookup_node.set(key, storage_node.to_s)
+    end
+
+    storage_node
+  end
+
+  #
+  # Locates a node in the lookup ring, returning the node if it is found, or
+  # nil otherwise.
+  def self.locate_node(key, system=DEFAULT_SYSTEM)
+    nodes = self.get_lookup_ring(system).iterate_nodes(key)
+    
+    lookups = 0
+    value = nil
+    nodes.each_index do |i|
+      lookups = i
+      return nil if lookups > 2
+      
+      node = nodes[lookups]
+      value = node.get(key)
+
+      break unless value.nil?
+    end
+
+    return nil if value.nil?
+    
+    if lookups == 0
+      return self.get_storage_node(value, system)
+    else
+      return self._clean_up_ring(key, value, system)
+    end
+  end
+
+  def self._clean_up_ring(key, value, system)
+    nodes = self.get_lookup_ring(system).iterate_nodes(key)
+
+    nodes.each_index do |i|
+      break if i > 1
+
+      node = nodes[i]
+      if i == 0
+        node.set(key, value)
+      else
+        node.delete(key)
+      end
+    end
+
+    return self.get_storage_node(value, system)
+  end
+
+  #--
+  # Accessors for rings
+  #++
+  def self.get_lookup_ring(system=DEFAULT_SYSTEM)
+    @@systems[system][0]
+  end
+
+  def self.get_storage_ring(system=DEFAULT_SYSTEM)
+    @@systems[system][1]
+  end
+
+  #--
+  # Accessors for nodes
+  #++
+  def self.get_storage_node(name, system=DEFAULT_SYSTEM)
+    @@systems[system][3][name]
+  end
+
+  #--
+  # Helpers
+  #++
+  def self.generate_nodes(config)
+    lookup_nodes = {}
+    storage_nodes = {}
+
+    config.each do |k,v|
+      if k.include?("lookup")
+        lookup_nodes[k] = v
+      elsif k.include?("storage")
+        storage_nodes[k] = v
+      end
+    end
+
+    return lookup_nodes, storage_nodes
+  end
+  
+  #
+  # Given a set of nodes it creates the the nodes as Tokyo
+  # Tyrant objects and returns a hash ring with them
+  def self.generate_ring(nodes)
+    objects = []
+    name_to_obj = {}
+    
+    nodes.each do |name, nodelist|
+      obj = TyrantNode.new(name, nodelist)
+      name_to_obj[name] = obj
+
+      objects.push(obj)
+    end
+
+    return HashRing.new(objects), name_to_obj
+  end
+end

data/spec/lightcloud_spec.rb

@@ -0,0 +1,239 @@
+require File.join(File.dirname(__FILE__), 'spec_base')
+
+describe LightCloud do
+  before do
+    @valid_servers = {
+      'lookup1_A' => ['127.0.0.1:1234', '127.0.0.1:4567'],
+      'storage1_A' => ['127.0.0.2:1234', '127.0.0.2:4567']
+    }
+
+    @valid_lookup_nodes, @valid_storage_nodes = LightCloud.generate_nodes(@valid_servers)
+
+    @generic_node = mock(TyrantNode)
+    [:get, :set, :delete].each do |meth|
+      @generic_node.stub!(meth).and_return(nil)
+    end
+
+    @nodes = [mock(TyrantNode), mock(TyrantNode), mock(TyrantNode)]
+    @lookup_valid_node = mock(TyrantNode)
+    @storage_valid_node = mock(TyrantNode)
+      
+    (@nodes + [@lookup_valid_node, @storage_valid_node]).each do |node|
+      node.stub!(:get).and_return(nil)
+      node.stub!(:set).and_return(nil)
+      node.stub!(:delete).and_return(nil)
+      node.stub!(:to_s).and_return(nil)
+    end
+
+    @storage_valid_node.stub!(:to_s).and_return('storage_valid_node')
+      
+    @lookup_ring = mock(HashRing)
+    @lookup_ring.stub!(:iterate_nodes).and_return(@nodes)
+    @lookup_ring.stub!(:get_node).and_return(@lookup_valid_node)
+      
+    @storage_ring = mock(HashRing)
+    @storage_ring.stub!(:get_node).and_return(@storage_valid_node)
+      
+    LightCloud.stub!(:get_lookup_ring).and_return(@lookup_ring)
+    LightCloud.stub!(:get_storage_ring).and_return(@storage_ring)
+  end
+
+  describe "node generation" do
+    it "should split lookup and storage nodes into their own arrays" do
+      lookup, storage = LightCloud.generate_nodes(@valid_servers)
+
+      lookup.should be_has_key('lookup1_A')
+      storage.should be_has_key('storage1_A')
+    end
+
+    it "should ignore configuration without 'lookup' or 'storage' in it" do
+      @valid_servers['foobarbaz'] = []
+
+      lookup, storage = LightCloud.generate_nodes(@valid_servers)
+
+      lookup.should_not be_has_key('foobarbaz')
+      storage.should_not be_has_key('foobarbaz')      
+    end
+  end
+
+  describe "ring generation" do
+    it "should create a TyrantNode for each node" do
+      TyrantNode.should_receive(:new).with('lookup1_A', anything).once
+      TyrantNode.should_receive(:new).with('storage1_A', anything).once
+      
+      LightCloud.init(@valid_lookup_nodes, @valid_storage_nodes)
+    end
+
+    it "should return the tyrant nodes as a name to node hash" do
+      unneeded, name_to_nodes = LightCloud.generate_ring(@valid_lookup_nodes)
+
+      name_to_nodes.should be_has_key('lookup1_A')
+      name_to_nodes['lookup1_A'].should be_kind_of(TyrantNode)
+    end
+
+    it "should return a hash ring with the nodes" do
+      ring, unneeded = LightCloud.generate_ring(@valid_lookup_nodes)
+
+      ring.should be_kind_of(HashRing)
+    end
+  end
+
+  describe "lookup cloud methods" do
+    before do
+      @key = 'foo'
+      @storage_node = 'bar'
+    end
+
+    describe "locating or initting a storage node by key" do
+      after do
+        LightCloud.should_receive(:locate_node).with(@key, anything).once.and_return(@storage_node)
+        LightCloud.locate_node_or_init(@key, LightCloud::DEFAULT_SYSTEM)
+      end
+
+      it "should just return the storage node if it was found" do
+        LightCloud.should_not_receive(:get_storage_ring)
+        LightCloud.should_not_receive(:get_lookup_ring)
+      end
+
+      it "should set the lookup ring to point to the new storage node if no previous storage node was found" do
+        @storage_node = nil
+        
+        @storage_ring.should_receive(:get_node).with(@key).once.and_return(@storage_valid_node)
+        @lookup_valid_node.should_receive(:set).with(@key, @storage_valid_node.to_s).once
+      end
+    end
+
+    describe "locating a storage node by key" do
+      it "should return the storage node if the key is found in the lookup ring" do    
+        @nodes[0].should_receive(:get).with(@key).and_return(@storage_node)
+        LightCloud.should_receive(:get_storage_node).with(@storage_node, anything).once
+
+        LightCloud.locate_node(@key)
+      end
+
+      it "should return nil if the key doesn't exist in the lookup ring" do
+        LightCloud.locate_node(@key).should be_nil
+      end
+
+      it "should attempt to clean up the lookup ring if the value is NOT found in the first node" do
+        @nodes[1].should_receive(:get).with(@key).and_return(@storage_node)
+        LightCloud.should_not_receive(:get_storage_node)
+        LightCloud.should_receive(:_clean_up_ring).with(@key, @storage_node, anything).once
+
+        LightCloud.locate_node(@key)
+      end
+    end
+
+    describe "cleaning the lookup ring" do
+      after do
+        LightCloud._clean_up_ring(@key, @storage_node, LightCloud::DEFAULT_SYSTEM)
+      end
+
+      it "should set the key/value onto the first node (index 0)" do
+        @nodes[0].should_receive(:set).with(@key, @storage_node).once
+      end
+
+      it "should delete the key from the second node (index 1)" do
+        @nodes[1].should_receive(:delete).with(@key).once
+      end
+
+      it "should not touch any other nodes" do
+        @nodes[2].should_not_receive(:get)
+        @nodes[2].should_not_receive(:set)
+        @nodes[2].should_not_receive(:delete)
+      end
+      
+      it "should return the storage node lookup" do
+        LightCloud.should_receive(:get_storage_node).with(@storage_node, anything).once
+      end
+    end
+  end
+
+  describe "setting" do
+    before do
+      @key = 'hello'
+      @value = 'world!'
+
+      LightCloud.stub!(:locate_node_or_init).and_return(@generic_node)
+    end
+
+    after do
+      LightCloud.set(@key, @value)
+    end
+
+    it "should lookup the node or init for where to place key" do
+      LightCloud.should_receive(:locate_node_or_init).with(@key, anything).once.and_return(@generic_node)
+    end
+
+    it "should set the value on the node returned by locate node or init" do
+      @generic_node.should_receive(:set).with(@key, @value)
+    end
+  end
+
+  describe "getting" do
+    before do
+      @key = 'foo'
+      @value = 'baz'
+      @storage_ring.should_receive(:get_node).with(@key).and_return(@generic_node)
+    end
+
+    after do
+      LightCloud.get(@key).should eql(@value)
+    end
+
+    it "should not resort to the lookup table if it can find the key directly" do
+      @generic_node.should_receive(:get).with(@key).and_return(@value)
+
+      LightCloud.should_not_receive(:locate_node)
+    end
+
+    it "should get the storage node from the lookup table if it can't find the key directly" do
+      @generic_node.should_receive(:get).once.and_return(nil)
+
+      LightCloud.should_receive(:locate_node).with(@key, anything).and_return(@storage_valid_node)
+      @storage_valid_node.should_receive(:get).with(@key).and_return(@value)
+    end
+  end
+
+  describe "deleting" do
+    before do
+      @key = 'foo'
+      @value = 'baz'
+    end
+
+    after do
+      # I wrap this in a should_not raise error for the final
+      # spec in this context, which WOULD raise an error if
+      # it failed
+      lambda do
+        LightCloud.delete(@key)
+      end.should_not raise_error
+    end
+
+    it "should delete the key from first two lookup nodes from iteration" do
+      @nodes[0].should_receive(:delete).with(@key).once
+      @nodes[1].should_receive(:delete).with(@key).once
+      @nodes[2].should_not_receive(:delete)
+    end
+
+    it "should first try to get the storage node from lookup ring" do
+      LightCloud.should_receive(:locate_node).with(@key, anything).once.and_return(@generic_node)
+      LightCloud.should_not_receive(:get_storage_ring)
+
+    end
+
+    it "should try to get storage node directly if lookup ring failed" do
+      LightCloud.should_receive(:locate_node).with(@key, anything).once.and_return(nil)
+      LightCloud.should_receive(:get_storage_ring).once.and_return(@storage_ring)
+      @storage_ring.should_receive(:get_node).with(@key).once.and_return(@generic_node)
+
+      @generic_node.should_receive(:delete).with(@key)
+    end
+
+    it "should only delete from a storage node if one was found" do
+      LightCloud.should_receive(:locate_node).with(@key, anything).once.and_return(nil)
+      LightCloud.should_receive(:get_storage_ring).once.and_return(@storage_ring)
+      @storage_ring.should_receive(:get_node).with(@key).once.and_return(nil)
+    end
+  end
+end

data/spec/spec_base.rb

@@ -0,0 +1,2 @@
+# Include the hash_ring library
+require File.expand_path(File.dirname(__FILE__) + '/../lib/lightcloud')

metadata

@@ -0,0 +1,84 @@
+--- !ruby/object:Gem::Specification 
+name: mitchellh-lightcloud
+version: !ruby/object:Gem::Version 
+  version: "0.1"
+platform: ruby
+authors: 
+- Mitchell Hashimoto
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2009-03-04 00:00:00 -08:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: rufus-tokyo
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 0.1.9
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: mitchellh-hash_ring
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0.2"
+    version: 
+description: LightCloud library for Ruby
+email: mitchell.hashimoto@gmail.com
+executables: []
+
+extensions: []
+
+extra_rdoc_files: 
+- README.rdoc
+- LICENSE
+files: 
+- CREDITS
+- LICENSE
+- README.rdoc
+- Rakefile
+- lib/lightcloud.rb
+- spec/spec_base.rb
+- spec/lightcloud_spec.rb
+has_rdoc: true
+homepage: http://github.com/mitchellh/lightcloud/
+post_install_message: 
+rdoc_options: 
+- --line-numbers
+- --inline-source
+- --title
+- hash_ring
+- --main
+- README.rdoc
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.2.0
+signing_key: 
+specification_version: 2
+summary: LightCloud library for Ruby
+test_files: []
+