nightcrawler 0.0.1 → 0.0.2

data/.gitignore

@@ -2,3 +2,4 @@
 .bundle
 Gemfile.lock
 pkg/*
+*.db

data/lib/nightcrawler/mosaic.rb

@@ -0,0 +1,20 @@
+## each instance controls the complete set of similar shards
+#class Mosaic
+
+  ## a parent namespace for all our sharded models
+  #module ShardPool; end 
+
+  ## name should be unique across mosaics
+  #def initialize(name, keys_configs, &block=nil)
+    #@name = name
+    #@keys_configs = keys_configs
+    #@block = block
+    #@shards = {}
+    #@keys_configs.each do |k,v|
+      #@shard_klass[k] = 
+  #end
+
+
+#end
+
+

data/lib/nightcrawler/version.rb

@@ -1,3 +1,3 @@
 module Nightcrawler
-  VERSION = "0.0.1"
+  VERSION = "0.0.2"
 end

data/spec/mosaic_spec.rb

@@ -0,0 +1,101 @@
+#require 'spec_helper'
+
+#describe Mosaic do
+  #before(:each) do
+    #@builder = ShardBuilder.new(3)
+    #shard_configs = (1..3).inject({}) {|h,i| h.merge "shard#{i}".to_sym => @builder.config(i)}
+    #Mosaic.new(shard_configs) do
+      #class Parent < Shard; end
+      #class Child < Shard; end
+    #end
+  #end
+
+  #it "should provide the correct model for the given shard" do
+    #(1..3).each do |i|
+      #Parent.shard("shard#{i}".to_sym).first.name.split(']')[0].should== "[shard ##{i}"
+    #end
+  #end
+
+  #it "should provide the correct model for the given shard" do
+    #Mosaic.shards.each do |shard|
+      #shard::Parent.first.name.split(']')[0].should== "[shard ##{shard.key}"
+    #end
+  #end
+#end
+
+#class Comment < ActiveRecord::Base
+  #include Nightcrawler::Sharding
+  #establish_connection :adapter => "sqlite3", :database => File.expand_path(File.join(File.dirname(__FILE__), "db", "master.db"))
+
+  #validates_presence_of :shard_key
+  
+  #shard_by :shard_key
+  
+  #def self.find_shard(value)
+    #"shard#{value}".to_sym
+  #end
+
+  #def self.shard_for(key)
+    #$shards[key]
+  #end
+
+#end
+
+
+  #@shard_builder = TestShardBuilder.new(4)
+
+  #before(:each) do
+    #@shard_builder.build
+  #end
+
+  #after(:each) do
+    #@shard_builder.destroy
+  #end
+
+  
+
+  #it "should do something" do
+    #Comment.shard(:shard1).should == CommentShard1
+  #end
+  #it "should create second shard" do
+    #Comment.shard(:shard2).should == CommentShard2
+  #end
+
+  #describe "creating a comment" do
+    #before(:all) do
+      #@comment = Comment.shard(:shard1).create!(:shard_key => 1, :comment => "comment number 1")
+    #end
+    #after(:all) do
+      #@comment.destroy
+    #end
+    #it "should exist on the first shard" do
+      #Comment.shard(:shard1).where(:comment => "comment number 1").first.should == @comment
+    #end
+    #it "should not exist on the second shard" do
+      #Comment.shard(:shard2).where(:comment => "comment number 1").should be_empty
+    #end
+  #end
+
+  #describe "search for a comment" do
+    #before(:all) do
+      #@comment1 = Comment.shard(:shard1).create!(:shard_key => 1, :comment => "blah")
+      #@comment2 = Comment.shard(:shard2).create!(:shard_key => 2, :comment => "blah")
+    #end
+    #after(:all) do
+      #@comment1.destroy rescue nil
+      #@comment2.destroy rescue nil
+    #end
+    #it "should lookup shard based on relation" do
+      #Comment.where(:shard_key => 1).where(:comment => "blah").size.should == 1
+      #Comment.where(:shard_key => 1).where(:comment => "blah").first.should be_a CommentShard1
+      #Comment.where(:shard_key => 2).size.should == 1
+      #Comment.where(:shard_key => 2).first.should be_a Comment
+      #Comment.where(:shard_key => 2).first.should be_a CommentShard2
+    #end
+    
+    #it "should not do blind relation" do
+      #lambda { Comment.where(:comment => "blah").to_a }.should raise_exception
+    #end
+  #end
+#end
+

data/spec/shard_builder_spec.rb

@@ -0,0 +1,88 @@
+require 'spec_helper'
+require 'active_record'
+
+# ActiveRecord classes do not like to be inside anonymous modules-- if you want to anonymize, then you'll need a nested module!
+
+
+describe ShardBuilder do
+  context "testing the shard builder" do
+
+    it "should raze the db dir as expected" do
+      builder = ShardBuilder.new
+      builder.raze
+      Dir.entries(File.expand_path("../db", __FILE__)).length.should == 2
+      FileUtils.touch File.expand_path("../db/blahblah", __FILE__)
+      FileUtils.touch File.expand_path("../db/yo.db", __FILE__)
+      Dir.entries(File.expand_path("../db", __FILE__)).length.should == 4
+      builder.raze
+      Dir.entries(File.expand_path("../db", __FILE__)).length.should == 2
+    end
+
+    it "should be able to connect to a single shard no problem" do
+      builder = ShardBuilder.new(1,3)
+      ActiveRecord::Base.establish_connection(builder.config(1))
+      module A
+        class Parent < ActiveRecord::Base; end
+      end
+      A::Parent.all.length.should == 3
+      A::Parent.all.each do |p|
+        p.name.should match(/^\[shard #1\]/)
+      end
+    end
+
+    # unfortunately this seems to require setting the table name!
+    it "should be able to leverage sandwich classes to hold the connections and still have relationships on a single shard" do
+      builder = ShardBuilder.new(1,3,5)
+      module B
+        class Shard < ActiveRecord::Base; end
+        class Parent < Shard; set_table_name "parents"; has_many :children end
+        class Child < Shard; set_table_name "children"; end
+      end
+      B::Shard.establish_connection(builder.config(1))
+      B::Parent.all.length.should == 3
+      B::Parent.all.each do |p|
+        p.name.should match(/^\[shard #1\]/)
+        p.children.length.should == 5
+        p.children.each do |c|
+          c.name.should match(/^\[shard #1\]/)
+        end
+      end
+    end
+
+    # unfortunately this seems to require setting the table name!
+    it "should be able to have relationships in each shard" do
+      builder = ShardBuilder.new(2,3,5)
+      module C1 
+        class Shard < ActiveRecord::Base; end
+        class Parent < Shard; set_table_name "parents"; has_many :children end
+        class Child < Shard; set_table_name "children"; end
+      end
+      module C2
+        class Shard < ActiveRecord::Base; end
+        class Parent < Shard; set_table_name "parents"; has_many :children end
+        class Child < Shard; set_table_name "children"; end
+      end
+      C1::Shard.establish_connection(builder.config(1))
+      C2::Shard.establish_connection(builder.config(2))
+      C1::Parent.all.length.should == 3
+      C1::Parent.all.each do |p|
+        p.name.should match(/^\[shard #1\]/)
+        p.children.length.should == 5
+        p.children.each do |c|
+          c.name.should match(/^\[shard #1\]/)
+        end
+      end
+      C2::Parent.all.length.should == 3
+      C2::Parent.all.each do |p|
+        p.name.should match(/^\[shard #2\]/)
+        p.children.length.should == 5
+        p.children.each do |c|
+          c.name.should match(/^\[shard #2\]/)
+        end
+      end
+    end
+
+  end
+end
+
+

data/spec/support/shard_builder.rb

@@ -0,0 +1,47 @@
+require 'spec_helper'
+require 'active_record'
+
+# a utility class used only for tests that builds out a number of shards with parent & child tables prepopulated with some test data
+class ShardBuilder
+  class Child < ActiveRecord::Base; end;
+  class Parent < ActiveRecord::Base; end;
+
+  def initialize(shard_count=3, parent_count=3, child_count=5)
+    @shard_count = shard_count
+    @parent_count = parent_count
+    @child_count = child_count
+    raze
+    build
+  end
+
+  def build
+    (1..@shard_count).each do |i|
+      base = ActiveRecord::Base
+      base.establish_connection(config(i))
+      base.connection.create_table "children" do |t|
+        t.integer :parent_id
+        t.string :name
+      end
+      base.connection.create_table "parents" do |t|
+        t.integer :id
+        t.string :name
+      end
+      (1..@parent_count).each do |j|
+        (1..@child_count).each do |k|
+          Child.create :parent_id=>j, :name=>"[shard ##{i}] child(##{k}) for parent(##{j})"
+        end
+        Parent.create :name=>"[shard ##{i}] parent(#{j})"
+      end
+      base.remove_connection(base)
+    end
+  end
+
+  def raze
+    FileUtils.rm_rf File.expand_path("../../db", __FILE__)
+    FileUtils.mkdir_p File.expand_path("../../db", __FILE__)
+  end
+
+  def config(shard_index)
+    {:adapter => "sqlite3", :database => File.expand_path("../../db/shard#{shard_index}.db", __FILE__)}
+  end
+end

metadata

@@ -2,7 +2,7 @@
 name: nightcrawler
 version: !ruby/object:Gem::Version 
   prerelease: 
-  version: 0.0.1
+  version: 0.0.2
 platform: ruby
 authors: 
 - Joshua Lane, Michael Prior
@@ -10,7 +10,8 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-05-02 00:00:00 Z
+date: 2011-05-04 00:00:00 -04:00
+default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
   name: rspec
@@ -108,13 +109,18 @@ files:
 - lib/nightcrawler.rb
 - lib/nightcrawler/manager.rb
 - lib/nightcrawler/migration.rb
+- lib/nightcrawler/mosaic.rb
 - lib/nightcrawler/relation.rb
 - lib/nightcrawler/shard.rb
 - lib/nightcrawler/shard_descriptor.rb
 - lib/nightcrawler/version.rb
 - nightcrawler.gemspec
 - spec/manager_spec.rb
+- spec/mosaic_spec.rb
+- spec/shard_builder_spec.rb
 - spec/spec_helper.rb
+- spec/support/shard_builder.rb
+has_rdoc: true
 homepage: ""
 licenses: []
 
@@ -138,7 +144,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
 requirements: []
 
 rubyforge_project: nightcrawler
-rubygems_version: 1.7.2
+rubygems_version: 1.6.2
 signing_key: 
 specification_version: 3
 summary: Minimal sharding solution for AR