nightcrawler 0.0.1

data/.gitignore

@@ -0,0 +1,4 @@
+*.gem
+.bundle
+Gemfile.lock
+pkg/*

data/.rspec

@@ -0,0 +1 @@
+--colour

data/Gemfile

@@ -0,0 +1,4 @@
+source "http://rubygems.org"
+
+# Specify your gem's dependencies in nightcrawler.gemspec
+gemspec

data/Rakefile

@@ -0,0 +1,2 @@
+require 'bundler'
+Bundler::GemHelper.install_tasks

data/TODO

@@ -0,0 +1,3 @@
+migration connection swap blocks
+support associations
+allow for meta class callbacks

data/lib/nightcrawler.rb

@@ -0,0 +1,13 @@
+
+require 'active_record'
+#require File.expand_path(File.join(File.dirname(__FILE__), "nightcrawler", "relation"))
+
+module Nightcrawler
+  autoload :ShardDescriptor, File.expand_path(File.join(File.dirname(__FILE__), "nightcrawler", "shard_descriptor"))
+  autoload :Manager, File.expand_path(File.join(File.dirname(__FILE__), "nightcrawler", "manager"))
+  autoload :Shard, File.expand_path(File.join(File.dirname(__FILE__), "nightcrawler", "shard"))
+  
+#  if defined? Rails
+#   require File.expand_path(File.join(File.dirname(__FILE__), "nightcrawler", "migration"))
+#  end
+end

data/lib/nightcrawler/manager.rb

@@ -0,0 +1,61 @@
+module Nightcrawler::Manager
+
+  def self.included(klass)
+    class << klass
+      attr_reader :prefix
+      def shards
+        Nightcrawler::Manager.mutex.synchronize do
+          @shards ||= Nightcrawler::Manager.shards_from_descriptions self, shard_descriptions
+        end
+      end
+
+      def shard(key)
+        shards[key]
+      end
+
+      def shard_prefix(prefix)
+        @prefix = prefix
+      end
+    end
+  end
+
+  class << self
+
+    def mutex
+      @mutex ||= Mutex.new
+    end
+
+    def shards_from_descriptions(from_class, description_hash)
+      Hash[description_hash.collect do |key, config|
+        [key, create_shard(from_class, key, config)]
+      end]
+    end
+
+    def create_shard(from_class, key, config)
+      raise "Could not find shard for key #{key}" unless config
+      class_name = "#{from_class.prefix}#{key.to_s.capitalize}"
+      klass = Class.new ActiveRecord::Base
+      if class_name.split("::").length > 1
+        _module = class_name.split("::").first
+        _class = class_name.split("::").last
+        eval(_module).module_eval{self.const_set _class, klass}
+      else
+        Object.const_set class_name, klass
+      end
+      klass.class_eval do
+        establish_connection config
+        include Nightcrawler::Shard
+        class << self
+          attr_accessor :shard_id
+        end
+
+        def self.sharded?
+          false
+        end
+      end
+      klass.shard_id = key
+      klass
+    end
+  end
+end
+

data/lib/nightcrawler/migration.rb

@@ -0,0 +1,20 @@
+module Nightcrawler::Migration
+
+  def self.extended(base)
+    class << base
+      def announce_with_nightcrawler(message)
+        announce_without_nightcrawler("#{message} - #{shard_key}")
+      end
+
+      alias_method_chain :migrate, :nightcrawler
+      alias_method_chain :announce, :nightcrawler
+    end
+  end
+  
+  def migrate_with_nightcrawler
+  end
+
+end
+
+ActiveRecord::Migration.extend(Nightcrawler::Migration)
+

data/lib/nightcrawler/relation.rb

@@ -0,0 +1,21 @@
+require 'active_support/core_ext/module/delegation.rb'
+
+ActiveRecord::Relation.class_eval do
+
+  def set_shard
+    if @klass.respond_to?(:sharded?) and @klass.sharded?
+      criteria_value = where_values.find{|node| node.left.name == @klass.shard_criteria.to_s rescue false}
+      raise "Missing shard critieria" unless criteria_value
+      @klass = @klass.shard(@klass.find_shard(criteria_value.right))
+    end
+  end
+  
+  %W{to_a size}.each do |method|
+    eval "alias original_#{method} #{method}"
+    define_method method do |*args|
+      set_shard
+      instance_eval "original_#{method}"
+    end
+  end
+end
+

data/lib/nightcrawler/shard.rb

@@ -0,0 +1,2 @@
+module Nightcrawler::Shard
+end

data/lib/nightcrawler/shard_descriptor.rb

@@ -0,0 +1,66 @@
+module Nightcrawler::ShardDescriptor
+  class << self
+    attr_writer :shard_mutex
+    def shard_mutex
+      @shard_mutex ||= Mutex.new
+    end
+  end
+  def self.included(klass)
+    class << klass
+      def shard(key)
+        Nightcrawler::ShardDescriptor.shard_mutex.synchronize do
+          shard_cache[key] ||= create_shard(key)
+        end
+      end
+
+      def shard_cache
+        @shard_cache ||= {}
+      end
+
+      attr_reader :shard_criteria, :manager, :shard_behavior
+
+      def shard_behavior(&block)
+        @behavior = block
+      end
+
+      def managed_by(manager)
+        @manager = manager
+      end
+
+      def defined(file, line, method)
+        manager.shard_descriptions.each do |key, config|
+          shard_cache[key] = create_shard(key)
+        end
+      end
+
+      def shard_by(key)
+        @shard_criteria = key
+      end
+
+      def create_shard(key)
+        raise "No shard for key: #{key.inspect}" unless manager.shard(key)
+        klass = Class.new manager.shard(key)
+        if self.name.split("::").length > 1
+          class_name = manager.prefix + key.to_s + self.name.split("::").last
+          _module = class_name.split("::").first
+          _klass = class_name.split("::").last
+          eval(_module).module_eval{self.const_set _klass, klass}
+        else
+          Object.const_set class_name, klass
+        end
+        klass.class_eval do
+          include Nightcrawler::Shard
+          class << self
+            attr_accessor :shard_id, :class_prefix, :described_by
+          end
+        end
+        klass.shard_id = key
+        klass.class_prefix = manager.prefix + key.to_s
+        klass.class_eval(&@behavior)
+        klass.described_by = self
+        klass
+      end
+    end
+  end
+end
+

data/lib/nightcrawler/version.rb

@@ -0,0 +1,3 @@
+module Nightcrawler
+  VERSION = "0.0.1"
+end

data/nightcrawler.gemspec

@@ -0,0 +1,29 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+require "nightcrawler/version"
+
+Gem::Specification.new do |s|
+  s.name        = "nightcrawler"
+  s.version     = Nightcrawler::VERSION
+  s.platform    = Gem::Platform::RUBY
+  s.authors     = ["Joshua Lane, Michael Prior"]
+  s.email       = ["lane.joshlane@gmail.com"]
+  s.homepage    = ""
+  s.summary     = %q{Minimal sharding solution for AR}
+  s.description = %q{Minimal sharding solution for AR}
+
+  s.rubyforge_project = "nightcrawler"
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+  
+  s.add_development_dependency "rspec", "~> 2.5.0"
+  s.add_development_dependency "faker", "~> 0.9.5"
+  s.add_development_dependency "mocha", "~> 0.9.12"
+  s.add_development_dependency "sqlite3", "~> 1.3.3"
+  s.add_development_dependency "autotest"
+  s.add_dependency "activerecord", "~> 3.0.5"
+  s.add_dependency "defined", "~> 0.0.1"
+end

data/spec/manager_spec.rb

@@ -0,0 +1,120 @@
+require 'spec_helper'
+
+
+
+$shards = {
+  1 => {:adapter => "sqlite3", :database => File.join(File.dirname(__FILE__), "db", "shard1.db")},
+  2 => {:adapter => "sqlite3", :database => File.join(File.dirname(__FILE__), "db", "shard2.db")},
+  3 => {:adapter => "sqlite3", :database => File.join(File.dirname(__FILE__), "db", "shard3.db")}
+}
+module Database
+  class Group
+    include Nightcrawler::Manager
+
+    shard_prefix "Database::Shard"
+
+    def self.shard_descriptions
+      $shards
+    end
+
+    def self.find_shard(value)
+      "shard#{value}".to_sym
+    end
+
+    def self.shard_for(key)
+      $shards[key]
+    end
+  end
+end
+
+class Database::Post
+  include Nightcrawler::ShardDescriptor
+  managed_by Database::Group
+
+  shard_behavior do
+    set_table_name "Posts"
+
+    has_many :comments, :class_name => "#{class_prefix}Comment", :foreign_key => "post_id"
+  end
+end
+
+class Database::Comment
+  include Nightcrawler::ShardDescriptor
+  managed_by Database::Group
+  shard_by :shard_key
+
+  shard_behavior do
+    set_table_name "Comments"
+    validates_presence_of :shard_key
+
+    belongs_to :post, :class_name => "#{class_prefix}Post"
+
+    scope :bad_comments, where(:comment => "bad")
+  end
+
+end
+
+describe Nightcrawler::Manager do
+
+  def clean_dbs
+    FileUtils.mkdir File.join(File.dirname(__FILE__), "db") unless File.exists? File.join(File.dirname(__FILE__), "db")
+    FileUtils.rm_f Dir[File.join(File.dirname(__FILE__), "db", "*.db")]
+  end
+
+  before(:all) do
+    clean_dbs
+    $shards.each_value do |config|
+      base = ActiveRecord::Base.establish_connection config
+      base.connection.create_table "Comments" do |t|
+        t.integer :shard_key
+        t.string :comment
+        t.integer :post_id
+      end
+      base.connection.create_table "Posts" do |t|
+        t.string :url
+      end
+    end
+  end
+
+  after(:all) do
+    clean_dbs
+    $shards.each_value do |config|
+      FileUtils.rm_f config[:database]
+    end
+  end
+
+  it "should create shards" do
+    Database::Group.shards.values.should == [Database::Shard1, Database::Shard2, Database::Shard3]
+  end
+
+  it "should create comment shards" do
+    Database::Comment.shard(1).to_s.should == "Database::Shard1Comment"
+  end
+
+  it "should respect shard behavior" do
+    lambda { Database::Comment.shard(1).create!(:comment => "no shard key") }.should raise_exception(ActiveRecord::RecordInvalid)
+  end
+
+  it "should share connection with shard class" do
+    Database::Comment.shard(1).connection.should eql Database::Shard1.connection
+  end
+
+  it "should respect scopes" do
+    bad = Database::Comment.shard(1).create!(:shard_key => 1, :comment => "bad")
+    good = Database::Comment.shard(1).create!(:shard_key => 1, :comment => "not bad")
+    Database::Comment.shard(1).bad_comments.should == [bad]
+    Database::Comment.shard(2).bad_comments.should == []
+  end
+
+  it "should respect relationships" do
+    respect = Database::Comment.shard(1).create!(:shard_key => 1, :comment => "respect")
+    Database::Comment.shard(1).where(:comment => "respect").should == [respect]
+  end
+
+  it "should respect associations" do
+    post = Database::Post.shard(1).create!(:url => "blah")
+    comment = Database::Comment.shard(1).create!(:shard_key => 1, :post => post)
+    post.comments.should == [comment]
+  end
+end
+

data/spec/spec_helper.rb

@@ -0,0 +1,16 @@
+require 'rubygems'
+require 'bundler'
+
+Bundler.require(:default, :development)
+
+Dir[File.join(File.dirname(__FILE__), "support/**/*.rb")].each {|f| require f}
+
+require File.join(File.dirname(__FILE__), "../lib/nightcrawler")
+require 'logger'
+FileUtils.mkdir_p File.expand_path(File.join(File.dirname(__FILE__), "..", "log"))
+ActiveRecord::Base.logger = Logger.new(File.join(File.dirname(__FILE__), "../log/test.log"))
+
+RSpec.configure do |config|
+  config.mock_with :mocha
+end
+

metadata

@@ -0,0 +1,146 @@
+--- !ruby/object:Gem::Specification 
+name: nightcrawler
+version: !ruby/object:Gem::Version 
+  prerelease: 
+  version: 0.0.1
+platform: ruby
+authors: 
+- Joshua Lane, Michael Prior
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2011-05-02 00:00:00 Z
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: rspec
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        version: 2.5.0
+  type: :development
+  version_requirements: *id001
+- !ruby/object:Gem::Dependency 
+  name: faker
+  prerelease: false
+  requirement: &id002 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        version: 0.9.5
+  type: :development
+  version_requirements: *id002
+- !ruby/object:Gem::Dependency 
+  name: mocha
+  prerelease: false
+  requirement: &id003 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        version: 0.9.12
+  type: :development
+  version_requirements: *id003
+- !ruby/object:Gem::Dependency 
+  name: sqlite3
+  prerelease: false
+  requirement: &id004 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        version: 1.3.3
+  type: :development
+  version_requirements: *id004
+- !ruby/object:Gem::Dependency 
+  name: autotest
+  prerelease: false
+  requirement: &id005 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+  type: :development
+  version_requirements: *id005
+- !ruby/object:Gem::Dependency 
+  name: activerecord
+  prerelease: false
+  requirement: &id006 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        version: 3.0.5
+  type: :runtime
+  version_requirements: *id006
+- !ruby/object:Gem::Dependency 
+  name: defined
+  prerelease: false
+  requirement: &id007 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        version: 0.0.1
+  type: :runtime
+  version_requirements: *id007
+description: Minimal sharding solution for AR
+email: 
+- lane.joshlane@gmail.com
+executables: []
+
+extensions: []
+
+extra_rdoc_files: []
+
+files: 
+- .gitignore
+- .rspec
+- Gemfile
+- README.md
+- Rakefile
+- TODO
+- lib/nightcrawler.rb
+- lib/nightcrawler/manager.rb
+- lib/nightcrawler/migration.rb
+- lib/nightcrawler/relation.rb
+- lib/nightcrawler/shard.rb
+- lib/nightcrawler/shard_descriptor.rb
+- lib/nightcrawler/version.rb
+- nightcrawler.gemspec
+- spec/manager_spec.rb
+- spec/spec_helper.rb
+homepage: ""
+licenses: []
+
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+requirements: []
+
+rubyforge_project: nightcrawler
+rubygems_version: 1.7.2
+signing_key: 
+specification_version: 3
+summary: Minimal sharding solution for AR
+test_files: []
+