nightcrawler 0.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/.gitignore +4 -0
- data/.rspec +1 -0
- data/Gemfile +4 -0
- data/README.md +0 -0
- data/Rakefile +2 -0
- data/TODO +3 -0
- data/lib/nightcrawler.rb +13 -0
- data/lib/nightcrawler/manager.rb +61 -0
- data/lib/nightcrawler/migration.rb +20 -0
- data/lib/nightcrawler/relation.rb +21 -0
- data/lib/nightcrawler/shard.rb +2 -0
- data/lib/nightcrawler/shard_descriptor.rb +66 -0
- data/lib/nightcrawler/version.rb +3 -0
- data/nightcrawler.gemspec +29 -0
- data/spec/manager_spec.rb +120 -0
- data/spec/spec_helper.rb +16 -0
- metadata +146 -0
data/.gitignore
ADDED
data/.rspec
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
--colour
|
data/Gemfile
ADDED
data/README.md
ADDED
File without changes
|
data/Rakefile
ADDED
data/TODO
ADDED
data/lib/nightcrawler.rb
ADDED
@@ -0,0 +1,13 @@
|
|
1
|
+
|
2
|
+
require 'active_record'
|
3
|
+
#require File.expand_path(File.join(File.dirname(__FILE__), "nightcrawler", "relation"))
|
4
|
+
|
5
|
+
module Nightcrawler
|
6
|
+
autoload :ShardDescriptor, File.expand_path(File.join(File.dirname(__FILE__), "nightcrawler", "shard_descriptor"))
|
7
|
+
autoload :Manager, File.expand_path(File.join(File.dirname(__FILE__), "nightcrawler", "manager"))
|
8
|
+
autoload :Shard, File.expand_path(File.join(File.dirname(__FILE__), "nightcrawler", "shard"))
|
9
|
+
|
10
|
+
# if defined? Rails
|
11
|
+
# require File.expand_path(File.join(File.dirname(__FILE__), "nightcrawler", "migration"))
|
12
|
+
# end
|
13
|
+
end
|
@@ -0,0 +1,61 @@
|
|
1
|
+
module Nightcrawler::Manager
|
2
|
+
|
3
|
+
def self.included(klass)
|
4
|
+
class << klass
|
5
|
+
attr_reader :prefix
|
6
|
+
def shards
|
7
|
+
Nightcrawler::Manager.mutex.synchronize do
|
8
|
+
@shards ||= Nightcrawler::Manager.shards_from_descriptions self, shard_descriptions
|
9
|
+
end
|
10
|
+
end
|
11
|
+
|
12
|
+
def shard(key)
|
13
|
+
shards[key]
|
14
|
+
end
|
15
|
+
|
16
|
+
def shard_prefix(prefix)
|
17
|
+
@prefix = prefix
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|
21
|
+
|
22
|
+
class << self
|
23
|
+
|
24
|
+
def mutex
|
25
|
+
@mutex ||= Mutex.new
|
26
|
+
end
|
27
|
+
|
28
|
+
def shards_from_descriptions(from_class, description_hash)
|
29
|
+
Hash[description_hash.collect do |key, config|
|
30
|
+
[key, create_shard(from_class, key, config)]
|
31
|
+
end]
|
32
|
+
end
|
33
|
+
|
34
|
+
def create_shard(from_class, key, config)
|
35
|
+
raise "Could not find shard for key #{key}" unless config
|
36
|
+
class_name = "#{from_class.prefix}#{key.to_s.capitalize}"
|
37
|
+
klass = Class.new ActiveRecord::Base
|
38
|
+
if class_name.split("::").length > 1
|
39
|
+
_module = class_name.split("::").first
|
40
|
+
_class = class_name.split("::").last
|
41
|
+
eval(_module).module_eval{self.const_set _class, klass}
|
42
|
+
else
|
43
|
+
Object.const_set class_name, klass
|
44
|
+
end
|
45
|
+
klass.class_eval do
|
46
|
+
establish_connection config
|
47
|
+
include Nightcrawler::Shard
|
48
|
+
class << self
|
49
|
+
attr_accessor :shard_id
|
50
|
+
end
|
51
|
+
|
52
|
+
def self.sharded?
|
53
|
+
false
|
54
|
+
end
|
55
|
+
end
|
56
|
+
klass.shard_id = key
|
57
|
+
klass
|
58
|
+
end
|
59
|
+
end
|
60
|
+
end
|
61
|
+
|
@@ -0,0 +1,20 @@
|
|
1
|
+
module Nightcrawler::Migration
|
2
|
+
|
3
|
+
def self.extended(base)
|
4
|
+
class << base
|
5
|
+
def announce_with_nightcrawler(message)
|
6
|
+
announce_without_nightcrawler("#{message} - #{shard_key}")
|
7
|
+
end
|
8
|
+
|
9
|
+
alias_method_chain :migrate, :nightcrawler
|
10
|
+
alias_method_chain :announce, :nightcrawler
|
11
|
+
end
|
12
|
+
end
|
13
|
+
|
14
|
+
def migrate_with_nightcrawler
|
15
|
+
end
|
16
|
+
|
17
|
+
end
|
18
|
+
|
19
|
+
ActiveRecord::Migration.extend(Nightcrawler::Migration)
|
20
|
+
|
@@ -0,0 +1,21 @@
|
|
1
|
+
require 'active_support/core_ext/module/delegation.rb'
|
2
|
+
|
3
|
+
ActiveRecord::Relation.class_eval do
|
4
|
+
|
5
|
+
def set_shard
|
6
|
+
if @klass.respond_to?(:sharded?) and @klass.sharded?
|
7
|
+
criteria_value = where_values.find{|node| node.left.name == @klass.shard_criteria.to_s rescue false}
|
8
|
+
raise "Missing shard critieria" unless criteria_value
|
9
|
+
@klass = @klass.shard(@klass.find_shard(criteria_value.right))
|
10
|
+
end
|
11
|
+
end
|
12
|
+
|
13
|
+
%W{to_a size}.each do |method|
|
14
|
+
eval "alias original_#{method} #{method}"
|
15
|
+
define_method method do |*args|
|
16
|
+
set_shard
|
17
|
+
instance_eval "original_#{method}"
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|
21
|
+
|
@@ -0,0 +1,66 @@
|
|
1
|
+
module Nightcrawler::ShardDescriptor
|
2
|
+
class << self
|
3
|
+
attr_writer :shard_mutex
|
4
|
+
def shard_mutex
|
5
|
+
@shard_mutex ||= Mutex.new
|
6
|
+
end
|
7
|
+
end
|
8
|
+
def self.included(klass)
|
9
|
+
class << klass
|
10
|
+
def shard(key)
|
11
|
+
Nightcrawler::ShardDescriptor.shard_mutex.synchronize do
|
12
|
+
shard_cache[key] ||= create_shard(key)
|
13
|
+
end
|
14
|
+
end
|
15
|
+
|
16
|
+
def shard_cache
|
17
|
+
@shard_cache ||= {}
|
18
|
+
end
|
19
|
+
|
20
|
+
attr_reader :shard_criteria, :manager, :shard_behavior
|
21
|
+
|
22
|
+
def shard_behavior(&block)
|
23
|
+
@behavior = block
|
24
|
+
end
|
25
|
+
|
26
|
+
def managed_by(manager)
|
27
|
+
@manager = manager
|
28
|
+
end
|
29
|
+
|
30
|
+
def defined(file, line, method)
|
31
|
+
manager.shard_descriptions.each do |key, config|
|
32
|
+
shard_cache[key] = create_shard(key)
|
33
|
+
end
|
34
|
+
end
|
35
|
+
|
36
|
+
def shard_by(key)
|
37
|
+
@shard_criteria = key
|
38
|
+
end
|
39
|
+
|
40
|
+
def create_shard(key)
|
41
|
+
raise "No shard for key: #{key.inspect}" unless manager.shard(key)
|
42
|
+
klass = Class.new manager.shard(key)
|
43
|
+
if self.name.split("::").length > 1
|
44
|
+
class_name = manager.prefix + key.to_s + self.name.split("::").last
|
45
|
+
_module = class_name.split("::").first
|
46
|
+
_klass = class_name.split("::").last
|
47
|
+
eval(_module).module_eval{self.const_set _klass, klass}
|
48
|
+
else
|
49
|
+
Object.const_set class_name, klass
|
50
|
+
end
|
51
|
+
klass.class_eval do
|
52
|
+
include Nightcrawler::Shard
|
53
|
+
class << self
|
54
|
+
attr_accessor :shard_id, :class_prefix, :described_by
|
55
|
+
end
|
56
|
+
end
|
57
|
+
klass.shard_id = key
|
58
|
+
klass.class_prefix = manager.prefix + key.to_s
|
59
|
+
klass.class_eval(&@behavior)
|
60
|
+
klass.described_by = self
|
61
|
+
klass
|
62
|
+
end
|
63
|
+
end
|
64
|
+
end
|
65
|
+
end
|
66
|
+
|
@@ -0,0 +1,29 @@
|
|
1
|
+
# -*- encoding: utf-8 -*-
|
2
|
+
$:.push File.expand_path("../lib", __FILE__)
|
3
|
+
require "nightcrawler/version"
|
4
|
+
|
5
|
+
Gem::Specification.new do |s|
|
6
|
+
s.name = "nightcrawler"
|
7
|
+
s.version = Nightcrawler::VERSION
|
8
|
+
s.platform = Gem::Platform::RUBY
|
9
|
+
s.authors = ["Joshua Lane, Michael Prior"]
|
10
|
+
s.email = ["lane.joshlane@gmail.com"]
|
11
|
+
s.homepage = ""
|
12
|
+
s.summary = %q{Minimal sharding solution for AR}
|
13
|
+
s.description = %q{Minimal sharding solution for AR}
|
14
|
+
|
15
|
+
s.rubyforge_project = "nightcrawler"
|
16
|
+
|
17
|
+
s.files = `git ls-files`.split("\n")
|
18
|
+
s.test_files = `git ls-files -- {test,spec,features}/*`.split("\n")
|
19
|
+
s.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
|
20
|
+
s.require_paths = ["lib"]
|
21
|
+
|
22
|
+
s.add_development_dependency "rspec", "~> 2.5.0"
|
23
|
+
s.add_development_dependency "faker", "~> 0.9.5"
|
24
|
+
s.add_development_dependency "mocha", "~> 0.9.12"
|
25
|
+
s.add_development_dependency "sqlite3", "~> 1.3.3"
|
26
|
+
s.add_development_dependency "autotest"
|
27
|
+
s.add_dependency "activerecord", "~> 3.0.5"
|
28
|
+
s.add_dependency "defined", "~> 0.0.1"
|
29
|
+
end
|
@@ -0,0 +1,120 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
|
4
|
+
|
5
|
+
$shards = {
|
6
|
+
1 => {:adapter => "sqlite3", :database => File.join(File.dirname(__FILE__), "db", "shard1.db")},
|
7
|
+
2 => {:adapter => "sqlite3", :database => File.join(File.dirname(__FILE__), "db", "shard2.db")},
|
8
|
+
3 => {:adapter => "sqlite3", :database => File.join(File.dirname(__FILE__), "db", "shard3.db")}
|
9
|
+
}
|
10
|
+
module Database
|
11
|
+
class Group
|
12
|
+
include Nightcrawler::Manager
|
13
|
+
|
14
|
+
shard_prefix "Database::Shard"
|
15
|
+
|
16
|
+
def self.shard_descriptions
|
17
|
+
$shards
|
18
|
+
end
|
19
|
+
|
20
|
+
def self.find_shard(value)
|
21
|
+
"shard#{value}".to_sym
|
22
|
+
end
|
23
|
+
|
24
|
+
def self.shard_for(key)
|
25
|
+
$shards[key]
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
29
|
+
|
30
|
+
class Database::Post
|
31
|
+
include Nightcrawler::ShardDescriptor
|
32
|
+
managed_by Database::Group
|
33
|
+
|
34
|
+
shard_behavior do
|
35
|
+
set_table_name "Posts"
|
36
|
+
|
37
|
+
has_many :comments, :class_name => "#{class_prefix}Comment", :foreign_key => "post_id"
|
38
|
+
end
|
39
|
+
end
|
40
|
+
|
41
|
+
class Database::Comment
|
42
|
+
include Nightcrawler::ShardDescriptor
|
43
|
+
managed_by Database::Group
|
44
|
+
shard_by :shard_key
|
45
|
+
|
46
|
+
shard_behavior do
|
47
|
+
set_table_name "Comments"
|
48
|
+
validates_presence_of :shard_key
|
49
|
+
|
50
|
+
belongs_to :post, :class_name => "#{class_prefix}Post"
|
51
|
+
|
52
|
+
scope :bad_comments, where(:comment => "bad")
|
53
|
+
end
|
54
|
+
|
55
|
+
end
|
56
|
+
|
57
|
+
describe Nightcrawler::Manager do
|
58
|
+
|
59
|
+
def clean_dbs
|
60
|
+
FileUtils.mkdir File.join(File.dirname(__FILE__), "db") unless File.exists? File.join(File.dirname(__FILE__), "db")
|
61
|
+
FileUtils.rm_f Dir[File.join(File.dirname(__FILE__), "db", "*.db")]
|
62
|
+
end
|
63
|
+
|
64
|
+
before(:all) do
|
65
|
+
clean_dbs
|
66
|
+
$shards.each_value do |config|
|
67
|
+
base = ActiveRecord::Base.establish_connection config
|
68
|
+
base.connection.create_table "Comments" do |t|
|
69
|
+
t.integer :shard_key
|
70
|
+
t.string :comment
|
71
|
+
t.integer :post_id
|
72
|
+
end
|
73
|
+
base.connection.create_table "Posts" do |t|
|
74
|
+
t.string :url
|
75
|
+
end
|
76
|
+
end
|
77
|
+
end
|
78
|
+
|
79
|
+
after(:all) do
|
80
|
+
clean_dbs
|
81
|
+
$shards.each_value do |config|
|
82
|
+
FileUtils.rm_f config[:database]
|
83
|
+
end
|
84
|
+
end
|
85
|
+
|
86
|
+
it "should create shards" do
|
87
|
+
Database::Group.shards.values.should == [Database::Shard1, Database::Shard2, Database::Shard3]
|
88
|
+
end
|
89
|
+
|
90
|
+
it "should create comment shards" do
|
91
|
+
Database::Comment.shard(1).to_s.should == "Database::Shard1Comment"
|
92
|
+
end
|
93
|
+
|
94
|
+
it "should respect shard behavior" do
|
95
|
+
lambda { Database::Comment.shard(1).create!(:comment => "no shard key") }.should raise_exception(ActiveRecord::RecordInvalid)
|
96
|
+
end
|
97
|
+
|
98
|
+
it "should share connection with shard class" do
|
99
|
+
Database::Comment.shard(1).connection.should eql Database::Shard1.connection
|
100
|
+
end
|
101
|
+
|
102
|
+
it "should respect scopes" do
|
103
|
+
bad = Database::Comment.shard(1).create!(:shard_key => 1, :comment => "bad")
|
104
|
+
good = Database::Comment.shard(1).create!(:shard_key => 1, :comment => "not bad")
|
105
|
+
Database::Comment.shard(1).bad_comments.should == [bad]
|
106
|
+
Database::Comment.shard(2).bad_comments.should == []
|
107
|
+
end
|
108
|
+
|
109
|
+
it "should respect relationships" do
|
110
|
+
respect = Database::Comment.shard(1).create!(:shard_key => 1, :comment => "respect")
|
111
|
+
Database::Comment.shard(1).where(:comment => "respect").should == [respect]
|
112
|
+
end
|
113
|
+
|
114
|
+
it "should respect associations" do
|
115
|
+
post = Database::Post.shard(1).create!(:url => "blah")
|
116
|
+
comment = Database::Comment.shard(1).create!(:shard_key => 1, :post => post)
|
117
|
+
post.comments.should == [comment]
|
118
|
+
end
|
119
|
+
end
|
120
|
+
|
data/spec/spec_helper.rb
ADDED
@@ -0,0 +1,16 @@
|
|
1
|
+
require 'rubygems'
|
2
|
+
require 'bundler'
|
3
|
+
|
4
|
+
Bundler.require(:default, :development)
|
5
|
+
|
6
|
+
Dir[File.join(File.dirname(__FILE__), "support/**/*.rb")].each {|f| require f}
|
7
|
+
|
8
|
+
require File.join(File.dirname(__FILE__), "../lib/nightcrawler")
|
9
|
+
require 'logger'
|
10
|
+
FileUtils.mkdir_p File.expand_path(File.join(File.dirname(__FILE__), "..", "log"))
|
11
|
+
ActiveRecord::Base.logger = Logger.new(File.join(File.dirname(__FILE__), "../log/test.log"))
|
12
|
+
|
13
|
+
RSpec.configure do |config|
|
14
|
+
config.mock_with :mocha
|
15
|
+
end
|
16
|
+
|
metadata
ADDED
@@ -0,0 +1,146 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: nightcrawler
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
prerelease:
|
5
|
+
version: 0.0.1
|
6
|
+
platform: ruby
|
7
|
+
authors:
|
8
|
+
- Joshua Lane, Michael Prior
|
9
|
+
autorequire:
|
10
|
+
bindir: bin
|
11
|
+
cert_chain: []
|
12
|
+
|
13
|
+
date: 2011-05-02 00:00:00 Z
|
14
|
+
dependencies:
|
15
|
+
- !ruby/object:Gem::Dependency
|
16
|
+
name: rspec
|
17
|
+
prerelease: false
|
18
|
+
requirement: &id001 !ruby/object:Gem::Requirement
|
19
|
+
none: false
|
20
|
+
requirements:
|
21
|
+
- - ~>
|
22
|
+
- !ruby/object:Gem::Version
|
23
|
+
version: 2.5.0
|
24
|
+
type: :development
|
25
|
+
version_requirements: *id001
|
26
|
+
- !ruby/object:Gem::Dependency
|
27
|
+
name: faker
|
28
|
+
prerelease: false
|
29
|
+
requirement: &id002 !ruby/object:Gem::Requirement
|
30
|
+
none: false
|
31
|
+
requirements:
|
32
|
+
- - ~>
|
33
|
+
- !ruby/object:Gem::Version
|
34
|
+
version: 0.9.5
|
35
|
+
type: :development
|
36
|
+
version_requirements: *id002
|
37
|
+
- !ruby/object:Gem::Dependency
|
38
|
+
name: mocha
|
39
|
+
prerelease: false
|
40
|
+
requirement: &id003 !ruby/object:Gem::Requirement
|
41
|
+
none: false
|
42
|
+
requirements:
|
43
|
+
- - ~>
|
44
|
+
- !ruby/object:Gem::Version
|
45
|
+
version: 0.9.12
|
46
|
+
type: :development
|
47
|
+
version_requirements: *id003
|
48
|
+
- !ruby/object:Gem::Dependency
|
49
|
+
name: sqlite3
|
50
|
+
prerelease: false
|
51
|
+
requirement: &id004 !ruby/object:Gem::Requirement
|
52
|
+
none: false
|
53
|
+
requirements:
|
54
|
+
- - ~>
|
55
|
+
- !ruby/object:Gem::Version
|
56
|
+
version: 1.3.3
|
57
|
+
type: :development
|
58
|
+
version_requirements: *id004
|
59
|
+
- !ruby/object:Gem::Dependency
|
60
|
+
name: autotest
|
61
|
+
prerelease: false
|
62
|
+
requirement: &id005 !ruby/object:Gem::Requirement
|
63
|
+
none: false
|
64
|
+
requirements:
|
65
|
+
- - ">="
|
66
|
+
- !ruby/object:Gem::Version
|
67
|
+
version: "0"
|
68
|
+
type: :development
|
69
|
+
version_requirements: *id005
|
70
|
+
- !ruby/object:Gem::Dependency
|
71
|
+
name: activerecord
|
72
|
+
prerelease: false
|
73
|
+
requirement: &id006 !ruby/object:Gem::Requirement
|
74
|
+
none: false
|
75
|
+
requirements:
|
76
|
+
- - ~>
|
77
|
+
- !ruby/object:Gem::Version
|
78
|
+
version: 3.0.5
|
79
|
+
type: :runtime
|
80
|
+
version_requirements: *id006
|
81
|
+
- !ruby/object:Gem::Dependency
|
82
|
+
name: defined
|
83
|
+
prerelease: false
|
84
|
+
requirement: &id007 !ruby/object:Gem::Requirement
|
85
|
+
none: false
|
86
|
+
requirements:
|
87
|
+
- - ~>
|
88
|
+
- !ruby/object:Gem::Version
|
89
|
+
version: 0.0.1
|
90
|
+
type: :runtime
|
91
|
+
version_requirements: *id007
|
92
|
+
description: Minimal sharding solution for AR
|
93
|
+
email:
|
94
|
+
- lane.joshlane@gmail.com
|
95
|
+
executables: []
|
96
|
+
|
97
|
+
extensions: []
|
98
|
+
|
99
|
+
extra_rdoc_files: []
|
100
|
+
|
101
|
+
files:
|
102
|
+
- .gitignore
|
103
|
+
- .rspec
|
104
|
+
- Gemfile
|
105
|
+
- README.md
|
106
|
+
- Rakefile
|
107
|
+
- TODO
|
108
|
+
- lib/nightcrawler.rb
|
109
|
+
- lib/nightcrawler/manager.rb
|
110
|
+
- lib/nightcrawler/migration.rb
|
111
|
+
- lib/nightcrawler/relation.rb
|
112
|
+
- lib/nightcrawler/shard.rb
|
113
|
+
- lib/nightcrawler/shard_descriptor.rb
|
114
|
+
- lib/nightcrawler/version.rb
|
115
|
+
- nightcrawler.gemspec
|
116
|
+
- spec/manager_spec.rb
|
117
|
+
- spec/spec_helper.rb
|
118
|
+
homepage: ""
|
119
|
+
licenses: []
|
120
|
+
|
121
|
+
post_install_message:
|
122
|
+
rdoc_options: []
|
123
|
+
|
124
|
+
require_paths:
|
125
|
+
- lib
|
126
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
127
|
+
none: false
|
128
|
+
requirements:
|
129
|
+
- - ">="
|
130
|
+
- !ruby/object:Gem::Version
|
131
|
+
version: "0"
|
132
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
133
|
+
none: false
|
134
|
+
requirements:
|
135
|
+
- - ">="
|
136
|
+
- !ruby/object:Gem::Version
|
137
|
+
version: "0"
|
138
|
+
requirements: []
|
139
|
+
|
140
|
+
rubyforge_project: nightcrawler
|
141
|
+
rubygems_version: 1.7.2
|
142
|
+
signing_key:
|
143
|
+
specification_version: 3
|
144
|
+
summary: Minimal sharding solution for AR
|
145
|
+
test_files: []
|
146
|
+
|