nightcrawler 0.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
data/.gitignore ADDED
@@ -0,0 +1,4 @@
1
+ *.gem
2
+ .bundle
3
+ Gemfile.lock
4
+ pkg/*
data/.rspec ADDED
@@ -0,0 +1 @@
1
+ --colour
data/Gemfile ADDED
@@ -0,0 +1,4 @@
1
+ source "http://rubygems.org"
2
+
3
+ # Specify your gem's dependencies in nightcrawler.gemspec
4
+ gemspec
data/README.md ADDED
File without changes
data/Rakefile ADDED
@@ -0,0 +1,2 @@
1
+ require 'bundler'
2
+ Bundler::GemHelper.install_tasks
data/TODO ADDED
@@ -0,0 +1,3 @@
1
+ migration connection swap blocks
2
+ support associations
3
+ allow for meta class callbacks
@@ -0,0 +1,13 @@
1
+
2
+ require 'active_record'
3
+ #require File.expand_path(File.join(File.dirname(__FILE__), "nightcrawler", "relation"))
4
+
5
+ module Nightcrawler
6
+ autoload :ShardDescriptor, File.expand_path(File.join(File.dirname(__FILE__), "nightcrawler", "shard_descriptor"))
7
+ autoload :Manager, File.expand_path(File.join(File.dirname(__FILE__), "nightcrawler", "manager"))
8
+ autoload :Shard, File.expand_path(File.join(File.dirname(__FILE__), "nightcrawler", "shard"))
9
+
10
+ # if defined? Rails
11
+ # require File.expand_path(File.join(File.dirname(__FILE__), "nightcrawler", "migration"))
12
+ # end
13
+ end
@@ -0,0 +1,61 @@
1
+ module Nightcrawler::Manager
2
+
3
+ def self.included(klass)
4
+ class << klass
5
+ attr_reader :prefix
6
+ def shards
7
+ Nightcrawler::Manager.mutex.synchronize do
8
+ @shards ||= Nightcrawler::Manager.shards_from_descriptions self, shard_descriptions
9
+ end
10
+ end
11
+
12
+ def shard(key)
13
+ shards[key]
14
+ end
15
+
16
+ def shard_prefix(prefix)
17
+ @prefix = prefix
18
+ end
19
+ end
20
+ end
21
+
22
+ class << self
23
+
24
+ def mutex
25
+ @mutex ||= Mutex.new
26
+ end
27
+
28
+ def shards_from_descriptions(from_class, description_hash)
29
+ Hash[description_hash.collect do |key, config|
30
+ [key, create_shard(from_class, key, config)]
31
+ end]
32
+ end
33
+
34
+ def create_shard(from_class, key, config)
35
+ raise "Could not find shard for key #{key}" unless config
36
+ class_name = "#{from_class.prefix}#{key.to_s.capitalize}"
37
+ klass = Class.new ActiveRecord::Base
38
+ if class_name.split("::").length > 1
39
+ _module = class_name.split("::").first
40
+ _class = class_name.split("::").last
41
+ eval(_module).module_eval{self.const_set _class, klass}
42
+ else
43
+ Object.const_set class_name, klass
44
+ end
45
+ klass.class_eval do
46
+ establish_connection config
47
+ include Nightcrawler::Shard
48
+ class << self
49
+ attr_accessor :shard_id
50
+ end
51
+
52
+ def self.sharded?
53
+ false
54
+ end
55
+ end
56
+ klass.shard_id = key
57
+ klass
58
+ end
59
+ end
60
+ end
61
+
@@ -0,0 +1,20 @@
1
+ module Nightcrawler::Migration
2
+
3
+ def self.extended(base)
4
+ class << base
5
+ def announce_with_nightcrawler(message)
6
+ announce_without_nightcrawler("#{message} - #{shard_key}")
7
+ end
8
+
9
+ alias_method_chain :migrate, :nightcrawler
10
+ alias_method_chain :announce, :nightcrawler
11
+ end
12
+ end
13
+
14
+ def migrate_with_nightcrawler
15
+ end
16
+
17
+ end
18
+
19
+ ActiveRecord::Migration.extend(Nightcrawler::Migration)
20
+
@@ -0,0 +1,21 @@
1
+ require 'active_support/core_ext/module/delegation.rb'
2
+
3
+ ActiveRecord::Relation.class_eval do
4
+
5
+ def set_shard
6
+ if @klass.respond_to?(:sharded?) and @klass.sharded?
7
+ criteria_value = where_values.find{|node| node.left.name == @klass.shard_criteria.to_s rescue false}
8
+ raise "Missing shard critieria" unless criteria_value
9
+ @klass = @klass.shard(@klass.find_shard(criteria_value.right))
10
+ end
11
+ end
12
+
13
+ %W{to_a size}.each do |method|
14
+ eval "alias original_#{method} #{method}"
15
+ define_method method do |*args|
16
+ set_shard
17
+ instance_eval "original_#{method}"
18
+ end
19
+ end
20
+ end
21
+
@@ -0,0 +1,2 @@
1
+ module Nightcrawler::Shard
2
+ end
@@ -0,0 +1,66 @@
1
+ module Nightcrawler::ShardDescriptor
2
+ class << self
3
+ attr_writer :shard_mutex
4
+ def shard_mutex
5
+ @shard_mutex ||= Mutex.new
6
+ end
7
+ end
8
+ def self.included(klass)
9
+ class << klass
10
+ def shard(key)
11
+ Nightcrawler::ShardDescriptor.shard_mutex.synchronize do
12
+ shard_cache[key] ||= create_shard(key)
13
+ end
14
+ end
15
+
16
+ def shard_cache
17
+ @shard_cache ||= {}
18
+ end
19
+
20
+ attr_reader :shard_criteria, :manager, :shard_behavior
21
+
22
+ def shard_behavior(&block)
23
+ @behavior = block
24
+ end
25
+
26
+ def managed_by(manager)
27
+ @manager = manager
28
+ end
29
+
30
+ def defined(file, line, method)
31
+ manager.shard_descriptions.each do |key, config|
32
+ shard_cache[key] = create_shard(key)
33
+ end
34
+ end
35
+
36
+ def shard_by(key)
37
+ @shard_criteria = key
38
+ end
39
+
40
+ def create_shard(key)
41
+ raise "No shard for key: #{key.inspect}" unless manager.shard(key)
42
+ klass = Class.new manager.shard(key)
43
+ if self.name.split("::").length > 1
44
+ class_name = manager.prefix + key.to_s + self.name.split("::").last
45
+ _module = class_name.split("::").first
46
+ _klass = class_name.split("::").last
47
+ eval(_module).module_eval{self.const_set _klass, klass}
48
+ else
49
+ Object.const_set class_name, klass
50
+ end
51
+ klass.class_eval do
52
+ include Nightcrawler::Shard
53
+ class << self
54
+ attr_accessor :shard_id, :class_prefix, :described_by
55
+ end
56
+ end
57
+ klass.shard_id = key
58
+ klass.class_prefix = manager.prefix + key.to_s
59
+ klass.class_eval(&@behavior)
60
+ klass.described_by = self
61
+ klass
62
+ end
63
+ end
64
+ end
65
+ end
66
+
@@ -0,0 +1,3 @@
1
+ module Nightcrawler
2
+ VERSION = "0.0.1"
3
+ end
@@ -0,0 +1,29 @@
1
+ # -*- encoding: utf-8 -*-
2
+ $:.push File.expand_path("../lib", __FILE__)
3
+ require "nightcrawler/version"
4
+
5
+ Gem::Specification.new do |s|
6
+ s.name = "nightcrawler"
7
+ s.version = Nightcrawler::VERSION
8
+ s.platform = Gem::Platform::RUBY
9
+ s.authors = ["Joshua Lane, Michael Prior"]
10
+ s.email = ["lane.joshlane@gmail.com"]
11
+ s.homepage = ""
12
+ s.summary = %q{Minimal sharding solution for AR}
13
+ s.description = %q{Minimal sharding solution for AR}
14
+
15
+ s.rubyforge_project = "nightcrawler"
16
+
17
+ s.files = `git ls-files`.split("\n")
18
+ s.test_files = `git ls-files -- {test,spec,features}/*`.split("\n")
19
+ s.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
20
+ s.require_paths = ["lib"]
21
+
22
+ s.add_development_dependency "rspec", "~> 2.5.0"
23
+ s.add_development_dependency "faker", "~> 0.9.5"
24
+ s.add_development_dependency "mocha", "~> 0.9.12"
25
+ s.add_development_dependency "sqlite3", "~> 1.3.3"
26
+ s.add_development_dependency "autotest"
27
+ s.add_dependency "activerecord", "~> 3.0.5"
28
+ s.add_dependency "defined", "~> 0.0.1"
29
+ end
@@ -0,0 +1,120 @@
1
+ require 'spec_helper'
2
+
3
+
4
+
5
+ $shards = {
6
+ 1 => {:adapter => "sqlite3", :database => File.join(File.dirname(__FILE__), "db", "shard1.db")},
7
+ 2 => {:adapter => "sqlite3", :database => File.join(File.dirname(__FILE__), "db", "shard2.db")},
8
+ 3 => {:adapter => "sqlite3", :database => File.join(File.dirname(__FILE__), "db", "shard3.db")}
9
+ }
10
+ module Database
11
+ class Group
12
+ include Nightcrawler::Manager
13
+
14
+ shard_prefix "Database::Shard"
15
+
16
+ def self.shard_descriptions
17
+ $shards
18
+ end
19
+
20
+ def self.find_shard(value)
21
+ "shard#{value}".to_sym
22
+ end
23
+
24
+ def self.shard_for(key)
25
+ $shards[key]
26
+ end
27
+ end
28
+ end
29
+
30
+ class Database::Post
31
+ include Nightcrawler::ShardDescriptor
32
+ managed_by Database::Group
33
+
34
+ shard_behavior do
35
+ set_table_name "Posts"
36
+
37
+ has_many :comments, :class_name => "#{class_prefix}Comment", :foreign_key => "post_id"
38
+ end
39
+ end
40
+
41
+ class Database::Comment
42
+ include Nightcrawler::ShardDescriptor
43
+ managed_by Database::Group
44
+ shard_by :shard_key
45
+
46
+ shard_behavior do
47
+ set_table_name "Comments"
48
+ validates_presence_of :shard_key
49
+
50
+ belongs_to :post, :class_name => "#{class_prefix}Post"
51
+
52
+ scope :bad_comments, where(:comment => "bad")
53
+ end
54
+
55
+ end
56
+
57
+ describe Nightcrawler::Manager do
58
+
59
+ def clean_dbs
60
+ FileUtils.mkdir File.join(File.dirname(__FILE__), "db") unless File.exists? File.join(File.dirname(__FILE__), "db")
61
+ FileUtils.rm_f Dir[File.join(File.dirname(__FILE__), "db", "*.db")]
62
+ end
63
+
64
+ before(:all) do
65
+ clean_dbs
66
+ $shards.each_value do |config|
67
+ base = ActiveRecord::Base.establish_connection config
68
+ base.connection.create_table "Comments" do |t|
69
+ t.integer :shard_key
70
+ t.string :comment
71
+ t.integer :post_id
72
+ end
73
+ base.connection.create_table "Posts" do |t|
74
+ t.string :url
75
+ end
76
+ end
77
+ end
78
+
79
+ after(:all) do
80
+ clean_dbs
81
+ $shards.each_value do |config|
82
+ FileUtils.rm_f config[:database]
83
+ end
84
+ end
85
+
86
+ it "should create shards" do
87
+ Database::Group.shards.values.should == [Database::Shard1, Database::Shard2, Database::Shard3]
88
+ end
89
+
90
+ it "should create comment shards" do
91
+ Database::Comment.shard(1).to_s.should == "Database::Shard1Comment"
92
+ end
93
+
94
+ it "should respect shard behavior" do
95
+ lambda { Database::Comment.shard(1).create!(:comment => "no shard key") }.should raise_exception(ActiveRecord::RecordInvalid)
96
+ end
97
+
98
+ it "should share connection with shard class" do
99
+ Database::Comment.shard(1).connection.should eql Database::Shard1.connection
100
+ end
101
+
102
+ it "should respect scopes" do
103
+ bad = Database::Comment.shard(1).create!(:shard_key => 1, :comment => "bad")
104
+ good = Database::Comment.shard(1).create!(:shard_key => 1, :comment => "not bad")
105
+ Database::Comment.shard(1).bad_comments.should == [bad]
106
+ Database::Comment.shard(2).bad_comments.should == []
107
+ end
108
+
109
+ it "should respect relationships" do
110
+ respect = Database::Comment.shard(1).create!(:shard_key => 1, :comment => "respect")
111
+ Database::Comment.shard(1).where(:comment => "respect").should == [respect]
112
+ end
113
+
114
+ it "should respect associations" do
115
+ post = Database::Post.shard(1).create!(:url => "blah")
116
+ comment = Database::Comment.shard(1).create!(:shard_key => 1, :post => post)
117
+ post.comments.should == [comment]
118
+ end
119
+ end
120
+
@@ -0,0 +1,16 @@
1
+ require 'rubygems'
2
+ require 'bundler'
3
+
4
+ Bundler.require(:default, :development)
5
+
6
+ Dir[File.join(File.dirname(__FILE__), "support/**/*.rb")].each {|f| require f}
7
+
8
+ require File.join(File.dirname(__FILE__), "../lib/nightcrawler")
9
+ require 'logger'
10
+ FileUtils.mkdir_p File.expand_path(File.join(File.dirname(__FILE__), "..", "log"))
11
+ ActiveRecord::Base.logger = Logger.new(File.join(File.dirname(__FILE__), "../log/test.log"))
12
+
13
+ RSpec.configure do |config|
14
+ config.mock_with :mocha
15
+ end
16
+
metadata ADDED
@@ -0,0 +1,146 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: nightcrawler
3
+ version: !ruby/object:Gem::Version
4
+ prerelease:
5
+ version: 0.0.1
6
+ platform: ruby
7
+ authors:
8
+ - Joshua Lane, Michael Prior
9
+ autorequire:
10
+ bindir: bin
11
+ cert_chain: []
12
+
13
+ date: 2011-05-02 00:00:00 Z
14
+ dependencies:
15
+ - !ruby/object:Gem::Dependency
16
+ name: rspec
17
+ prerelease: false
18
+ requirement: &id001 !ruby/object:Gem::Requirement
19
+ none: false
20
+ requirements:
21
+ - - ~>
22
+ - !ruby/object:Gem::Version
23
+ version: 2.5.0
24
+ type: :development
25
+ version_requirements: *id001
26
+ - !ruby/object:Gem::Dependency
27
+ name: faker
28
+ prerelease: false
29
+ requirement: &id002 !ruby/object:Gem::Requirement
30
+ none: false
31
+ requirements:
32
+ - - ~>
33
+ - !ruby/object:Gem::Version
34
+ version: 0.9.5
35
+ type: :development
36
+ version_requirements: *id002
37
+ - !ruby/object:Gem::Dependency
38
+ name: mocha
39
+ prerelease: false
40
+ requirement: &id003 !ruby/object:Gem::Requirement
41
+ none: false
42
+ requirements:
43
+ - - ~>
44
+ - !ruby/object:Gem::Version
45
+ version: 0.9.12
46
+ type: :development
47
+ version_requirements: *id003
48
+ - !ruby/object:Gem::Dependency
49
+ name: sqlite3
50
+ prerelease: false
51
+ requirement: &id004 !ruby/object:Gem::Requirement
52
+ none: false
53
+ requirements:
54
+ - - ~>
55
+ - !ruby/object:Gem::Version
56
+ version: 1.3.3
57
+ type: :development
58
+ version_requirements: *id004
59
+ - !ruby/object:Gem::Dependency
60
+ name: autotest
61
+ prerelease: false
62
+ requirement: &id005 !ruby/object:Gem::Requirement
63
+ none: false
64
+ requirements:
65
+ - - ">="
66
+ - !ruby/object:Gem::Version
67
+ version: "0"
68
+ type: :development
69
+ version_requirements: *id005
70
+ - !ruby/object:Gem::Dependency
71
+ name: activerecord
72
+ prerelease: false
73
+ requirement: &id006 !ruby/object:Gem::Requirement
74
+ none: false
75
+ requirements:
76
+ - - ~>
77
+ - !ruby/object:Gem::Version
78
+ version: 3.0.5
79
+ type: :runtime
80
+ version_requirements: *id006
81
+ - !ruby/object:Gem::Dependency
82
+ name: defined
83
+ prerelease: false
84
+ requirement: &id007 !ruby/object:Gem::Requirement
85
+ none: false
86
+ requirements:
87
+ - - ~>
88
+ - !ruby/object:Gem::Version
89
+ version: 0.0.1
90
+ type: :runtime
91
+ version_requirements: *id007
92
+ description: Minimal sharding solution for AR
93
+ email:
94
+ - lane.joshlane@gmail.com
95
+ executables: []
96
+
97
+ extensions: []
98
+
99
+ extra_rdoc_files: []
100
+
101
+ files:
102
+ - .gitignore
103
+ - .rspec
104
+ - Gemfile
105
+ - README.md
106
+ - Rakefile
107
+ - TODO
108
+ - lib/nightcrawler.rb
109
+ - lib/nightcrawler/manager.rb
110
+ - lib/nightcrawler/migration.rb
111
+ - lib/nightcrawler/relation.rb
112
+ - lib/nightcrawler/shard.rb
113
+ - lib/nightcrawler/shard_descriptor.rb
114
+ - lib/nightcrawler/version.rb
115
+ - nightcrawler.gemspec
116
+ - spec/manager_spec.rb
117
+ - spec/spec_helper.rb
118
+ homepage: ""
119
+ licenses: []
120
+
121
+ post_install_message:
122
+ rdoc_options: []
123
+
124
+ require_paths:
125
+ - lib
126
+ required_ruby_version: !ruby/object:Gem::Requirement
127
+ none: false
128
+ requirements:
129
+ - - ">="
130
+ - !ruby/object:Gem::Version
131
+ version: "0"
132
+ required_rubygems_version: !ruby/object:Gem::Requirement
133
+ none: false
134
+ requirements:
135
+ - - ">="
136
+ - !ruby/object:Gem::Version
137
+ version: "0"
138
+ requirements: []
139
+
140
+ rubyforge_project: nightcrawler
141
+ rubygems_version: 1.7.2
142
+ signing_key:
143
+ specification_version: 3
144
+ summary: Minimal sharding solution for AR
145
+ test_files: []
146
+