acts_as_muschable 0.1.0
Sign up to get free protection for your applications and to get access to all the features.
- data/.gitignore +3 -0
- data/Rakefile +22 -0
- data/VERSION +1 -0
- data/acts_as_muschable.gemspec +50 -0
- data/init.rb +1 -0
- data/lib/acts_as_muschable.rb +178 -0
- data/rails/init.rb +5 -0
- data/spec/acts_as_muschable/acts_as_muschable_spec.rb +215 -0
- data/spec/acts_as_muschable/integration_spec.rb +139 -0
- data/spec/schema.rb +52 -0
- data/spec/spec.opts +4 -0
- data/spec/spec_helper.rb +47 -0
- metadata +69 -0
data/.gitignore
ADDED
data/Rakefile
ADDED
@@ -0,0 +1,22 @@
|
|
1
|
+
require 'spec/rake/spectask'
|
2
|
+
|
3
|
+
desc 'Default: run specs'
|
4
|
+
task :default => :spec
|
5
|
+
Spec::Rake::SpecTask.new do |t|
|
6
|
+
t.spec_files = FileList["spec/**/*_spec.rb"]
|
7
|
+
end
|
8
|
+
|
9
|
+
begin
|
10
|
+
require 'jeweler'
|
11
|
+
Jeweler::Tasks.new do |gemspec|
|
12
|
+
gemspec.name = "acts_as_muschable"
|
13
|
+
gemspec.summary = "Easy peasy model sharding"
|
14
|
+
gemspec.description = "Do not use me unless you are not you but somebody else."
|
15
|
+
gemspec.email = "jannis@moviepilot.de"
|
16
|
+
gemspec.homepage = "http://github.com/moviepilot/acts_as_muschable"
|
17
|
+
gemspec.authors = ["Moviepilot"]
|
18
|
+
end
|
19
|
+
Jeweler::GemcutterTasks.new
|
20
|
+
rescue LoadError
|
21
|
+
puts "Jeweler not available. Install it with: gem install jeweler"
|
22
|
+
end
|
data/VERSION
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
0.1.0
|
@@ -0,0 +1,50 @@
|
|
1
|
+
# Generated by jeweler
|
2
|
+
# DO NOT EDIT THIS FILE
|
3
|
+
# Instead, edit Jeweler::Tasks in Rakefile, and run `rake gemspec`
|
4
|
+
# -*- encoding: utf-8 -*-
|
5
|
+
|
6
|
+
Gem::Specification.new do |s|
|
7
|
+
s.name = %q{acts_as_muschable}
|
8
|
+
s.version = "0.1.0"
|
9
|
+
|
10
|
+
s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
|
11
|
+
s.authors = ["Moviepilot"]
|
12
|
+
s.date = %q{2010-03-04}
|
13
|
+
s.description = %q{Do not use me unless you are not you but somebody else.}
|
14
|
+
s.email = %q{jannis@moviepilot.de}
|
15
|
+
s.files = [
|
16
|
+
".gitignore",
|
17
|
+
"Rakefile",
|
18
|
+
"VERSION",
|
19
|
+
"acts_as_muschable.gemspec",
|
20
|
+
"init.rb",
|
21
|
+
"lib/acts_as_muschable.rb",
|
22
|
+
"rails/init.rb",
|
23
|
+
"spec/acts_as_muschable/acts_as_muschable_spec.rb",
|
24
|
+
"spec/acts_as_muschable/integration_spec.rb",
|
25
|
+
"spec/schema.rb",
|
26
|
+
"spec/spec.opts",
|
27
|
+
"spec/spec_helper.rb"
|
28
|
+
]
|
29
|
+
s.homepage = %q{http://github.com/moviepilot/acts_as_muschable}
|
30
|
+
s.rdoc_options = ["--charset=UTF-8"]
|
31
|
+
s.require_paths = ["lib"]
|
32
|
+
s.rubygems_version = %q{1.3.5}
|
33
|
+
s.summary = %q{Easy peasy model sharding}
|
34
|
+
s.test_files = [
|
35
|
+
"spec/acts_as_muschable/acts_as_muschable_spec.rb",
|
36
|
+
"spec/acts_as_muschable/integration_spec.rb",
|
37
|
+
"spec/schema.rb",
|
38
|
+
"spec/spec_helper.rb"
|
39
|
+
]
|
40
|
+
|
41
|
+
if s.respond_to? :specification_version then
|
42
|
+
current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
|
43
|
+
s.specification_version = 3
|
44
|
+
|
45
|
+
if Gem::Version.new(Gem::RubyGemsVersion) >= Gem::Version.new('1.2.0') then
|
46
|
+
else
|
47
|
+
end
|
48
|
+
else
|
49
|
+
end
|
50
|
+
end
|
data/init.rb
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
require File.dirname(__FILE__) + "/rails/init"
|
@@ -0,0 +1,178 @@
|
|
1
|
+
#
|
2
|
+
# acts_as_muschable adds support for sharding data over multiple tables by
|
3
|
+
# messing with the #table_name method. If you want to shard your data over
|
4
|
+
# multiple databases, use the DataFabric gem, which does the sharding at
|
5
|
+
# connection level.
|
6
|
+
#
|
7
|
+
module ActiveRecord
|
8
|
+
|
9
|
+
# originally table_name and quoted_table_name are cached both in the model
|
10
|
+
# and in the reflection object. disabling the cache in the reflection object
|
11
|
+
# doesn't mean the table name is recalculated on every access, it only gets
|
12
|
+
# delegated to (the cached) ReflectedModel.table_name every time, meaning
|
13
|
+
# one more method call.
|
14
|
+
module Reflection
|
15
|
+
class AssociationReflection
|
16
|
+
def table_name
|
17
|
+
klass.table_name
|
18
|
+
end
|
19
|
+
|
20
|
+
def quoted_table_name
|
21
|
+
klass.quoted_table_name
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
25
|
+
|
26
|
+
module Acts
|
27
|
+
module Muschable
|
28
|
+
def self.included(base)
|
29
|
+
raise StandardError, "acts_as_muschable is only tested against ActiveRecord -v=2.3.5" if defined?(::Rails) and ::Rails.version>'2.3.5'
|
30
|
+
base.extend(ActsAsMuschableLoader)
|
31
|
+
end
|
32
|
+
|
33
|
+
module ActsAsMuschableLoader
|
34
|
+
def acts_as_muschable(*args)
|
35
|
+
raise RuntimeError, "You called acts_as_muschable twice" unless @class_musched.nil?
|
36
|
+
@class_musched = true
|
37
|
+
extend ClassMethods
|
38
|
+
class_eval do
|
39
|
+
class << self; alias_method_chain :table_name, :shard; end
|
40
|
+
self.shard_amount = args.last[:shard_amount] || 0
|
41
|
+
end
|
42
|
+
end
|
43
|
+
private :acts_as_muschable
|
44
|
+
end
|
45
|
+
|
46
|
+
module ClassMethods
|
47
|
+
|
48
|
+
def initialize_shards
|
49
|
+
0.upto(@shard_amount-1) do |i|
|
50
|
+
connection.execute("CREATE TABLE #{table_name_for_shard(i)} LIKE #{table_name_without_shard}")
|
51
|
+
end
|
52
|
+
end
|
53
|
+
|
54
|
+
@shard_amount = nil
|
55
|
+
attr_reader :shard_amount
|
56
|
+
def shard_amount=(amount)
|
57
|
+
ensure_positive_int('shard_amount', amount)
|
58
|
+
@shard_amount = amount
|
59
|
+
end
|
60
|
+
|
61
|
+
def activate_shard(shard)
|
62
|
+
ensure_positive_int('shard identifier', shard)
|
63
|
+
raise ArgumentError, "Can't activate shard, out of range. Adjust #{self.name}.shard_amount=" unless shard<@shard_amount
|
64
|
+
|
65
|
+
ensure_setup
|
66
|
+
Thread.current[:shards][self.name.to_sym] = shard.to_s
|
67
|
+
end
|
68
|
+
|
69
|
+
def activate_base_shard
|
70
|
+
ensure_setup
|
71
|
+
Thread.current[:shards][self.name.to_sym] = -1
|
72
|
+
end
|
73
|
+
|
74
|
+
def detect_corrupt_shards
|
75
|
+
base_schema = extract_relevant_part_from_schema_definition(table_schema(table_name_without_shard))
|
76
|
+
returning Array.new do |corrupt_shards|
|
77
|
+
0.upto(shard_amount-1) do |shard|
|
78
|
+
shard_schema = extract_relevant_part_from_schema_definition(table_schema(table_name_for_shard(shard)))
|
79
|
+
corrupt_shards << shard if shard_schema!=base_schema or base_schema.blank?
|
80
|
+
end
|
81
|
+
end
|
82
|
+
end
|
83
|
+
|
84
|
+
def drop_shards(amount = nil)
|
85
|
+
amount ||= detect_shard_amount_in_database
|
86
|
+
ensure_positive_int('parameter for #drop_shards', amount)
|
87
|
+
0.upto(amount-1) do |i|
|
88
|
+
connection.execute("DROP TABLE #{table_name_for_shard(i)}")
|
89
|
+
end
|
90
|
+
end
|
91
|
+
|
92
|
+
def table_name_with_shard
|
93
|
+
ensure_setup
|
94
|
+
shard = Thread.current[:shards][self.name.to_sym]
|
95
|
+
|
96
|
+
return table_name_without_shard if @shard_amount==0 or shard == -1
|
97
|
+
raise ArgumentError, 'No shard has been activated' unless shard
|
98
|
+
table_name_for_shard(shard)
|
99
|
+
end
|
100
|
+
|
101
|
+
def detect_shard_amount_in_database
|
102
|
+
result = connection.execute "SHOW TABLES LIKE '#{table_name_without_shard}%'"
|
103
|
+
tables = []
|
104
|
+
result.each do |row|
|
105
|
+
tables << row[0].gsub(table_name_without_shard, '').to_i
|
106
|
+
end
|
107
|
+
result.free
|
108
|
+
return 0 if tables.size<=1
|
109
|
+
tables.sort.last.to_i + 1
|
110
|
+
end
|
111
|
+
|
112
|
+
def shard_levels
|
113
|
+
return [] unless shard_amount > 0
|
114
|
+
levels = []
|
115
|
+
(0...shard_amount).each do |i|
|
116
|
+
result = connection.execute "SELECT COUNT(*) FROM #{table_name_for_shard(i)}"
|
117
|
+
result.each do |row|
|
118
|
+
levels[i] = row[0].to_i
|
119
|
+
end
|
120
|
+
end
|
121
|
+
levels
|
122
|
+
end
|
123
|
+
|
124
|
+
def ensure_setup
|
125
|
+
raise ArgumentError, "You have to set #{self.name}.shard_amount" if @shard_amount.nil?
|
126
|
+
Thread.current[:shards] ||= Hash.new
|
127
|
+
end
|
128
|
+
|
129
|
+
def table_name_for_shard(shard)
|
130
|
+
"#{table_name_without_shard}#{shard}"
|
131
|
+
end
|
132
|
+
|
133
|
+
def ensure_positive_int(name, i)
|
134
|
+
raise ArgumentError, "Only positive integers are allowed as #{name}" unless i.is_a?(Integer) and i>=0
|
135
|
+
end
|
136
|
+
|
137
|
+
def table_schema(table_name)
|
138
|
+
result = connection.execute "SHOW CREATE TABLE #{table_name}"
|
139
|
+
schema = ""
|
140
|
+
result.each do |row|
|
141
|
+
schema << row[1]
|
142
|
+
end
|
143
|
+
schema
|
144
|
+
rescue ActiveRecord::StatementInvalid
|
145
|
+
""
|
146
|
+
ensure
|
147
|
+
result.free if result
|
148
|
+
end
|
149
|
+
|
150
|
+
def extract_relevant_part_from_schema_definition(definition)
|
151
|
+
definition.gsub!(/ AUTO_INCREMENT=[\d]+/, '')
|
152
|
+
match = definition.match(/[^\(]+(.*)$/m)
|
153
|
+
return match[1] if match
|
154
|
+
""
|
155
|
+
end
|
156
|
+
|
157
|
+
#
|
158
|
+
# This is here because ActiveRecord::Base's table_name method
|
159
|
+
# does something funky. If you call ActiveRecord::Base#table_name
|
160
|
+
# for the first time, it goes through the class hierarchy and
|
161
|
+
# puts together a table name.
|
162
|
+
#
|
163
|
+
# Then it redefines the table_name method to instantly return that
|
164
|
+
# name from the first run without going through the class hierarchy
|
165
|
+
# again.
|
166
|
+
#
|
167
|
+
# So this method overwrites :table_name_without_shard instead of
|
168
|
+
# the actual #table_name method, so that the name of the shard is
|
169
|
+
# returned.
|
170
|
+
#
|
171
|
+
def set_table_name(value = nil, &block)
|
172
|
+
define_attr_method :table_name_without_shard, value, &block
|
173
|
+
end
|
174
|
+
|
175
|
+
end
|
176
|
+
end
|
177
|
+
end
|
178
|
+
end
|
data/rails/init.rb
ADDED
@@ -0,0 +1,215 @@
|
|
1
|
+
require File.dirname(__FILE__) + '/../spec_helper'
|
2
|
+
|
3
|
+
describe "Acts as Muschable" do
|
4
|
+
|
5
|
+
before(:all) do
|
6
|
+
Thread.current[:shards] = nil
|
7
|
+
end
|
8
|
+
|
9
|
+
describe "changing active shards" do
|
10
|
+
it "should throw an exception when MuschableModel.table_name is called before MuschableModel.set_shard" do
|
11
|
+
lambda{
|
12
|
+
MuschableModel.table_name
|
13
|
+
}.should raise_error(ArgumentError, 'No shard has been activated')
|
14
|
+
end
|
15
|
+
|
16
|
+
it "should not affect UnMuschableModels" do
|
17
|
+
UnmuschableModel.table_name.should == "unmuschable_models"
|
18
|
+
end
|
19
|
+
|
20
|
+
it "should be able to MuschableModel.activate_shard" do
|
21
|
+
MuschableModel.should respond_to(:activate_shard)
|
22
|
+
end
|
23
|
+
|
24
|
+
it "should be able to MuschableModel.activate_base_shard" do
|
25
|
+
MuschableModel.should respond_to(:activate_base_shard)
|
26
|
+
end
|
27
|
+
|
28
|
+
it "should return the correct MuschableModel.table_name according to shard" do
|
29
|
+
MuschableModel.activate_shard(0)
|
30
|
+
MuschableModel.table_name.should == "muschable_models0"
|
31
|
+
MuschableModel.activate_shard(1)
|
32
|
+
MuschableModel.table_name.should == "muschable_models1"
|
33
|
+
MuschableModel.activate_base_shard
|
34
|
+
MuschableModel.table_name.should == "muschable_models"
|
35
|
+
end
|
36
|
+
|
37
|
+
it "should only accept positive numeric shard identifiers" do
|
38
|
+
lambda{
|
39
|
+
MuschableModel.activate_shard("0")
|
40
|
+
}.should raise_error(ArgumentError, 'Only positive integers are allowed as shard identifier')
|
41
|
+
lambda{
|
42
|
+
MuschableModel.activate_shard(-1)
|
43
|
+
}.should raise_error(ArgumentError, 'Only positive integers are allowed as shard identifier')
|
44
|
+
end
|
45
|
+
|
46
|
+
it "should be somewhat thread safe" do
|
47
|
+
MuschableModel.shard_amount = 1_000_000
|
48
|
+
threads = []
|
49
|
+
300.times do
|
50
|
+
threads << Thread.new do
|
51
|
+
shard = rand(1_000_000)
|
52
|
+
MuschableModel.activate_shard(shard)
|
53
|
+
sleep(rand(10))
|
54
|
+
MuschableModel.table_name.should == "muschable_models#{shard}"
|
55
|
+
MuschableModel.activate_base_shard
|
56
|
+
sleep(rand(5))
|
57
|
+
MuschableModel.table_name.should == "muschable_models"
|
58
|
+
end
|
59
|
+
end
|
60
|
+
threads.each { |thread| thread.join }
|
61
|
+
end
|
62
|
+
end
|
63
|
+
|
64
|
+
describe "managing the amount of shards" do
|
65
|
+
it "should have a method to set MuschableModel.shard_amount=" do
|
66
|
+
MuschableModel.shard_amount = 1
|
67
|
+
MuschableModel.shard_amount.should == 1
|
68
|
+
MuschableModel.shard_amount = 15
|
69
|
+
MuschableModel.shard_amount.should == 15
|
70
|
+
end
|
71
|
+
|
72
|
+
it "should do no sharding when MuschableModel.shard_amount is set to 0 (useful for test environments and such)" do
|
73
|
+
MuschableModel.shard_amount = 0
|
74
|
+
MuschableModel.table_name.should == "muschable_models"
|
75
|
+
end
|
76
|
+
|
77
|
+
it "should not accept non-integers as MuschableModel.shard_amount=" do
|
78
|
+
lambda{
|
79
|
+
MuschableModel.shard_amount = "15"
|
80
|
+
}.should raise_error(ArgumentError, 'Only positive integers are allowed as shard_amount')
|
81
|
+
lambda{
|
82
|
+
MuschableModel.shard_amount = -1
|
83
|
+
}.should raise_error(ArgumentError, 'Only positive integers are allowed as shard_amount')
|
84
|
+
end
|
85
|
+
|
86
|
+
it "should not accept a shard identifier larger MuschableModel.shard_amount" do
|
87
|
+
MuschableModel.shard_amount = 16
|
88
|
+
lambda{
|
89
|
+
MuschableModel.activate_shard(16)
|
90
|
+
}.should raise_error(ArgumentError, "Can't activate shard, out of range. Adjust MuschableModel.shard_amount=")
|
91
|
+
end
|
92
|
+
|
93
|
+
it "should have a method MuschableModel.initialize_shards to create MuschableModel.shard_amount new shards" do
|
94
|
+
MuschableModel.activate_shard(0)
|
95
|
+
connection = mock("Connection")
|
96
|
+
connection.stub!(:table_exists?).with(any_args).and_return true
|
97
|
+
MuschableModel.should_receive(:connection).exactly(16).and_return connection
|
98
|
+
|
99
|
+
0.upto(15) do |i|
|
100
|
+
query = "CREATE TABLE muschable_models#{i} LIKE muschable_models"
|
101
|
+
connection.should_receive(:execute).with(query).once.and_return(mock("execute #{i}", :null_object => true))
|
102
|
+
end
|
103
|
+
|
104
|
+
MuschableModel.initialize_shards
|
105
|
+
end
|
106
|
+
|
107
|
+
it "should have a method MuschableModel.drop_shards(12) to drop shards 0...12" do
|
108
|
+
MuschableModel.activate_shard(0)
|
109
|
+
connection = mock("Connection")
|
110
|
+
connection.stub!(:table_exists?).with(any_args).and_return true
|
111
|
+
MuschableModel.should_receive(:connection).exactly(12).and_return connection
|
112
|
+
|
113
|
+
0.upto(11) do |i|
|
114
|
+
query = "DROP TABLE muschable_models#{i}"
|
115
|
+
connection.should_receive(:execute).with(query).once.and_return(mock("execute #{i}", :null_object => true))
|
116
|
+
end
|
117
|
+
|
118
|
+
MuschableModel.drop_shards(12)
|
119
|
+
end
|
120
|
+
|
121
|
+
[-1, "1", "a"].each do |i|
|
122
|
+
it "should not allow #drop_shards(#{i})" do
|
123
|
+
lambda{
|
124
|
+
MuschableModel.drop_shards(i)
|
125
|
+
}.should raise_error(ArgumentError, 'Only positive integers are allowed as parameter for #drop_shards')
|
126
|
+
end
|
127
|
+
end
|
128
|
+
|
129
|
+
it "should have a method MuschableModel.assure_shards_health that goes through all shards and makes sure their structure equals that of the base table" do
|
130
|
+
MuschableModel.should respond_to(:detect_corrupt_shards)
|
131
|
+
end
|
132
|
+
end
|
133
|
+
|
134
|
+
describe "utility methods" do
|
135
|
+
|
136
|
+
it "should have a method :shard_level to deliver statistics about how many rows each shard has" do
|
137
|
+
MuschableModel.should respond_to(:shard_levels)
|
138
|
+
end
|
139
|
+
|
140
|
+
it "should extract the relevant parts from a schema definition in order to compare definitions" do
|
141
|
+
base_table_definition =<<-SQL
|
142
|
+
CREATE TABLE `movies_users` (
|
143
|
+
`user_id` int(11) DEFAULT NULL,
|
144
|
+
`movie_id` int(11) DEFAULT NULL,
|
145
|
+
`rating_date` datetime DEFAULT NULL,
|
146
|
+
`rating` int(3) DEFAULT NULL,
|
147
|
+
`top` tinyint(1) NOT NULL DEFAULT '0',
|
148
|
+
`flop` tinyint(1) NOT NULL DEFAULT '0',
|
149
|
+
`id` int(11) NOT NULL AUTO_INCREMENT,
|
150
|
+
`watchlist` tinyint(1) NOT NULL DEFAULT '0',
|
151
|
+
`blacklist` tinyint(1) NOT NULL DEFAULT '0',
|
152
|
+
`watchlist_date` datetime DEFAULT NULL,
|
153
|
+
`blacklist_date` datetime DEFAULT NULL,
|
154
|
+
`forecast` float DEFAULT NULL,
|
155
|
+
`forecast_relevance` float DEFAULT NULL,
|
156
|
+
`forecast_neighbour_count` int(4) DEFAULT NULL,
|
157
|
+
`forecast_or_rating` int(3) DEFAULT NULL,
|
158
|
+
PRIMARY KEY (`id`),
|
159
|
+
UNIQUE KEY `index_movies_users_on_user_id_and_movie_id` (`user_id`,`movie_id`),
|
160
|
+
KEY `user_ratings` (`movie_id`,`user_id`,`rating`),
|
161
|
+
KEY `index_movies_users_on_updated_at_and_id` (`rating_date`,`id`),
|
162
|
+
KEY `movie_id_and_rating_date` (`movie_id`,`rating_date`),
|
163
|
+
KEY `movie_id_and_user_id_and_forecast_or_rating` (`movie_id`,`user_id`,`forecast_or_rating`),
|
164
|
+
KEY `rating_date` (`rating_date`),
|
165
|
+
KEY `top` (`top`),
|
166
|
+
KEY `flop` (`flop`),
|
167
|
+
KEY `index_movies_users_on_user_id_and_blacklist_and_movie_id` (`user_id`,`blacklist`,`movie_id`),
|
168
|
+
KEY `movies_users_on_user_id_and_rating_and_movie_id` (`user_id`,`rating`,`movie_id`),
|
169
|
+
KEY `movies_users_on_movie_id_and_rating` (`movie_id`,`rating`),
|
170
|
+
KEY `movies_users_on_rating_and_user_id_and_movie_id` (`rating`,`user_id`,`movie_id`),
|
171
|
+
KEY `user_id_and_movie_id_and_forecast_or_rating_on_movies_users` (`user_id`,`movie_id`,`forecast_or_rating`)
|
172
|
+
) ENGINE=MyISAM AUTO_INCREMENT=241855247 DEFAULT CHARSET=utf8
|
173
|
+
SQL
|
174
|
+
|
175
|
+
shard_table_definition =<<-SQL
|
176
|
+
CREATE TABLE `movies_users0` (
|
177
|
+
`user_id` int(11) DEFAULT NULL,
|
178
|
+
`movie_id` int(11) DEFAULT NULL,
|
179
|
+
`rating_date` datetime DEFAULT NULL,
|
180
|
+
`rating` int(3) DEFAULT NULL,
|
181
|
+
`top` tinyint(1) NOT NULL DEFAULT '0',
|
182
|
+
`flop` tinyint(1) NOT NULL DEFAULT '0',
|
183
|
+
`id` int(11) NOT NULL AUTO_INCREMENT,
|
184
|
+
`watchlist` tinyint(1) NOT NULL DEFAULT '0',
|
185
|
+
`blacklist` tinyint(1) NOT NULL DEFAULT '0',
|
186
|
+
`watchlist_date` datetime DEFAULT NULL,
|
187
|
+
`blacklist_date` datetime DEFAULT NULL,
|
188
|
+
`forecast` float DEFAULT NULL,
|
189
|
+
`forecast_relevance` float DEFAULT NULL,
|
190
|
+
`forecast_neighbour_count` int(4) DEFAULT NULL,
|
191
|
+
`forecast_or_rating` int(3) DEFAULT NULL,
|
192
|
+
PRIMARY KEY (`id`),
|
193
|
+
UNIQUE KEY `index_movies_users_on_user_id_and_movie_id` (`user_id`,`movie_id`),
|
194
|
+
KEY `user_ratings` (`movie_id`,`user_id`,`rating`),
|
195
|
+
KEY `index_movies_users_on_updated_at_and_id` (`rating_date`,`id`),
|
196
|
+
KEY `movie_id_and_rating_date` (`movie_id`,`rating_date`),
|
197
|
+
KEY `movie_id_and_user_id_and_forecast_or_rating` (`movie_id`,`user_id`,`forecast_or_rating`),
|
198
|
+
KEY `rating_date` (`rating_date`),
|
199
|
+
KEY `top` (`top`),
|
200
|
+
KEY `flop` (`flop`),
|
201
|
+
KEY `index_movies_users_on_user_id_and_blacklist_and_movie_id` (`user_id`,`blacklist`,`movie_id`),
|
202
|
+
KEY `movies_users_on_user_id_and_rating_and_movie_id` (`user_id`,`rating`,`movie_id`),
|
203
|
+
KEY `movies_users_on_movie_id_and_rating` (`movie_id`,`rating`),
|
204
|
+
KEY `movies_users_on_rating_and_user_id_and_movie_id` (`rating`,`user_id`,`movie_id`),
|
205
|
+
KEY `user_id_and_movie_id_and_forecast_or_rating_on_movies_users` (`user_id`,`movie_id`,`forecast_or_rating`)
|
206
|
+
) ENGINE=MyISAM DEFAULT CHARSET=utf8
|
207
|
+
SQL
|
208
|
+
|
209
|
+
base_definition = MuschableModel.extract_relevant_part_from_schema_definition(base_table_definition)
|
210
|
+
shard_definition = MuschableModel.extract_relevant_part_from_schema_definition(shard_table_definition)
|
211
|
+
|
212
|
+
shard_definition.should == base_definition
|
213
|
+
end
|
214
|
+
end
|
215
|
+
end
|
@@ -0,0 +1,139 @@
|
|
1
|
+
require File.dirname(__FILE__) + '/../spec_helper'
|
2
|
+
|
3
|
+
describe "Integration tests with database access" do
|
4
|
+
|
5
|
+
unless ENV['RUN_MUSCH_INTEGRATION_TESTS']=="true"
|
6
|
+
print <<-HELP
|
7
|
+
<pre>==================== INTEGRATION TESTS ===========================
|
8
|
+
If you want to run functional tests on your DB, set
|
9
|
+
ENV['RUN_MUSCH_INTEGRATION_TESTS'] to 'true' and call rake spec again.
|
10
|
+
|
11
|
+
Then make sure the mysql user rails:rails@localhost has all mysql
|
12
|
+
privileges for database 'test' (should work out of the box if you
|
13
|
+
have a rails user in mysql)
|
14
|
+
=================================================================</pre>
|
15
|
+
HELP
|
16
|
+
break
|
17
|
+
end
|
18
|
+
|
19
|
+
describe "shard management" do
|
20
|
+
|
21
|
+
before(:all) do
|
22
|
+
Thread.current[:shards] = nil
|
23
|
+
puts "\nRunning integration tests"
|
24
|
+
@conn = MuschableModel.connection
|
25
|
+
end
|
26
|
+
|
27
|
+
it "should detect how many must be deleted with #detect_shard_amount_in_database when there are shards" do
|
28
|
+
MuschableModel.detect_shard_amount_in_database.should == 4
|
29
|
+
end
|
30
|
+
|
31
|
+
it "should detect how many must be deleted with #detect_shard_amount_in_database when there are no shards" do
|
32
|
+
OtherMuschableModel.detect_shard_amount_in_database.should == 0
|
33
|
+
end
|
34
|
+
|
35
|
+
it "should detect corrupt shards during #detect_corrupt_shards" do
|
36
|
+
MuschableModel.shard_amount = 4
|
37
|
+
MuschableModel.detect_corrupt_shards.should == [1,2,3]
|
38
|
+
end
|
39
|
+
|
40
|
+
it "should not detect corrupt shards when database is healthy" do
|
41
|
+
MuschableModel.shard_amount = 1
|
42
|
+
MuschableModel.detect_corrupt_shards.should be_blank
|
43
|
+
end
|
44
|
+
|
45
|
+
it "should drop all shards during #drop_shards(3)" do
|
46
|
+
class YetAnotherMuschableModel < ActiveRecord::Base
|
47
|
+
acts_as_muschable :shard_amount => 5
|
48
|
+
end
|
49
|
+
create_base_table_and_shards "yet_another_muschable_models", 10
|
50
|
+
YetAnotherMuschableModel.detect_shard_amount_in_database.should == 10
|
51
|
+
YetAnotherMuschableModel.drop_shards(10)
|
52
|
+
YetAnotherMuschableModel.detect_shard_amount_in_database.should == 0
|
53
|
+
end
|
54
|
+
|
55
|
+
it "should drop all shards during #drop_shards (automatically guessing how many shards exist)" do
|
56
|
+
class YetYetAnotherMuschableModel < ActiveRecord::Base
|
57
|
+
acts_as_muschable :shard_amount => 5
|
58
|
+
end
|
59
|
+
create_base_table_and_shards "yet_yet_another_muschable_models", 15
|
60
|
+
YetYetAnotherMuschableModel.activate_shard 0
|
61
|
+
YetYetAnotherMuschableModel.detect_shard_amount_in_database.should == 15
|
62
|
+
YetYetAnotherMuschableModel.drop_shards
|
63
|
+
YetYetAnotherMuschableModel.detect_shard_amount_in_database.should == 0
|
64
|
+
end
|
65
|
+
|
66
|
+
it "should create all shards during #initialize_shards" do
|
67
|
+
OtherMuschableModel.detect_shard_amount_in_database.should == 0
|
68
|
+
OtherMuschableModel.shard_amount = 3
|
69
|
+
OtherMuschableModel.initialize_shards
|
70
|
+
|
71
|
+
OtherMuschableModel.detect_shard_amount_in_database.should == 3
|
72
|
+
OtherMuschableModel.detect_corrupt_shards.should be_blank
|
73
|
+
end
|
74
|
+
|
75
|
+
end
|
76
|
+
|
77
|
+
describe "objects living in different shards" do
|
78
|
+
|
79
|
+
before(:all) do
|
80
|
+
class MyMuschableModel < ActiveRecord::Base
|
81
|
+
acts_as_muschable :shard_amount => 2
|
82
|
+
end
|
83
|
+
create_base_table_and_shards "my_muschable_models", 0
|
84
|
+
MyMuschableModel.initialize_shards
|
85
|
+
@conn = MuschableModel.connection
|
86
|
+
end
|
87
|
+
|
88
|
+
it "should create different models in different shards" do
|
89
|
+
MyMuschableModel.activate_shard 0
|
90
|
+
model1_in_shard0 = MyMuschableModel.create :name => "model1_in_shard0"
|
91
|
+
model2_in_shard0 = MyMuschableModel.create :name => "model2_in_shard0"
|
92
|
+
MyMuschableModel.count.should == 2
|
93
|
+
MyMuschableModel.find(model2_in_shard0.id).should == model2_in_shard0
|
94
|
+
|
95
|
+
MyMuschableModel.activate_shard 1
|
96
|
+
model1_in_shard1 = MyMuschableModel.create :name => "model1_in_shard1"
|
97
|
+
MyMuschableModel.count.should == 1
|
98
|
+
MyMuschableModel.destroy_all
|
99
|
+
MyMuschableModel.count.should == 0
|
100
|
+
|
101
|
+
MyMuschableModel.activate_shard 0
|
102
|
+
MyMuschableModel.count.should == 2
|
103
|
+
end
|
104
|
+
|
105
|
+
end
|
106
|
+
|
107
|
+
#
|
108
|
+
# Drop all tables created during this test (i.e. have muschable in their name)
|
109
|
+
after(:all) do
|
110
|
+
@conn.execute "DROP TABLE schema_migrations"
|
111
|
+
result = @conn.execute "SHOW TABLES LIKE '%muschable%'"
|
112
|
+
result.each do |row|
|
113
|
+
@conn.execute("DROP TABLE #{row[0]}")
|
114
|
+
end
|
115
|
+
result.free
|
116
|
+
end
|
117
|
+
end
|
118
|
+
|
119
|
+
|
120
|
+
#
|
121
|
+
# Santa's little helper
|
122
|
+
#
|
123
|
+
def create_base_table_and_shards(table_name, shard_amount)
|
124
|
+
ActiveRecord::Schema.define :version => 0 do
|
125
|
+
create_table table_name, :force => true do |t|
|
126
|
+
t.integer "id", :limit => 11
|
127
|
+
t.string "name"
|
128
|
+
end
|
129
|
+
add_index table_name, ["id"], :name => "index_on_id"
|
130
|
+
|
131
|
+
0.upto(shard_amount-1) do |i|
|
132
|
+
create_table "#{table_name}#{i}", :force => true do |t|
|
133
|
+
t.integer "id", :limit => 11
|
134
|
+
t.string "name"
|
135
|
+
end
|
136
|
+
add_index "#{table_name}#{i}", ["id"], :name => "index_on_id"
|
137
|
+
end
|
138
|
+
end
|
139
|
+
end
|
data/spec/schema.rb
ADDED
@@ -0,0 +1,52 @@
|
|
1
|
+
RAILS_DEFAULT_LOGGER.silence do
|
2
|
+
ActiveRecord::Schema.define :version => 0 do
|
3
|
+
|
4
|
+
#
|
5
|
+
# Base table
|
6
|
+
#
|
7
|
+
create_table "muschable_models", :force => true do |t|
|
8
|
+
t.integer "id", :limit => 11
|
9
|
+
t.string "name"
|
10
|
+
end
|
11
|
+
add_index "muschable_models", ["id"], :name => "index_on_id"
|
12
|
+
|
13
|
+
#
|
14
|
+
# First shard, healthy
|
15
|
+
#
|
16
|
+
create_table "muschable_models0", :force => true do |t|
|
17
|
+
t.integer "id", :limit => 11
|
18
|
+
t.string "name"
|
19
|
+
end
|
20
|
+
add_index "muschable_models0", ["id"], :name => "index_on_id"
|
21
|
+
|
22
|
+
#
|
23
|
+
# Second shard, wrong column
|
24
|
+
#
|
25
|
+
create_table "muschable_models1", :force => true do |t|
|
26
|
+
t.integer "id", :limit => 11
|
27
|
+
t.string "nume"
|
28
|
+
end
|
29
|
+
add_index "muschable_models1", ["id"], :name => "index_on_id"
|
30
|
+
|
31
|
+
#
|
32
|
+
# Third shard, missing in action
|
33
|
+
#
|
34
|
+
|
35
|
+
#
|
36
|
+
# Fourth shard, missing index
|
37
|
+
#
|
38
|
+
create_table "muschable_models3", :force => true do |t|
|
39
|
+
t.integer "id", :limit => 11
|
40
|
+
t.string "name"
|
41
|
+
end
|
42
|
+
|
43
|
+
#
|
44
|
+
# Another base shard
|
45
|
+
#
|
46
|
+
create_table "other_muschable_models", :force => true do |t|
|
47
|
+
t.integer "id", :limit => 11
|
48
|
+
t.string "name"
|
49
|
+
end
|
50
|
+
add_index "other_muschable_models", ["id"], :name => "index_on_id"
|
51
|
+
end
|
52
|
+
end
|
data/spec/spec.opts
ADDED
data/spec/spec_helper.rb
ADDED
@@ -0,0 +1,47 @@
|
|
1
|
+
ENV["RAILS_ENV"] = "test"
|
2
|
+
|
3
|
+
require 'rubygems'
|
4
|
+
require 'active_support'
|
5
|
+
require 'active_record'
|
6
|
+
require 'spec'
|
7
|
+
|
8
|
+
RAILS_DEFAULT_LOGGER = Logger.new(File.join(File.dirname(__FILE__), "debug.log"))
|
9
|
+
|
10
|
+
$: << File.join(File.dirname(__FILE__), '..', 'lib')
|
11
|
+
require File.join(File.dirname(__FILE__), '..', 'init')
|
12
|
+
|
13
|
+
if ENV['RUN_MUSCH_INTEGRATION_TESTS']=="true"
|
14
|
+
ActiveRecord::Base.establish_connection(
|
15
|
+
"adapter" => "mysql",
|
16
|
+
"database" => "test",
|
17
|
+
"host" => "localhost",
|
18
|
+
"socket" => "/tmp/mysql.sock",
|
19
|
+
"port" => 3306,
|
20
|
+
"username" => "rails",
|
21
|
+
"password" => "rails"
|
22
|
+
)
|
23
|
+
|
24
|
+
load(File.dirname(__FILE__) + '/schema.rb')
|
25
|
+
end
|
26
|
+
|
27
|
+
class MuschableModel < ActiveRecord::Base
|
28
|
+
acts_as_muschable :shard_amount => 16
|
29
|
+
end
|
30
|
+
|
31
|
+
class OtherMuschableModel < ActiveRecord::Base
|
32
|
+
acts_as_muschable :shard_amount => 16
|
33
|
+
end
|
34
|
+
|
35
|
+
class UnmuschableModel < ActiveRecord::Base
|
36
|
+
end
|
37
|
+
|
38
|
+
# This is here so the schema definitions don't produce too much output
|
39
|
+
BEGIN {
|
40
|
+
class<<Object
|
41
|
+
def puts_with_crap_cleansing(msg)
|
42
|
+
puts_without_crap_cleansing(msg) if ['-- ', ' ->'].reject{|⎮| msg.starts_with?(⎮)}.count == 2
|
43
|
+
end
|
44
|
+
alias_method :puts_without_crap_cleansing, :puts
|
45
|
+
alias_method :puts, :puts_with_crap_cleansing
|
46
|
+
end
|
47
|
+
}
|
metadata
ADDED
@@ -0,0 +1,69 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: acts_as_muschable
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.1.0
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Moviepilot
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
|
12
|
+
date: 2010-03-04 00:00:00 +01:00
|
13
|
+
default_executable:
|
14
|
+
dependencies: []
|
15
|
+
|
16
|
+
description: Do not use me unless you are not you but somebody else.
|
17
|
+
email: jannis@moviepilot.de
|
18
|
+
executables: []
|
19
|
+
|
20
|
+
extensions: []
|
21
|
+
|
22
|
+
extra_rdoc_files: []
|
23
|
+
|
24
|
+
files:
|
25
|
+
- .gitignore
|
26
|
+
- Rakefile
|
27
|
+
- VERSION
|
28
|
+
- acts_as_muschable.gemspec
|
29
|
+
- init.rb
|
30
|
+
- lib/acts_as_muschable.rb
|
31
|
+
- rails/init.rb
|
32
|
+
- spec/acts_as_muschable/acts_as_muschable_spec.rb
|
33
|
+
- spec/acts_as_muschable/integration_spec.rb
|
34
|
+
- spec/schema.rb
|
35
|
+
- spec/spec.opts
|
36
|
+
- spec/spec_helper.rb
|
37
|
+
has_rdoc: true
|
38
|
+
homepage: http://github.com/moviepilot/acts_as_muschable
|
39
|
+
licenses: []
|
40
|
+
|
41
|
+
post_install_message:
|
42
|
+
rdoc_options:
|
43
|
+
- --charset=UTF-8
|
44
|
+
require_paths:
|
45
|
+
- lib
|
46
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
47
|
+
requirements:
|
48
|
+
- - ">="
|
49
|
+
- !ruby/object:Gem::Version
|
50
|
+
version: "0"
|
51
|
+
version:
|
52
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
53
|
+
requirements:
|
54
|
+
- - ">="
|
55
|
+
- !ruby/object:Gem::Version
|
56
|
+
version: "0"
|
57
|
+
version:
|
58
|
+
requirements: []
|
59
|
+
|
60
|
+
rubyforge_project:
|
61
|
+
rubygems_version: 1.3.5
|
62
|
+
signing_key:
|
63
|
+
specification_version: 3
|
64
|
+
summary: Easy peasy model sharding
|
65
|
+
test_files:
|
66
|
+
- spec/acts_as_muschable/acts_as_muschable_spec.rb
|
67
|
+
- spec/acts_as_muschable/integration_spec.rb
|
68
|
+
- spec/schema.rb
|
69
|
+
- spec/spec_helper.rb
|