acts_as_muschable 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/.gitignore +3 -0
- data/Rakefile +22 -0
- data/VERSION +1 -0
- data/acts_as_muschable.gemspec +50 -0
- data/init.rb +1 -0
- data/lib/acts_as_muschable.rb +178 -0
- data/rails/init.rb +5 -0
- data/spec/acts_as_muschable/acts_as_muschable_spec.rb +215 -0
- data/spec/acts_as_muschable/integration_spec.rb +139 -0
- data/spec/schema.rb +52 -0
- data/spec/spec.opts +4 -0
- data/spec/spec_helper.rb +47 -0
- metadata +69 -0
data/.gitignore
ADDED
data/Rakefile
ADDED
@@ -0,0 +1,22 @@
|
|
1
|
+
require 'spec/rake/spectask'
|
2
|
+
|
3
|
+
desc 'Default: run specs'
|
4
|
+
task :default => :spec
|
5
|
+
Spec::Rake::SpecTask.new do |t|
|
6
|
+
t.spec_files = FileList["spec/**/*_spec.rb"]
|
7
|
+
end
|
8
|
+
|
9
|
+
begin
|
10
|
+
require 'jeweler'
|
11
|
+
Jeweler::Tasks.new do |gemspec|
|
12
|
+
gemspec.name = "acts_as_muschable"
|
13
|
+
gemspec.summary = "Easy peasy model sharding"
|
14
|
+
gemspec.description = "Do not use me unless you are not you but somebody else."
|
15
|
+
gemspec.email = "jannis@moviepilot.de"
|
16
|
+
gemspec.homepage = "http://github.com/moviepilot/acts_as_muschable"
|
17
|
+
gemspec.authors = ["Moviepilot"]
|
18
|
+
end
|
19
|
+
Jeweler::GemcutterTasks.new
|
20
|
+
rescue LoadError
|
21
|
+
puts "Jeweler not available. Install it with: gem install jeweler"
|
22
|
+
end
|
data/VERSION
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
0.1.0
|
@@ -0,0 +1,50 @@
|
|
1
|
+
# Generated by jeweler
|
2
|
+
# DO NOT EDIT THIS FILE
|
3
|
+
# Instead, edit Jeweler::Tasks in Rakefile, and run `rake gemspec`
|
4
|
+
# -*- encoding: utf-8 -*-
|
5
|
+
|
6
|
+
Gem::Specification.new do |s|
|
7
|
+
s.name = %q{acts_as_muschable}
|
8
|
+
s.version = "0.1.0"
|
9
|
+
|
10
|
+
s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
|
11
|
+
s.authors = ["Moviepilot"]
|
12
|
+
s.date = %q{2010-03-04}
|
13
|
+
s.description = %q{Do not use me unless you are not you but somebody else.}
|
14
|
+
s.email = %q{jannis@moviepilot.de}
|
15
|
+
s.files = [
|
16
|
+
".gitignore",
|
17
|
+
"Rakefile",
|
18
|
+
"VERSION",
|
19
|
+
"acts_as_muschable.gemspec",
|
20
|
+
"init.rb",
|
21
|
+
"lib/acts_as_muschable.rb",
|
22
|
+
"rails/init.rb",
|
23
|
+
"spec/acts_as_muschable/acts_as_muschable_spec.rb",
|
24
|
+
"spec/acts_as_muschable/integration_spec.rb",
|
25
|
+
"spec/schema.rb",
|
26
|
+
"spec/spec.opts",
|
27
|
+
"spec/spec_helper.rb"
|
28
|
+
]
|
29
|
+
s.homepage = %q{http://github.com/moviepilot/acts_as_muschable}
|
30
|
+
s.rdoc_options = ["--charset=UTF-8"]
|
31
|
+
s.require_paths = ["lib"]
|
32
|
+
s.rubygems_version = %q{1.3.5}
|
33
|
+
s.summary = %q{Easy peasy model sharding}
|
34
|
+
s.test_files = [
|
35
|
+
"spec/acts_as_muschable/acts_as_muschable_spec.rb",
|
36
|
+
"spec/acts_as_muschable/integration_spec.rb",
|
37
|
+
"spec/schema.rb",
|
38
|
+
"spec/spec_helper.rb"
|
39
|
+
]
|
40
|
+
|
41
|
+
if s.respond_to? :specification_version then
|
42
|
+
current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
|
43
|
+
s.specification_version = 3
|
44
|
+
|
45
|
+
if Gem::Version.new(Gem::RubyGemsVersion) >= Gem::Version.new('1.2.0') then
|
46
|
+
else
|
47
|
+
end
|
48
|
+
else
|
49
|
+
end
|
50
|
+
end
|
data/init.rb
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
require File.dirname(__FILE__) + "/rails/init"
|
@@ -0,0 +1,178 @@
|
|
1
|
+
#
|
2
|
+
# acts_as_muschable adds support for sharding data over multiple tables by
|
3
|
+
# messing with the #table_name method. If you want to shard your data over
|
4
|
+
# multiple databases, use the DataFabric gem, which does the sharding at
|
5
|
+
# connection level.
|
6
|
+
#
|
7
|
+
module ActiveRecord
|
8
|
+
|
9
|
+
# originally table_name and quoted_table_name are cached both in the model
|
10
|
+
# and in the reflection object. disabling the cache in the reflection object
|
11
|
+
# doesn't mean the table name is recalculated on every access, it only gets
|
12
|
+
# delegated to (the cached) ReflectedModel.table_name every time, meaning
|
13
|
+
# one more method call.
|
14
|
+
module Reflection
|
15
|
+
class AssociationReflection
|
16
|
+
def table_name
|
17
|
+
klass.table_name
|
18
|
+
end
|
19
|
+
|
20
|
+
def quoted_table_name
|
21
|
+
klass.quoted_table_name
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
25
|
+
|
26
|
+
module Acts
|
27
|
+
module Muschable
|
28
|
+
def self.included(base)
|
29
|
+
raise StandardError, "acts_as_muschable is only tested against ActiveRecord -v=2.3.5" if defined?(::Rails) and ::Rails.version>'2.3.5'
|
30
|
+
base.extend(ActsAsMuschableLoader)
|
31
|
+
end
|
32
|
+
|
33
|
+
module ActsAsMuschableLoader
|
34
|
+
def acts_as_muschable(*args)
|
35
|
+
raise RuntimeError, "You called acts_as_muschable twice" unless @class_musched.nil?
|
36
|
+
@class_musched = true
|
37
|
+
extend ClassMethods
|
38
|
+
class_eval do
|
39
|
+
class << self; alias_method_chain :table_name, :shard; end
|
40
|
+
self.shard_amount = args.last[:shard_amount] || 0
|
41
|
+
end
|
42
|
+
end
|
43
|
+
private :acts_as_muschable
|
44
|
+
end
|
45
|
+
|
46
|
+
module ClassMethods
|
47
|
+
|
48
|
+
def initialize_shards
|
49
|
+
0.upto(@shard_amount-1) do |i|
|
50
|
+
connection.execute("CREATE TABLE #{table_name_for_shard(i)} LIKE #{table_name_without_shard}")
|
51
|
+
end
|
52
|
+
end
|
53
|
+
|
54
|
+
@shard_amount = nil
|
55
|
+
attr_reader :shard_amount
|
56
|
+
def shard_amount=(amount)
|
57
|
+
ensure_positive_int('shard_amount', amount)
|
58
|
+
@shard_amount = amount
|
59
|
+
end
|
60
|
+
|
61
|
+
def activate_shard(shard)
|
62
|
+
ensure_positive_int('shard identifier', shard)
|
63
|
+
raise ArgumentError, "Can't activate shard, out of range. Adjust #{self.name}.shard_amount=" unless shard<@shard_amount
|
64
|
+
|
65
|
+
ensure_setup
|
66
|
+
Thread.current[:shards][self.name.to_sym] = shard.to_s
|
67
|
+
end
|
68
|
+
|
69
|
+
def activate_base_shard
|
70
|
+
ensure_setup
|
71
|
+
Thread.current[:shards][self.name.to_sym] = -1
|
72
|
+
end
|
73
|
+
|
74
|
+
def detect_corrupt_shards
|
75
|
+
base_schema = extract_relevant_part_from_schema_definition(table_schema(table_name_without_shard))
|
76
|
+
returning Array.new do |corrupt_shards|
|
77
|
+
0.upto(shard_amount-1) do |shard|
|
78
|
+
shard_schema = extract_relevant_part_from_schema_definition(table_schema(table_name_for_shard(shard)))
|
79
|
+
corrupt_shards << shard if shard_schema!=base_schema or base_schema.blank?
|
80
|
+
end
|
81
|
+
end
|
82
|
+
end
|
83
|
+
|
84
|
+
def drop_shards(amount = nil)
|
85
|
+
amount ||= detect_shard_amount_in_database
|
86
|
+
ensure_positive_int('parameter for #drop_shards', amount)
|
87
|
+
0.upto(amount-1) do |i|
|
88
|
+
connection.execute("DROP TABLE #{table_name_for_shard(i)}")
|
89
|
+
end
|
90
|
+
end
|
91
|
+
|
92
|
+
def table_name_with_shard
|
93
|
+
ensure_setup
|
94
|
+
shard = Thread.current[:shards][self.name.to_sym]
|
95
|
+
|
96
|
+
return table_name_without_shard if @shard_amount==0 or shard == -1
|
97
|
+
raise ArgumentError, 'No shard has been activated' unless shard
|
98
|
+
table_name_for_shard(shard)
|
99
|
+
end
|
100
|
+
|
101
|
+
def detect_shard_amount_in_database
|
102
|
+
result = connection.execute "SHOW TABLES LIKE '#{table_name_without_shard}%'"
|
103
|
+
tables = []
|
104
|
+
result.each do |row|
|
105
|
+
tables << row[0].gsub(table_name_without_shard, '').to_i
|
106
|
+
end
|
107
|
+
result.free
|
108
|
+
return 0 if tables.size<=1
|
109
|
+
tables.sort.last.to_i + 1
|
110
|
+
end
|
111
|
+
|
112
|
+
def shard_levels
|
113
|
+
return [] unless shard_amount > 0
|
114
|
+
levels = []
|
115
|
+
(0...shard_amount).each do |i|
|
116
|
+
result = connection.execute "SELECT COUNT(*) FROM #{table_name_for_shard(i)}"
|
117
|
+
result.each do |row|
|
118
|
+
levels[i] = row[0].to_i
|
119
|
+
end
|
120
|
+
end
|
121
|
+
levels
|
122
|
+
end
|
123
|
+
|
124
|
+
def ensure_setup
|
125
|
+
raise ArgumentError, "You have to set #{self.name}.shard_amount" if @shard_amount.nil?
|
126
|
+
Thread.current[:shards] ||= Hash.new
|
127
|
+
end
|
128
|
+
|
129
|
+
def table_name_for_shard(shard)
|
130
|
+
"#{table_name_without_shard}#{shard}"
|
131
|
+
end
|
132
|
+
|
133
|
+
def ensure_positive_int(name, i)
|
134
|
+
raise ArgumentError, "Only positive integers are allowed as #{name}" unless i.is_a?(Integer) and i>=0
|
135
|
+
end
|
136
|
+
|
137
|
+
def table_schema(table_name)
|
138
|
+
result = connection.execute "SHOW CREATE TABLE #{table_name}"
|
139
|
+
schema = ""
|
140
|
+
result.each do |row|
|
141
|
+
schema << row[1]
|
142
|
+
end
|
143
|
+
schema
|
144
|
+
rescue ActiveRecord::StatementInvalid
|
145
|
+
""
|
146
|
+
ensure
|
147
|
+
result.free if result
|
148
|
+
end
|
149
|
+
|
150
|
+
def extract_relevant_part_from_schema_definition(definition)
|
151
|
+
definition.gsub!(/ AUTO_INCREMENT=[\d]+/, '')
|
152
|
+
match = definition.match(/[^\(]+(.*)$/m)
|
153
|
+
return match[1] if match
|
154
|
+
""
|
155
|
+
end
|
156
|
+
|
157
|
+
#
|
158
|
+
# This is here because ActiveRecord::Base's table_name method
|
159
|
+
# does something funky. If you call ActiveRecord::Base#table_name
|
160
|
+
# for the first time, it goes through the class hierarchy and
|
161
|
+
# puts together a table name.
|
162
|
+
#
|
163
|
+
# Then it redefines the table_name method to instantly return that
|
164
|
+
# name from the first run without going through the class hierarchy
|
165
|
+
# again.
|
166
|
+
#
|
167
|
+
# So this method overwrites :table_name_without_shard instead of
|
168
|
+
# the actual #table_name method, so that the name of the shard is
|
169
|
+
# returned.
|
170
|
+
#
|
171
|
+
def set_table_name(value = nil, &block)
|
172
|
+
define_attr_method :table_name_without_shard, value, &block
|
173
|
+
end
|
174
|
+
|
175
|
+
end
|
176
|
+
end
|
177
|
+
end
|
178
|
+
end
|
data/rails/init.rb
ADDED
@@ -0,0 +1,215 @@
|
|
1
|
+
require File.dirname(__FILE__) + '/../spec_helper'
|
2
|
+
|
3
|
+
describe "Acts as Muschable" do
|
4
|
+
|
5
|
+
before(:all) do
|
6
|
+
Thread.current[:shards] = nil
|
7
|
+
end
|
8
|
+
|
9
|
+
describe "changing active shards" do
|
10
|
+
it "should throw an exception when MuschableModel.table_name is called before MuschableModel.set_shard" do
|
11
|
+
lambda{
|
12
|
+
MuschableModel.table_name
|
13
|
+
}.should raise_error(ArgumentError, 'No shard has been activated')
|
14
|
+
end
|
15
|
+
|
16
|
+
it "should not affect UnMuschableModels" do
|
17
|
+
UnmuschableModel.table_name.should == "unmuschable_models"
|
18
|
+
end
|
19
|
+
|
20
|
+
it "should be able to MuschableModel.activate_shard" do
|
21
|
+
MuschableModel.should respond_to(:activate_shard)
|
22
|
+
end
|
23
|
+
|
24
|
+
it "should be able to MuschableModel.activate_base_shard" do
|
25
|
+
MuschableModel.should respond_to(:activate_base_shard)
|
26
|
+
end
|
27
|
+
|
28
|
+
it "should return the correct MuschableModel.table_name according to shard" do
|
29
|
+
MuschableModel.activate_shard(0)
|
30
|
+
MuschableModel.table_name.should == "muschable_models0"
|
31
|
+
MuschableModel.activate_shard(1)
|
32
|
+
MuschableModel.table_name.should == "muschable_models1"
|
33
|
+
MuschableModel.activate_base_shard
|
34
|
+
MuschableModel.table_name.should == "muschable_models"
|
35
|
+
end
|
36
|
+
|
37
|
+
it "should only accept positive numeric shard identifiers" do
|
38
|
+
lambda{
|
39
|
+
MuschableModel.activate_shard("0")
|
40
|
+
}.should raise_error(ArgumentError, 'Only positive integers are allowed as shard identifier')
|
41
|
+
lambda{
|
42
|
+
MuschableModel.activate_shard(-1)
|
43
|
+
}.should raise_error(ArgumentError, 'Only positive integers are allowed as shard identifier')
|
44
|
+
end
|
45
|
+
|
46
|
+
it "should be somewhat thread safe" do
|
47
|
+
MuschableModel.shard_amount = 1_000_000
|
48
|
+
threads = []
|
49
|
+
300.times do
|
50
|
+
threads << Thread.new do
|
51
|
+
shard = rand(1_000_000)
|
52
|
+
MuschableModel.activate_shard(shard)
|
53
|
+
sleep(rand(10))
|
54
|
+
MuschableModel.table_name.should == "muschable_models#{shard}"
|
55
|
+
MuschableModel.activate_base_shard
|
56
|
+
sleep(rand(5))
|
57
|
+
MuschableModel.table_name.should == "muschable_models"
|
58
|
+
end
|
59
|
+
end
|
60
|
+
threads.each { |thread| thread.join }
|
61
|
+
end
|
62
|
+
end
|
63
|
+
|
64
|
+
describe "managing the amount of shards" do
|
65
|
+
it "should have a method to set MuschableModel.shard_amount=" do
|
66
|
+
MuschableModel.shard_amount = 1
|
67
|
+
MuschableModel.shard_amount.should == 1
|
68
|
+
MuschableModel.shard_amount = 15
|
69
|
+
MuschableModel.shard_amount.should == 15
|
70
|
+
end
|
71
|
+
|
72
|
+
it "should do no sharding when MuschableModel.shard_amount is set to 0 (useful for test environments and such)" do
|
73
|
+
MuschableModel.shard_amount = 0
|
74
|
+
MuschableModel.table_name.should == "muschable_models"
|
75
|
+
end
|
76
|
+
|
77
|
+
it "should not accept non-integers as MuschableModel.shard_amount=" do
|
78
|
+
lambda{
|
79
|
+
MuschableModel.shard_amount = "15"
|
80
|
+
}.should raise_error(ArgumentError, 'Only positive integers are allowed as shard_amount')
|
81
|
+
lambda{
|
82
|
+
MuschableModel.shard_amount = -1
|
83
|
+
}.should raise_error(ArgumentError, 'Only positive integers are allowed as shard_amount')
|
84
|
+
end
|
85
|
+
|
86
|
+
it "should not accept a shard identifier larger MuschableModel.shard_amount" do
|
87
|
+
MuschableModel.shard_amount = 16
|
88
|
+
lambda{
|
89
|
+
MuschableModel.activate_shard(16)
|
90
|
+
}.should raise_error(ArgumentError, "Can't activate shard, out of range. Adjust MuschableModel.shard_amount=")
|
91
|
+
end
|
92
|
+
|
93
|
+
it "should have a method MuschableModel.initialize_shards to create MuschableModel.shard_amount new shards" do
|
94
|
+
MuschableModel.activate_shard(0)
|
95
|
+
connection = mock("Connection")
|
96
|
+
connection.stub!(:table_exists?).with(any_args).and_return true
|
97
|
+
MuschableModel.should_receive(:connection).exactly(16).and_return connection
|
98
|
+
|
99
|
+
0.upto(15) do |i|
|
100
|
+
query = "CREATE TABLE muschable_models#{i} LIKE muschable_models"
|
101
|
+
connection.should_receive(:execute).with(query).once.and_return(mock("execute #{i}", :null_object => true))
|
102
|
+
end
|
103
|
+
|
104
|
+
MuschableModel.initialize_shards
|
105
|
+
end
|
106
|
+
|
107
|
+
it "should have a method MuschableModel.drop_shards(12) to drop shards 0...12" do
|
108
|
+
MuschableModel.activate_shard(0)
|
109
|
+
connection = mock("Connection")
|
110
|
+
connection.stub!(:table_exists?).with(any_args).and_return true
|
111
|
+
MuschableModel.should_receive(:connection).exactly(12).and_return connection
|
112
|
+
|
113
|
+
0.upto(11) do |i|
|
114
|
+
query = "DROP TABLE muschable_models#{i}"
|
115
|
+
connection.should_receive(:execute).with(query).once.and_return(mock("execute #{i}", :null_object => true))
|
116
|
+
end
|
117
|
+
|
118
|
+
MuschableModel.drop_shards(12)
|
119
|
+
end
|
120
|
+
|
121
|
+
[-1, "1", "a"].each do |i|
|
122
|
+
it "should not allow #drop_shards(#{i})" do
|
123
|
+
lambda{
|
124
|
+
MuschableModel.drop_shards(i)
|
125
|
+
}.should raise_error(ArgumentError, 'Only positive integers are allowed as parameter for #drop_shards')
|
126
|
+
end
|
127
|
+
end
|
128
|
+
|
129
|
+
it "should have a method MuschableModel.assure_shards_health that goes through all shards and makes sure their structure equals that of the base table" do
|
130
|
+
MuschableModel.should respond_to(:detect_corrupt_shards)
|
131
|
+
end
|
132
|
+
end
|
133
|
+
|
134
|
+
describe "utility methods" do
|
135
|
+
|
136
|
+
it "should have a method :shard_level to deliver statistics about how many rows each shard has" do
|
137
|
+
MuschableModel.should respond_to(:shard_levels)
|
138
|
+
end
|
139
|
+
|
140
|
+
it "should extract the relevant parts from a schema definition in order to compare definitions" do
|
141
|
+
base_table_definition =<<-SQL
|
142
|
+
CREATE TABLE `movies_users` (
|
143
|
+
`user_id` int(11) DEFAULT NULL,
|
144
|
+
`movie_id` int(11) DEFAULT NULL,
|
145
|
+
`rating_date` datetime DEFAULT NULL,
|
146
|
+
`rating` int(3) DEFAULT NULL,
|
147
|
+
`top` tinyint(1) NOT NULL DEFAULT '0',
|
148
|
+
`flop` tinyint(1) NOT NULL DEFAULT '0',
|
149
|
+
`id` int(11) NOT NULL AUTO_INCREMENT,
|
150
|
+
`watchlist` tinyint(1) NOT NULL DEFAULT '0',
|
151
|
+
`blacklist` tinyint(1) NOT NULL DEFAULT '0',
|
152
|
+
`watchlist_date` datetime DEFAULT NULL,
|
153
|
+
`blacklist_date` datetime DEFAULT NULL,
|
154
|
+
`forecast` float DEFAULT NULL,
|
155
|
+
`forecast_relevance` float DEFAULT NULL,
|
156
|
+
`forecast_neighbour_count` int(4) DEFAULT NULL,
|
157
|
+
`forecast_or_rating` int(3) DEFAULT NULL,
|
158
|
+
PRIMARY KEY (`id`),
|
159
|
+
UNIQUE KEY `index_movies_users_on_user_id_and_movie_id` (`user_id`,`movie_id`),
|
160
|
+
KEY `user_ratings` (`movie_id`,`user_id`,`rating`),
|
161
|
+
KEY `index_movies_users_on_updated_at_and_id` (`rating_date`,`id`),
|
162
|
+
KEY `movie_id_and_rating_date` (`movie_id`,`rating_date`),
|
163
|
+
KEY `movie_id_and_user_id_and_forecast_or_rating` (`movie_id`,`user_id`,`forecast_or_rating`),
|
164
|
+
KEY `rating_date` (`rating_date`),
|
165
|
+
KEY `top` (`top`),
|
166
|
+
KEY `flop` (`flop`),
|
167
|
+
KEY `index_movies_users_on_user_id_and_blacklist_and_movie_id` (`user_id`,`blacklist`,`movie_id`),
|
168
|
+
KEY `movies_users_on_user_id_and_rating_and_movie_id` (`user_id`,`rating`,`movie_id`),
|
169
|
+
KEY `movies_users_on_movie_id_and_rating` (`movie_id`,`rating`),
|
170
|
+
KEY `movies_users_on_rating_and_user_id_and_movie_id` (`rating`,`user_id`,`movie_id`),
|
171
|
+
KEY `user_id_and_movie_id_and_forecast_or_rating_on_movies_users` (`user_id`,`movie_id`,`forecast_or_rating`)
|
172
|
+
) ENGINE=MyISAM AUTO_INCREMENT=241855247 DEFAULT CHARSET=utf8
|
173
|
+
SQL
|
174
|
+
|
175
|
+
shard_table_definition =<<-SQL
|
176
|
+
CREATE TABLE `movies_users0` (
|
177
|
+
`user_id` int(11) DEFAULT NULL,
|
178
|
+
`movie_id` int(11) DEFAULT NULL,
|
179
|
+
`rating_date` datetime DEFAULT NULL,
|
180
|
+
`rating` int(3) DEFAULT NULL,
|
181
|
+
`top` tinyint(1) NOT NULL DEFAULT '0',
|
182
|
+
`flop` tinyint(1) NOT NULL DEFAULT '0',
|
183
|
+
`id` int(11) NOT NULL AUTO_INCREMENT,
|
184
|
+
`watchlist` tinyint(1) NOT NULL DEFAULT '0',
|
185
|
+
`blacklist` tinyint(1) NOT NULL DEFAULT '0',
|
186
|
+
`watchlist_date` datetime DEFAULT NULL,
|
187
|
+
`blacklist_date` datetime DEFAULT NULL,
|
188
|
+
`forecast` float DEFAULT NULL,
|
189
|
+
`forecast_relevance` float DEFAULT NULL,
|
190
|
+
`forecast_neighbour_count` int(4) DEFAULT NULL,
|
191
|
+
`forecast_or_rating` int(3) DEFAULT NULL,
|
192
|
+
PRIMARY KEY (`id`),
|
193
|
+
UNIQUE KEY `index_movies_users_on_user_id_and_movie_id` (`user_id`,`movie_id`),
|
194
|
+
KEY `user_ratings` (`movie_id`,`user_id`,`rating`),
|
195
|
+
KEY `index_movies_users_on_updated_at_and_id` (`rating_date`,`id`),
|
196
|
+
KEY `movie_id_and_rating_date` (`movie_id`,`rating_date`),
|
197
|
+
KEY `movie_id_and_user_id_and_forecast_or_rating` (`movie_id`,`user_id`,`forecast_or_rating`),
|
198
|
+
KEY `rating_date` (`rating_date`),
|
199
|
+
KEY `top` (`top`),
|
200
|
+
KEY `flop` (`flop`),
|
201
|
+
KEY `index_movies_users_on_user_id_and_blacklist_and_movie_id` (`user_id`,`blacklist`,`movie_id`),
|
202
|
+
KEY `movies_users_on_user_id_and_rating_and_movie_id` (`user_id`,`rating`,`movie_id`),
|
203
|
+
KEY `movies_users_on_movie_id_and_rating` (`movie_id`,`rating`),
|
204
|
+
KEY `movies_users_on_rating_and_user_id_and_movie_id` (`rating`,`user_id`,`movie_id`),
|
205
|
+
KEY `user_id_and_movie_id_and_forecast_or_rating_on_movies_users` (`user_id`,`movie_id`,`forecast_or_rating`)
|
206
|
+
) ENGINE=MyISAM DEFAULT CHARSET=utf8
|
207
|
+
SQL
|
208
|
+
|
209
|
+
base_definition = MuschableModel.extract_relevant_part_from_schema_definition(base_table_definition)
|
210
|
+
shard_definition = MuschableModel.extract_relevant_part_from_schema_definition(shard_table_definition)
|
211
|
+
|
212
|
+
shard_definition.should == base_definition
|
213
|
+
end
|
214
|
+
end
|
215
|
+
end
|
@@ -0,0 +1,139 @@
|
|
1
|
+
require File.dirname(__FILE__) + '/../spec_helper'
|
2
|
+
|
3
|
+
describe "Integration tests with database access" do
|
4
|
+
|
5
|
+
unless ENV['RUN_MUSCH_INTEGRATION_TESTS']=="true"
|
6
|
+
print <<-HELP
|
7
|
+
<pre>==================== INTEGRATION TESTS ===========================
|
8
|
+
If you want to run functional tests on your DB, set
|
9
|
+
ENV['RUN_MUSCH_INTEGRATION_TESTS'] to 'true' and call rake spec again.
|
10
|
+
|
11
|
+
Then make sure the mysql user rails:rails@localhost has all mysql
|
12
|
+
privileges for database 'test' (should work out of the box if you
|
13
|
+
have a rails user in mysql)
|
14
|
+
=================================================================</pre>
|
15
|
+
HELP
|
16
|
+
break
|
17
|
+
end
|
18
|
+
|
19
|
+
describe "shard management" do
|
20
|
+
|
21
|
+
before(:all) do
|
22
|
+
Thread.current[:shards] = nil
|
23
|
+
puts "\nRunning integration tests"
|
24
|
+
@conn = MuschableModel.connection
|
25
|
+
end
|
26
|
+
|
27
|
+
it "should detect how many must be deleted with #detect_shard_amount_in_database when there are shards" do
|
28
|
+
MuschableModel.detect_shard_amount_in_database.should == 4
|
29
|
+
end
|
30
|
+
|
31
|
+
it "should detect how many must be deleted with #detect_shard_amount_in_database when there are no shards" do
|
32
|
+
OtherMuschableModel.detect_shard_amount_in_database.should == 0
|
33
|
+
end
|
34
|
+
|
35
|
+
it "should detect corrupt shards during #detect_corrupt_shards" do
|
36
|
+
MuschableModel.shard_amount = 4
|
37
|
+
MuschableModel.detect_corrupt_shards.should == [1,2,3]
|
38
|
+
end
|
39
|
+
|
40
|
+
it "should not detect corrupt shards when database is healthy" do
|
41
|
+
MuschableModel.shard_amount = 1
|
42
|
+
MuschableModel.detect_corrupt_shards.should be_blank
|
43
|
+
end
|
44
|
+
|
45
|
+
it "should drop all shards during #drop_shards(3)" do
|
46
|
+
class YetAnotherMuschableModel < ActiveRecord::Base
|
47
|
+
acts_as_muschable :shard_amount => 5
|
48
|
+
end
|
49
|
+
create_base_table_and_shards "yet_another_muschable_models", 10
|
50
|
+
YetAnotherMuschableModel.detect_shard_amount_in_database.should == 10
|
51
|
+
YetAnotherMuschableModel.drop_shards(10)
|
52
|
+
YetAnotherMuschableModel.detect_shard_amount_in_database.should == 0
|
53
|
+
end
|
54
|
+
|
55
|
+
it "should drop all shards during #drop_shards (automatically guessing how many shards exist)" do
|
56
|
+
class YetYetAnotherMuschableModel < ActiveRecord::Base
|
57
|
+
acts_as_muschable :shard_amount => 5
|
58
|
+
end
|
59
|
+
create_base_table_and_shards "yet_yet_another_muschable_models", 15
|
60
|
+
YetYetAnotherMuschableModel.activate_shard 0
|
61
|
+
YetYetAnotherMuschableModel.detect_shard_amount_in_database.should == 15
|
62
|
+
YetYetAnotherMuschableModel.drop_shards
|
63
|
+
YetYetAnotherMuschableModel.detect_shard_amount_in_database.should == 0
|
64
|
+
end
|
65
|
+
|
66
|
+
it "should create all shards during #initialize_shards" do
|
67
|
+
OtherMuschableModel.detect_shard_amount_in_database.should == 0
|
68
|
+
OtherMuschableModel.shard_amount = 3
|
69
|
+
OtherMuschableModel.initialize_shards
|
70
|
+
|
71
|
+
OtherMuschableModel.detect_shard_amount_in_database.should == 3
|
72
|
+
OtherMuschableModel.detect_corrupt_shards.should be_blank
|
73
|
+
end
|
74
|
+
|
75
|
+
end
|
76
|
+
|
77
|
+
describe "objects living in different shards" do
|
78
|
+
|
79
|
+
before(:all) do
|
80
|
+
class MyMuschableModel < ActiveRecord::Base
|
81
|
+
acts_as_muschable :shard_amount => 2
|
82
|
+
end
|
83
|
+
create_base_table_and_shards "my_muschable_models", 0
|
84
|
+
MyMuschableModel.initialize_shards
|
85
|
+
@conn = MuschableModel.connection
|
86
|
+
end
|
87
|
+
|
88
|
+
it "should create different models in different shards" do
|
89
|
+
MyMuschableModel.activate_shard 0
|
90
|
+
model1_in_shard0 = MyMuschableModel.create :name => "model1_in_shard0"
|
91
|
+
model2_in_shard0 = MyMuschableModel.create :name => "model2_in_shard0"
|
92
|
+
MyMuschableModel.count.should == 2
|
93
|
+
MyMuschableModel.find(model2_in_shard0.id).should == model2_in_shard0
|
94
|
+
|
95
|
+
MyMuschableModel.activate_shard 1
|
96
|
+
model1_in_shard1 = MyMuschableModel.create :name => "model1_in_shard1"
|
97
|
+
MyMuschableModel.count.should == 1
|
98
|
+
MyMuschableModel.destroy_all
|
99
|
+
MyMuschableModel.count.should == 0
|
100
|
+
|
101
|
+
MyMuschableModel.activate_shard 0
|
102
|
+
MyMuschableModel.count.should == 2
|
103
|
+
end
|
104
|
+
|
105
|
+
end
|
106
|
+
|
107
|
+
#
|
108
|
+
# Drop all tables created during this test (i.e. have muschable in their name)
|
109
|
+
after(:all) do
|
110
|
+
@conn.execute "DROP TABLE schema_migrations"
|
111
|
+
result = @conn.execute "SHOW TABLES LIKE '%muschable%'"
|
112
|
+
result.each do |row|
|
113
|
+
@conn.execute("DROP TABLE #{row[0]}")
|
114
|
+
end
|
115
|
+
result.free
|
116
|
+
end
|
117
|
+
end
|
118
|
+
|
119
|
+
|
120
|
+
#
|
121
|
+
# Santa's little helper
|
122
|
+
#
|
123
|
+
def create_base_table_and_shards(table_name, shard_amount)
|
124
|
+
ActiveRecord::Schema.define :version => 0 do
|
125
|
+
create_table table_name, :force => true do |t|
|
126
|
+
t.integer "id", :limit => 11
|
127
|
+
t.string "name"
|
128
|
+
end
|
129
|
+
add_index table_name, ["id"], :name => "index_on_id"
|
130
|
+
|
131
|
+
0.upto(shard_amount-1) do |i|
|
132
|
+
create_table "#{table_name}#{i}", :force => true do |t|
|
133
|
+
t.integer "id", :limit => 11
|
134
|
+
t.string "name"
|
135
|
+
end
|
136
|
+
add_index "#{table_name}#{i}", ["id"], :name => "index_on_id"
|
137
|
+
end
|
138
|
+
end
|
139
|
+
end
|
data/spec/schema.rb
ADDED
@@ -0,0 +1,52 @@
|
|
1
|
+
RAILS_DEFAULT_LOGGER.silence do
|
2
|
+
ActiveRecord::Schema.define :version => 0 do
|
3
|
+
|
4
|
+
#
|
5
|
+
# Base table
|
6
|
+
#
|
7
|
+
create_table "muschable_models", :force => true do |t|
|
8
|
+
t.integer "id", :limit => 11
|
9
|
+
t.string "name"
|
10
|
+
end
|
11
|
+
add_index "muschable_models", ["id"], :name => "index_on_id"
|
12
|
+
|
13
|
+
#
|
14
|
+
# First shard, healthy
|
15
|
+
#
|
16
|
+
create_table "muschable_models0", :force => true do |t|
|
17
|
+
t.integer "id", :limit => 11
|
18
|
+
t.string "name"
|
19
|
+
end
|
20
|
+
add_index "muschable_models0", ["id"], :name => "index_on_id"
|
21
|
+
|
22
|
+
#
|
23
|
+
# Second shard, wrong column
|
24
|
+
#
|
25
|
+
create_table "muschable_models1", :force => true do |t|
|
26
|
+
t.integer "id", :limit => 11
|
27
|
+
t.string "nume"
|
28
|
+
end
|
29
|
+
add_index "muschable_models1", ["id"], :name => "index_on_id"
|
30
|
+
|
31
|
+
#
|
32
|
+
# Third shard, missing in action
|
33
|
+
#
|
34
|
+
|
35
|
+
#
|
36
|
+
# Fourth shard, missing index
|
37
|
+
#
|
38
|
+
create_table "muschable_models3", :force => true do |t|
|
39
|
+
t.integer "id", :limit => 11
|
40
|
+
t.string "name"
|
41
|
+
end
|
42
|
+
|
43
|
+
#
|
44
|
+
# Another base shard
|
45
|
+
#
|
46
|
+
create_table "other_muschable_models", :force => true do |t|
|
47
|
+
t.integer "id", :limit => 11
|
48
|
+
t.string "name"
|
49
|
+
end
|
50
|
+
add_index "other_muschable_models", ["id"], :name => "index_on_id"
|
51
|
+
end
|
52
|
+
end
|
data/spec/spec.opts
ADDED
data/spec/spec_helper.rb
ADDED
@@ -0,0 +1,47 @@
|
|
1
|
+
ENV["RAILS_ENV"] = "test"
|
2
|
+
|
3
|
+
require 'rubygems'
|
4
|
+
require 'active_support'
|
5
|
+
require 'active_record'
|
6
|
+
require 'spec'
|
7
|
+
|
8
|
+
RAILS_DEFAULT_LOGGER = Logger.new(File.join(File.dirname(__FILE__), "debug.log"))
|
9
|
+
|
10
|
+
$: << File.join(File.dirname(__FILE__), '..', 'lib')
|
11
|
+
require File.join(File.dirname(__FILE__), '..', 'init')
|
12
|
+
|
13
|
+
if ENV['RUN_MUSCH_INTEGRATION_TESTS']=="true"
|
14
|
+
ActiveRecord::Base.establish_connection(
|
15
|
+
"adapter" => "mysql",
|
16
|
+
"database" => "test",
|
17
|
+
"host" => "localhost",
|
18
|
+
"socket" => "/tmp/mysql.sock",
|
19
|
+
"port" => 3306,
|
20
|
+
"username" => "rails",
|
21
|
+
"password" => "rails"
|
22
|
+
)
|
23
|
+
|
24
|
+
load(File.dirname(__FILE__) + '/schema.rb')
|
25
|
+
end
|
26
|
+
|
27
|
+
class MuschableModel < ActiveRecord::Base
|
28
|
+
acts_as_muschable :shard_amount => 16
|
29
|
+
end
|
30
|
+
|
31
|
+
class OtherMuschableModel < ActiveRecord::Base
|
32
|
+
acts_as_muschable :shard_amount => 16
|
33
|
+
end
|
34
|
+
|
35
|
+
class UnmuschableModel < ActiveRecord::Base
|
36
|
+
end
|
37
|
+
|
38
|
+
# This is here so the schema definitions don't produce too much output
|
39
|
+
BEGIN {
|
40
|
+
class<<Object
|
41
|
+
def puts_with_crap_cleansing(msg)
|
42
|
+
puts_without_crap_cleansing(msg) if ['-- ', ' ->'].reject{|⎮| msg.starts_with?(⎮)}.count == 2
|
43
|
+
end
|
44
|
+
alias_method :puts_without_crap_cleansing, :puts
|
45
|
+
alias_method :puts, :puts_with_crap_cleansing
|
46
|
+
end
|
47
|
+
}
|
metadata
ADDED
@@ -0,0 +1,69 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: acts_as_muschable
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.1.0
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Moviepilot
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
|
12
|
+
date: 2010-03-04 00:00:00 +01:00
|
13
|
+
default_executable:
|
14
|
+
dependencies: []
|
15
|
+
|
16
|
+
description: Do not use me unless you are not you but somebody else.
|
17
|
+
email: jannis@moviepilot.de
|
18
|
+
executables: []
|
19
|
+
|
20
|
+
extensions: []
|
21
|
+
|
22
|
+
extra_rdoc_files: []
|
23
|
+
|
24
|
+
files:
|
25
|
+
- .gitignore
|
26
|
+
- Rakefile
|
27
|
+
- VERSION
|
28
|
+
- acts_as_muschable.gemspec
|
29
|
+
- init.rb
|
30
|
+
- lib/acts_as_muschable.rb
|
31
|
+
- rails/init.rb
|
32
|
+
- spec/acts_as_muschable/acts_as_muschable_spec.rb
|
33
|
+
- spec/acts_as_muschable/integration_spec.rb
|
34
|
+
- spec/schema.rb
|
35
|
+
- spec/spec.opts
|
36
|
+
- spec/spec_helper.rb
|
37
|
+
has_rdoc: true
|
38
|
+
homepage: http://github.com/moviepilot/acts_as_muschable
|
39
|
+
licenses: []
|
40
|
+
|
41
|
+
post_install_message:
|
42
|
+
rdoc_options:
|
43
|
+
- --charset=UTF-8
|
44
|
+
require_paths:
|
45
|
+
- lib
|
46
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
47
|
+
requirements:
|
48
|
+
- - ">="
|
49
|
+
- !ruby/object:Gem::Version
|
50
|
+
version: "0"
|
51
|
+
version:
|
52
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
53
|
+
requirements:
|
54
|
+
- - ">="
|
55
|
+
- !ruby/object:Gem::Version
|
56
|
+
version: "0"
|
57
|
+
version:
|
58
|
+
requirements: []
|
59
|
+
|
60
|
+
rubyforge_project:
|
61
|
+
rubygems_version: 1.3.5
|
62
|
+
signing_key:
|
63
|
+
specification_version: 3
|
64
|
+
summary: Easy peasy model sharding
|
65
|
+
test_files:
|
66
|
+
- spec/acts_as_muschable/acts_as_muschable_spec.rb
|
67
|
+
- spec/acts_as_muschable/integration_spec.rb
|
68
|
+
- spec/schema.rb
|
69
|
+
- spec/spec_helper.rb
|