schema_transformer 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/README.markdown +83 -0
- data/Rakefile +29 -0
- data/TODO +11 -0
- data/bin/schema_transformer +4 -0
- data/gemspec.rb +20 -0
- data/lib/schema_transformer/base.rb +260 -0
- data/lib/schema_transformer/cli.rb +99 -0
- data/lib/schema_transformer/help.rb +43 -0
- data/lib/schema_transformer/version.rb +3 -0
- data/lib/schema_transformer.rb +10 -0
- data/notes/copier.rb +14 -0
- data/notes/copier_scratchpad.rb +45 -0
- data/notes/pager.rb +101 -0
- data/notes/schema_transformer_notes.txt +44 -0
- data/test/fake_app/config/database.yml +34 -0
- data/test/fake_app/config/schema_transformations/books.json +1 -0
- data/test/fake_app/config/schema_transformations/users.json +1 -0
- data/test/fake_app/log/schema_transformer.log +58795 -0
- data/test/schema_transformer_test.rb +210 -0
- metadata +85 -0
@@ -0,0 +1,210 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
ENV['RAILS_ENV'] = 'test'
|
4
|
+
|
5
|
+
require 'rubygems'
|
6
|
+
require 'test/unit'
|
7
|
+
require 'pp'
|
8
|
+
require File.expand_path("../../lib/schema_transformer", __FILE__)
|
9
|
+
|
10
|
+
# open to mock out methods
|
11
|
+
$testing_books = false # im being lazy, should use mocks
|
12
|
+
module SchemaTransformer
|
13
|
+
class Base
|
14
|
+
def ask(msg)
|
15
|
+
nil
|
16
|
+
end
|
17
|
+
def gets(name = nil)
|
18
|
+
case name
|
19
|
+
when :table
|
20
|
+
if $testing_books
|
21
|
+
out = "books"
|
22
|
+
else
|
23
|
+
out = "users"
|
24
|
+
end
|
25
|
+
when :mod
|
26
|
+
if $testing_books
|
27
|
+
out = "ADD COLUMN active tinyint(1) DEFAULT '0'"
|
28
|
+
else
|
29
|
+
out = "ADD COLUMN active tinyint(1) DEFAULT '0',
|
30
|
+
ADD COLUMN title varchar(255) DEFAULT 'Mr',
|
31
|
+
DROP COLUMN about_me"
|
32
|
+
end
|
33
|
+
else
|
34
|
+
raise "gets method: need to mock out #{name}"
|
35
|
+
end
|
36
|
+
out
|
37
|
+
end
|
38
|
+
def help(msg)
|
39
|
+
nil
|
40
|
+
end
|
41
|
+
end
|
42
|
+
end
|
43
|
+
|
44
|
+
def setup_fixtures
|
45
|
+
ActiveRecord::Base.connection.drop_table(:users_st_temp, :force => true) rescue nil
|
46
|
+
ActiveRecord::Base.connection.drop_table(:users_st_trash, :force => true) rescue nil
|
47
|
+
ActiveRecord::Base.connection.create_table :users, :force => true do |table|
|
48
|
+
table.column :name, :string
|
49
|
+
table.column :about_me, :string
|
50
|
+
table.column :updated_at, :datetime
|
51
|
+
table.column :created_at, :datetime
|
52
|
+
end
|
53
|
+
Object.send(:remove_const, "User") rescue nil
|
54
|
+
Object.const_set("User", Class.new(ActiveRecord::Base))
|
55
|
+
35.times do |i|
|
56
|
+
User.create(:name => "name_#{i}")
|
57
|
+
end
|
58
|
+
Object.send(:remove_const, "User") rescue nil
|
59
|
+
|
60
|
+
ActiveRecord::Base.connection.drop_table(:books_st_temp, :force => true) rescue nil
|
61
|
+
ActiveRecord::Base.connection.drop_table(:books_st_trash, :force => true) rescue nil
|
62
|
+
# no timestamp
|
63
|
+
ActiveRecord::Base.connection.create_table :books, :force => true do |table|
|
64
|
+
table.column :title, :string
|
65
|
+
table.column :author, :string
|
66
|
+
end
|
67
|
+
Object.send(:remove_const, "Book") rescue nil
|
68
|
+
Object.const_set("Book", Class.new(ActiveRecord::Base))
|
69
|
+
4.times do |i|
|
70
|
+
Book.create(:title => "title_#{i}")
|
71
|
+
end
|
72
|
+
Object.send(:remove_const, "Book") rescue nil
|
73
|
+
end
|
74
|
+
|
75
|
+
class SchemaTransformerTest < Test::Unit::TestCase
|
76
|
+
def count(table)
|
77
|
+
@conn = ActiveRecord::Base.connection
|
78
|
+
res = @conn.execute("SELECT count(*) AS c FROM #{table}")
|
79
|
+
c = res.fetch_row[0].to_i # nil case is okay: [nil][0].to_i => 0
|
80
|
+
end
|
81
|
+
|
82
|
+
def setup
|
83
|
+
@base = File.expand_path("../fake_app", __FILE__)
|
84
|
+
@transform_file = @base+"/config/schema_transformations/users.json"
|
85
|
+
File.delete(@transform_file) if File.exist?(@transform_file)
|
86
|
+
@transformer = SchemaTransformer::Base.new(@base, :batch_size => 10, :stagger => 0)
|
87
|
+
@conn = ActiveRecord::Base.connection
|
88
|
+
setup_fixtures
|
89
|
+
end
|
90
|
+
|
91
|
+
def test_no_updated_at_no_data
|
92
|
+
@conn.execute("delete from books")
|
93
|
+
$testing_books = true
|
94
|
+
@transformer.generate
|
95
|
+
@transformer.gather_info("books")
|
96
|
+
|
97
|
+
assert @conn.tables.include?("books")
|
98
|
+
assert !@conn.tables.include?("books_st_temp")
|
99
|
+
@transformer.create
|
100
|
+
assert @conn.tables.include?("books_st_temp")
|
101
|
+
|
102
|
+
@transformer.final_sync
|
103
|
+
$testing_books = false
|
104
|
+
end
|
105
|
+
|
106
|
+
def test_no_updated_at_with_data
|
107
|
+
$testing_books = true
|
108
|
+
@transformer.generate
|
109
|
+
@transformer.gather_info("books")
|
110
|
+
|
111
|
+
assert @conn.tables.include?("books")
|
112
|
+
assert !@conn.tables.include?("books_st_temp")
|
113
|
+
@transformer.create
|
114
|
+
assert @conn.tables.include?("books_st_temp")
|
115
|
+
|
116
|
+
@transformer.final_sync
|
117
|
+
$testing_books = false
|
118
|
+
end
|
119
|
+
|
120
|
+
def test_find_in_batches
|
121
|
+
i = 0
|
122
|
+
bounds = [[8, 17], [18, 27], [28,35]]
|
123
|
+
@transformer.find_in_batches("users", :start => 8, :batch_size => 10) do |batch|
|
124
|
+
# puts "batch #{batch.inspect}"
|
125
|
+
lower = batch.first
|
126
|
+
upper = batch.last
|
127
|
+
assert_equal bounds[i][0], lower
|
128
|
+
assert_equal bounds[i][1], upper
|
129
|
+
# puts("syncing over records #{lower} to #{upper}...")
|
130
|
+
i += 1
|
131
|
+
end
|
132
|
+
end
|
133
|
+
|
134
|
+
def test_run_sync_black_box
|
135
|
+
@transformer.generate
|
136
|
+
c1 = count("users")
|
137
|
+
SchemaTransformer::Base.run(:base => @base, :action => ["sync", "users"])
|
138
|
+
c2 = count("users_st_temp")
|
139
|
+
assert_equal c1, c2
|
140
|
+
end
|
141
|
+
|
142
|
+
def test_run_sync_black_box_repeatedly
|
143
|
+
@transformer.generate
|
144
|
+
c1 = count("users")
|
145
|
+
# first run
|
146
|
+
SchemaTransformer::Base.run(:base => @base, :action => ["sync", "users"])
|
147
|
+
assert_equal c1, count("users_st_temp")
|
148
|
+
@conn.execute("delete from users_st_temp order by id desc limit 10")
|
149
|
+
assert_equal c1, count("users_st_temp") + 10
|
150
|
+
# second run
|
151
|
+
SchemaTransformer::Base.run(:base => @base, :action => ["sync", "users"])
|
152
|
+
assert_equal c1, count("users_st_temp")
|
153
|
+
end
|
154
|
+
|
155
|
+
def test_run_switch_black_box
|
156
|
+
@transformer.generate
|
157
|
+
c1 = count("users")
|
158
|
+
SchemaTransformer::Base.run(:base => @base, :action => ["sync", "users"])
|
159
|
+
c2 = count("users_st_temp")
|
160
|
+
assert_equal c1, c2
|
161
|
+
@conn.execute("delete from users_st_temp order by id desc limit 10")
|
162
|
+
assert_equal c1, count("users_st_temp") + 10
|
163
|
+
|
164
|
+
# This is what Im testing
|
165
|
+
col1 = User.columns.size
|
166
|
+
SchemaTransformer::Base.run(:base => @base, :action => ["switch", "users"])
|
167
|
+
User.reset_column_information
|
168
|
+
assert_equal col1 + 1, User.columns.size
|
169
|
+
assert_equal c1, count("users") # this is the new table
|
170
|
+
end
|
171
|
+
|
172
|
+
def test_run_tranformations_white_box
|
173
|
+
@transformer.generate
|
174
|
+
@transformer.gather_info("users")
|
175
|
+
|
176
|
+
assert @conn.tables.include?("users")
|
177
|
+
assert !@conn.tables.include?("users_st_temp")
|
178
|
+
@transformer.create
|
179
|
+
assert @conn.tables.include?("users_st_temp")
|
180
|
+
|
181
|
+
assert_equal 0, UsersStTemp.count
|
182
|
+
@transformer.sync
|
183
|
+
assert_equal User.count, UsersStTemp.count
|
184
|
+
|
185
|
+
assert @conn.tables.include?("users")
|
186
|
+
@transformer.switch
|
187
|
+
assert @conn.tables.include?("users_st_trash")
|
188
|
+
assert !@conn.tables.include?("users_st_temp")
|
189
|
+
|
190
|
+
@transformer.cleanup
|
191
|
+
assert @conn.tables.include?("users")
|
192
|
+
assert !@conn.tables.include?("users_st_trash")
|
193
|
+
assert !@conn.tables.include?("users_st_temp")
|
194
|
+
end
|
195
|
+
|
196
|
+
def test_generate_transformations
|
197
|
+
assert !File.exist?(@transform_file)
|
198
|
+
@transformer.generate
|
199
|
+
assert File.exist?(@transform_file)
|
200
|
+
data = JSON.parse(IO.read(@transform_file))
|
201
|
+
assert_equal "users", data["table"]
|
202
|
+
assert_match /ADD COLUMN/, data["mod"]
|
203
|
+
|
204
|
+
@transformer.gather_info("users")
|
205
|
+
assert_equal "users", @transformer.instance_variable_get(:@table)
|
206
|
+
assert_match /ADD COLUMN/, @transformer.instance_variable_get(:@mod)
|
207
|
+
end
|
208
|
+
end
|
209
|
+
|
210
|
+
|
metadata
ADDED
@@ -0,0 +1,85 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: schema_transformer
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
hash: 27
|
5
|
+
prerelease: false
|
6
|
+
segments:
|
7
|
+
- 0
|
8
|
+
- 1
|
9
|
+
- 0
|
10
|
+
version: 0.1.0
|
11
|
+
platform: ruby
|
12
|
+
authors:
|
13
|
+
- Tung Nguyen
|
14
|
+
autorequire:
|
15
|
+
bindir: bin
|
16
|
+
cert_chain: []
|
17
|
+
|
18
|
+
date: 2010-10-22 00:00:00 -07:00
|
19
|
+
default_executable:
|
20
|
+
dependencies: []
|
21
|
+
|
22
|
+
description:
|
23
|
+
email: tongueroo@gmail.com
|
24
|
+
executables:
|
25
|
+
- schema_transformer
|
26
|
+
extensions: []
|
27
|
+
|
28
|
+
extra_rdoc_files:
|
29
|
+
- README.markdown
|
30
|
+
files:
|
31
|
+
- bin/schema_transformer
|
32
|
+
- gemspec.rb
|
33
|
+
- lib/schema_transformer/base.rb
|
34
|
+
- lib/schema_transformer/cli.rb
|
35
|
+
- lib/schema_transformer/help.rb
|
36
|
+
- lib/schema_transformer/version.rb
|
37
|
+
- lib/schema_transformer.rb
|
38
|
+
- notes/copier.rb
|
39
|
+
- notes/copier_scratchpad.rb
|
40
|
+
- notes/pager.rb
|
41
|
+
- notes/schema_transformer_notes.txt
|
42
|
+
- Rakefile
|
43
|
+
- README.markdown
|
44
|
+
- test/fake_app/config/database.yml
|
45
|
+
- test/fake_app/config/schema_transformations/books.json
|
46
|
+
- test/fake_app/config/schema_transformations/users.json
|
47
|
+
- test/fake_app/log/schema_transformer.log
|
48
|
+
- test/schema_transformer_test.rb
|
49
|
+
- TODO
|
50
|
+
has_rdoc: true
|
51
|
+
homepage: http://github.com/tongueroo/schema_transformer
|
52
|
+
licenses: []
|
53
|
+
|
54
|
+
post_install_message:
|
55
|
+
rdoc_options: []
|
56
|
+
|
57
|
+
require_paths:
|
58
|
+
- lib
|
59
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
60
|
+
none: false
|
61
|
+
requirements:
|
62
|
+
- - ">="
|
63
|
+
- !ruby/object:Gem::Version
|
64
|
+
hash: 3
|
65
|
+
segments:
|
66
|
+
- 0
|
67
|
+
version: "0"
|
68
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
69
|
+
none: false
|
70
|
+
requirements:
|
71
|
+
- - ">="
|
72
|
+
- !ruby/object:Gem::Version
|
73
|
+
hash: 3
|
74
|
+
segments:
|
75
|
+
- 0
|
76
|
+
version: "0"
|
77
|
+
requirements: []
|
78
|
+
|
79
|
+
rubyforge_project:
|
80
|
+
rubygems_version: 1.3.7
|
81
|
+
signing_key:
|
82
|
+
specification_version: 3
|
83
|
+
summary: Way is alter database schemas on large tables with little downtime
|
84
|
+
test_files: []
|
85
|
+
|