data_migrator 1.3 → 1.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/lib/data_migrator.rb +197 -219
- data/lib/tasks/data_migrator.rake +26 -26
- metadata +4 -5
- data/lib/version.rb +0 -3
data/lib/data_migrator.rb
CHANGED
@@ -6,304 +6,282 @@ class DataMigrationTask < Rails::Railtie
|
|
6
6
|
end
|
7
7
|
end
|
8
8
|
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
9
|
+
module RussellEdge
|
10
|
+
class DataMigrator
|
11
|
+
REMOVE_FILES_REGEX = /^\./
|
12
|
+
|
13
|
+
def initialize(migrations_path=nil)
|
14
|
+
@default_migrations_path = migrations_path || "#{Rails.root}/db/data_migrations"
|
15
|
+
end
|
16
|
+
|
17
|
+
def migrate(passed_location=nil, passed_version=nil)
|
18
|
+
setup
|
15
19
|
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
20
|
+
location = passed_location.nil? ? @default_migrations_path : passed_location
|
21
|
+
@@current_version = get_current_version
|
22
|
+
|
23
|
+
if passed_version.nil? || @@current_version.nil?
|
24
|
+
run_all_non_migrated(passed_version)
|
25
|
+
elsif passed_version < @@current_version
|
26
|
+
handle_lower_passed_version(passed_version)
|
27
|
+
elsif passed_version > @@current_version
|
28
|
+
handle_higher_passed_version(passed_version)
|
29
|
+
end
|
25
30
|
end
|
26
|
-
end
|
27
31
|
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
versions << filename
|
32
|
+
def pending_migrations
|
33
|
+
versions = []
|
34
|
+
files = get_all_files
|
35
|
+
files.each do |file|
|
36
|
+
filename, version, klass_name = seperate_file_parts(file)
|
37
|
+
versions << filename unless version_has_been_migrated?(version)
|
35
38
|
end
|
36
|
-
end
|
37
39
|
|
38
|
-
|
39
|
-
|
40
|
+
versions
|
41
|
+
end
|
40
42
|
|
41
|
-
|
42
|
-
|
43
|
+
def get_current_version
|
44
|
+
result = ActiveRecord::Base.connection.select_all("select max(version) as current_version from data_migrations")
|
43
45
|
|
44
|
-
|
46
|
+
current_version = result[0]['current_version'] unless result == -1
|
45
47
|
|
46
|
-
|
48
|
+
current_version = current_version.to_i unless current_version.nil?
|
47
49
|
|
48
|
-
|
49
|
-
|
50
|
+
current_version
|
51
|
+
end
|
50
52
|
|
51
|
-
|
52
|
-
|
53
|
-
|
53
|
+
def next_migration_number
|
54
|
+
Time.now.utc.strftime("%Y%m%d%H%M%S")
|
55
|
+
end
|
54
56
|
|
55
|
-
|
56
|
-
|
57
|
+
def run_up(passed_version)
|
58
|
+
setup
|
57
59
|
|
58
|
-
|
60
|
+
raise "VERSION is required" unless passed_version
|
59
61
|
|
60
|
-
|
61
|
-
|
62
|
+
files = get_all_files
|
63
|
+
found = false
|
62
64
|
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
-
self.handle_action(file, klass_name, version, :up)
|
69
|
-
else
|
70
|
-
puts "** Version #{passed_version} has already been migrated"
|
65
|
+
files.each do |file|
|
66
|
+
filename, version, klass_name = seperate_file_parts(file)
|
67
|
+
if passed_version == version
|
68
|
+
found = true
|
69
|
+
(version_has_been_migrated?(version)) ? (puts "** Version #{passed_version} has already been migrated") : handle_action(file, klass_name, version, :up)
|
71
70
|
end
|
72
71
|
end
|
73
|
-
end
|
74
72
|
|
75
|
-
|
76
|
-
puts "** Version #{passed_version} not found"
|
77
|
-
end
|
73
|
+
puts "** Version #{passed_version} not found" unless found
|
78
74
|
|
79
|
-
|
75
|
+
end
|
80
76
|
|
81
|
-
|
82
|
-
|
77
|
+
def run_down(passed_version)
|
78
|
+
setup
|
83
79
|
|
84
|
-
|
80
|
+
raise "VERSION is required" unless passed_version
|
85
81
|
|
86
|
-
|
87
|
-
|
88
|
-
|
89
|
-
|
90
|
-
|
91
|
-
|
92
|
-
|
93
|
-
|
94
|
-
self.handle_action(file, klass_name, version, :down)
|
95
|
-
else
|
96
|
-
puts "** Version #{passed_version} has not been migrated"
|
82
|
+
files = get_all_files
|
83
|
+
found = false
|
84
|
+
|
85
|
+
files.each do |file|
|
86
|
+
filename, version, klass_name = seperate_file_parts(file)
|
87
|
+
if passed_version == version
|
88
|
+
found = true
|
89
|
+
(version_has_been_migrated?(version)) ? (puts "** Version #{passed_version} has not been migrated") : handle_action(file, klass_name, version, :down)
|
97
90
|
end
|
98
91
|
end
|
99
|
-
end
|
100
92
|
|
101
|
-
|
102
|
-
puts "** Version #{passed_version} not found"
|
103
|
-
end
|
93
|
+
puts "** Version #{passed_version} not found" unless found
|
104
94
|
|
105
|
-
|
95
|
+
end
|
106
96
|
|
107
|
-
|
108
|
-
|
97
|
+
def prepare_migrations
|
98
|
+
target = "#{Rails.root}/db/data_migrations/"
|
109
99
|
|
110
|
-
|
111
|
-
|
100
|
+
# first copy all app data_migrations away
|
101
|
+
files = Dir["#{target}*.rb"]
|
112
102
|
|
113
|
-
|
114
|
-
|
115
|
-
|
116
|
-
|
117
|
-
|
103
|
+
unless files.empty?
|
104
|
+
FileUtils.mkdir_p "#{target}/ignore/app/"
|
105
|
+
FileUtils.cp files, "#{target}/ignore/app/"
|
106
|
+
puts "copied #{files.size} data_migrations to db/data_migrations/ignore/app"
|
107
|
+
end
|
118
108
|
|
119
|
-
|
120
|
-
|
109
|
+
dirs = Rails::Application::Railties.engines.map{|p| p.config.root.to_s}
|
110
|
+
files = Dir["{#{dirs.join(',')}}/db/data_migrations/*.rb"]
|
121
111
|
|
122
|
-
|
123
|
-
|
124
|
-
|
125
|
-
|
112
|
+
unless files.empty?
|
113
|
+
FileUtils.mkdir_p target
|
114
|
+
FileUtils.cp files, target
|
115
|
+
puts "copied #{files.size} migrations to db/data_migrations"
|
116
|
+
end
|
126
117
|
end
|
127
|
-
end
|
128
118
|
|
129
|
-
|
130
|
-
|
119
|
+
def cleanup_migrations
|
120
|
+
target = "#{Rails.root}/db/data_migrations/"
|
131
121
|
|
132
|
-
|
133
|
-
|
134
|
-
|
135
|
-
|
136
|
-
|
137
|
-
|
138
|
-
|
139
|
-
|
140
|
-
|
122
|
+
files = Dir["#{target}*.rb"]
|
123
|
+
unless files.empty?
|
124
|
+
FileUtils.rm files
|
125
|
+
puts "removed #{files.size} data_migrations from db/data_migrations"
|
126
|
+
end
|
127
|
+
files = Dir["#{target}/ignore/app/*.rb"]
|
128
|
+
unless files.empty?
|
129
|
+
FileUtils.cp files, target
|
130
|
+
puts "copied #{files.size} data_migrations back to db/data_migrations"
|
131
|
+
end
|
132
|
+
FileUtils.rm_rf "#{target}/ignore/app"
|
141
133
|
end
|
142
|
-
FileUtils.rm_rf "#{target}/ignore/app"
|
143
|
-
end
|
144
134
|
|
145
|
-
|
135
|
+
private
|
146
136
|
|
147
|
-
|
148
|
-
|
149
|
-
|
150
|
-
|
151
|
-
|
152
|
-
|
153
|
-
|
154
|
-
|
155
|
-
FileUtils.mkdir_p(DEFAULT_MIGRATIONS_PATH + '/ignore/')
|
137
|
+
def setup
|
138
|
+
create_data_migrations_table unless data_migrations_table_exists?
|
139
|
+
|
140
|
+
unless File.directory? @default_migrations_path
|
141
|
+
FileUtils.mkdir_p( @default_migrations_path)
|
142
|
+
#create ignore folder
|
143
|
+
FileUtils.mkdir_p(File.join(@default_migrations_path,'ignore/'))
|
144
|
+
end
|
156
145
|
end
|
157
|
-
end
|
158
146
|
|
159
|
-
|
160
|
-
|
161
|
-
|
162
|
-
|
163
|
-
|
164
|
-
|
165
|
-
|
166
|
-
|
147
|
+
def handle_higher_passed_version(passed_version)
|
148
|
+
files = get_all_files
|
149
|
+
|
150
|
+
files.each do |file|
|
151
|
+
filename, version, klass_name = seperate_file_parts(file)
|
152
|
+
if version <= passed_version
|
153
|
+
unless version_has_been_migrated?(version)
|
154
|
+
handle_action(file, klass_name, version, :up)
|
155
|
+
end
|
167
156
|
end
|
168
157
|
end
|
169
|
-
end
|
170
158
|
|
171
|
-
|
159
|
+
end
|
172
160
|
|
173
|
-
|
174
|
-
|
161
|
+
def run_all_non_migrated(passed_version)
|
162
|
+
files = get_all_files
|
175
163
|
|
176
|
-
|
177
|
-
|
178
|
-
|
179
|
-
|
180
|
-
|
164
|
+
files.each do |file|
|
165
|
+
filename, version, klass_name = seperate_file_parts(file)
|
166
|
+
if passed_version.nil? or version <= passed_version
|
167
|
+
if !version_has_been_migrated?(version)
|
168
|
+
handle_action(file, klass_name, version, :up)
|
169
|
+
end
|
181
170
|
end
|
182
171
|
end
|
183
172
|
end
|
184
|
-
end
|
185
173
|
|
186
|
-
|
187
|
-
|
174
|
+
def handle_lower_passed_version(passed_version)
|
175
|
+
files = get_all_files
|
188
176
|
|
189
|
-
|
190
|
-
|
191
|
-
|
192
|
-
|
193
|
-
self.handle_action(file, klass_name, version, :down)
|
177
|
+
files.each do |file|
|
178
|
+
filename, version, klass_name = seperate_file_parts(file)
|
179
|
+
if version > passed_version
|
180
|
+
handle_action(file, klass_name, version, :down) if version_has_been_migrated?(version)
|
194
181
|
end
|
195
182
|
end
|
196
|
-
end
|
197
183
|
|
198
|
-
|
184
|
+
end
|
199
185
|
|
200
|
-
|
201
|
-
|
202
|
-
|
203
|
-
|
204
|
-
|
205
|
-
|
206
|
-
|
207
|
-
|
186
|
+
def handle_action(file, klass_name, version, action)
|
187
|
+
require file
|
188
|
+
klass = klass_name.camelize.constantize
|
189
|
+
puts "=================Migrating #{klass.to_s} #{action.to_s.upcase}============"
|
190
|
+
begin
|
191
|
+
time = Benchmark.measure do
|
192
|
+
ActiveRecord::Base.transaction do
|
193
|
+
klass.send(action.to_s)
|
194
|
+
end
|
208
195
|
end
|
196
|
+
rescue Exception=>ex
|
197
|
+
cleanup_migrations
|
198
|
+
raise ex
|
209
199
|
end
|
210
|
-
|
211
|
-
|
212
|
-
|
200
|
+
time_str = "(%.4fs)" % time.real
|
201
|
+
puts "================Finished #{klass.to_s} in #{time_str}=="
|
202
|
+
insert_migration_version(version)
|
213
203
|
end
|
214
|
-
time_str = "(%.4fs)" % time.real
|
215
|
-
puts "================Finished #{klass.to_s} in #{time_str}=="
|
216
|
-
self.insert_migration_version(version)
|
217
|
-
end
|
218
204
|
|
219
|
-
|
220
|
-
|
221
|
-
|
205
|
+
def insert_migration_version(version)
|
206
|
+
ActiveRecord::Base.connection.execute("insert into data_migrations (version) values ('#{version}')")
|
207
|
+
end
|
222
208
|
|
223
|
-
|
224
|
-
|
225
|
-
|
209
|
+
def remove_migration_version(version)
|
210
|
+
ActiveRecord::Base.connection.execute("delete from data_migrations where version = '#{version}'")
|
211
|
+
end
|
226
212
|
|
227
|
-
|
228
|
-
|
213
|
+
def version_has_been_migrated?(version)
|
214
|
+
result = true
|
229
215
|
|
230
|
-
|
216
|
+
db_result = ActiveRecord::Base.connection.select_all("select count(*) as num_rows from data_migrations where version = '#{version}'")
|
231
217
|
|
232
|
-
|
218
|
+
num_rows = db_result[0]['num_rows'] unless db_result == -1
|
233
219
|
|
234
|
-
|
235
|
-
|
236
|
-
|
220
|
+
if num_rows.nil? || num_rows.to_i == 0
|
221
|
+
result = false
|
222
|
+
end
|
237
223
|
|
238
|
-
|
239
|
-
|
224
|
+
result
|
225
|
+
end
|
240
226
|
|
241
|
-
|
242
|
-
|
243
|
-
|
244
|
-
|
227
|
+
def data_migrations_table_exists?
|
228
|
+
table_names = ActiveRecord::Base.connection.tables
|
229
|
+
table_names.include?('data_migrations')
|
230
|
+
end
|
245
231
|
|
246
|
-
|
247
|
-
|
248
|
-
|
249
|
-
|
250
|
-
|
232
|
+
def create_data_migrations_table
|
233
|
+
puts "** data_migrations table missing creating now...."
|
234
|
+
puts ActiveRecord::Migrator.run(:up, File.join(File.dirname(__FILE__),'../db/migrate/'), 20100819181805)
|
235
|
+
puts "** done"
|
236
|
+
end
|
251
237
|
|
252
|
-
|
253
|
-
|
254
|
-
|
255
|
-
|
256
|
-
|
238
|
+
def seperate_file_parts(file)
|
239
|
+
paths = file.split('/')
|
240
|
+
filename = paths[paths.length - 1]
|
241
|
+
version = filename.split('_')[0]
|
242
|
+
klass_name = filename.gsub(/#{version}/, "").gsub(/.rb/, "")[1..filename.length]
|
257
243
|
|
258
|
-
|
259
|
-
|
244
|
+
return filename, version.to_i, klass_name
|
245
|
+
end
|
260
246
|
|
261
|
-
|
262
|
-
|
247
|
+
def get_all_files
|
248
|
+
files = []
|
263
249
|
|
264
|
-
|
250
|
+
if File.directory? @default_migrations_path
|
265
251
|
|
266
|
-
|
252
|
+
files_or_directories = Dir.entries(@default_migrations_path).map{|directory| directory}
|
267
253
|
|
268
|
-
|
269
|
-
|
254
|
+
files_or_directories.delete_if{|name| name =~ REMOVE_FILES_REGEX} #remove any file leading with . or ..
|
255
|
+
files_or_directories.delete_if{|name| name == 'ignore'} #ignore the ignore folder
|
270
256
|
|
271
257
|
|
272
|
-
|
273
|
-
|
274
|
-
|
258
|
+
files_or_directories.each do |file_or_directory|
|
259
|
+
file_or_directory = @default_migrations_path + "/" + file_or_directory
|
260
|
+
files = get_files_in_directory(file_or_directory, files)
|
261
|
+
end
|
262
|
+
|
263
|
+
files.sort! {|x,y| File.basename(x) <=> File.basename(y)}
|
275
264
|
end
|
276
265
|
|
277
|
-
files
|
266
|
+
files
|
278
267
|
end
|
279
268
|
|
280
|
-
files
|
281
|
-
|
282
|
-
|
283
|
-
def self.get_files_in_directory(file_or_directory, files)
|
284
|
-
unless file_or_directory =~ /\w\.rb/
|
285
|
-
files_or_directories = Dir.entries(file_or_directory).map{|directory| directory}
|
269
|
+
def get_files_in_directory(file_or_directory, files)
|
270
|
+
unless file_or_directory =~ /\w\.rb/
|
271
|
+
files_or_directories = Dir.entries(file_or_directory).map{|directory| directory}
|
286
272
|
|
287
|
-
|
288
|
-
|
273
|
+
files_or_directories.delete_if{|name| name =~ REMOVE_FILES_REGEX} #remove any file leading with . or ..
|
274
|
+
files_or_directories.delete_if{|name| name == 'ignore'} #ignore the ignore folder
|
289
275
|
|
290
|
-
|
291
|
-
|
292
|
-
|
276
|
+
files_or_directories.each do |_file_or_directory|
|
277
|
+
_file_or_directory = file_or_directory + "/" + _file_or_directory
|
278
|
+
files = get_files_in_directory(_file_or_directory, files)
|
279
|
+
end
|
280
|
+
else
|
281
|
+
files << file_or_directory
|
293
282
|
end
|
294
|
-
else
|
295
|
-
files << file_or_directory
|
296
|
-
end
|
297
|
-
|
298
|
-
files
|
299
|
-
end
|
300
|
-
|
301
|
-
def self.get_file_name(dir)
|
302
|
-
file_name = dir
|
303
283
|
|
304
|
-
|
305
|
-
|
306
|
-
|
307
|
-
file_name
|
284
|
+
files
|
285
|
+
end
|
308
286
|
end
|
309
287
|
end
|
@@ -1,53 +1,53 @@
|
|
1
1
|
namespace :db do
|
2
2
|
desc 'migrates data into database'
|
3
3
|
task :migrate_data => :environment do
|
4
|
-
|
5
4
|
passed_version = ENV["VERSION"] ? ENV["VERSION"].to_i : nil
|
6
|
-
DataMigrator.
|
7
|
-
|
8
|
-
|
5
|
+
data_migrator = RussellEdge::DataMigrator.new
|
6
|
+
data_migrator.prepare_migrations
|
7
|
+
data_migrator.migrate(nil, passed_version)
|
8
|
+
data_migrator.cleanup_migrations
|
9
9
|
|
10
|
-
end#end task
|
10
|
+
end#end migrate_data task
|
11
11
|
|
12
12
|
namespace :migrate_data do
|
13
13
|
task :list_pending => :environment do
|
14
|
-
DataMigrator.
|
15
|
-
|
14
|
+
data_migrator = RussellEdge::DataMigrator.new
|
15
|
+
data_migrator.prepare_migrations
|
16
|
+
pending_migrations = data_migrator.pending_migrations
|
16
17
|
puts "================Pending Data Migrations=========="
|
17
18
|
puts pending_migrations
|
18
19
|
puts "================================================="
|
19
|
-
|
20
|
-
end
|
20
|
+
data_migrator.cleanup_migrations
|
21
|
+
end#end list_pending task
|
21
22
|
|
22
23
|
task :version => :environment do
|
23
|
-
|
24
|
+
data_migrator = RussellEdge::DataMigrator.new
|
25
|
+
version = data_migrator.get_current_version
|
24
26
|
|
25
|
-
|
26
|
-
|
27
|
-
else
|
28
|
-
puts "** No migrations ran"
|
29
|
-
end
|
30
|
-
end
|
27
|
+
puts (version.nil?) ? "** No migrations ran" : "** Current version #{version}"
|
28
|
+
end#end version task
|
31
29
|
|
32
30
|
task :up => :environment do
|
33
31
|
passed_version = ENV["VERSION"] ? ENV["VERSION"].to_i : nil
|
34
32
|
raise "VERSION is required" unless passed_version
|
33
|
+
|
34
|
+
data_migrator = RussellEdge::DataMigrator.new
|
35
|
+
data_migrator.prepare_migrations
|
36
|
+
data_migrator.run_up(passed_version)
|
37
|
+
data_migrator.cleanup_migrations
|
35
38
|
|
36
|
-
|
37
|
-
DataMigrator.run_up(passed_version)
|
38
|
-
DataMigrator.cleanup_migrations
|
39
|
-
|
40
|
-
end#end task
|
39
|
+
end#end up task
|
41
40
|
|
42
41
|
task :down => :environment do
|
43
42
|
passed_version = ENV["VERSION"] ? ENV["VERSION"].to_i : nil
|
44
43
|
raise "VERSION is required" unless passed_version
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
44
|
+
|
45
|
+
data_migrator = RussellEdge::DataMigrator.new
|
46
|
+
data_migrator.prepare_migrations
|
47
|
+
data_migrator.run_down(passed_version)
|
48
|
+
data_migrator.cleanup_migrations
|
49
49
|
|
50
|
-
end#end task
|
50
|
+
end#end down task
|
51
51
|
end#end namespace
|
52
52
|
end
|
53
53
|
|
metadata
CHANGED
@@ -1,12 +1,12 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: data_migrator
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
hash:
|
4
|
+
hash: 7
|
5
5
|
prerelease: false
|
6
6
|
segments:
|
7
7
|
- 1
|
8
|
-
-
|
9
|
-
version: "1.
|
8
|
+
- 4
|
9
|
+
version: "1.4"
|
10
10
|
platform: ruby
|
11
11
|
authors:
|
12
12
|
- Russell Holmes, Adam Hull
|
@@ -14,7 +14,7 @@ autorequire:
|
|
14
14
|
bindir: bin
|
15
15
|
cert_chain: []
|
16
16
|
|
17
|
-
date: 2011-08-
|
17
|
+
date: 2011-08-25 00:00:00 -04:00
|
18
18
|
default_executable:
|
19
19
|
dependencies:
|
20
20
|
- !ruby/object:Gem::Dependency
|
@@ -48,7 +48,6 @@ files:
|
|
48
48
|
- lib/generators/data_migration/templates/migration_template.rb
|
49
49
|
- lib/generators/data_migration/USAGE
|
50
50
|
- lib/tasks/data_migrator.rake
|
51
|
-
- lib/version.rb
|
52
51
|
- MIT-LICENSE
|
53
52
|
- README.rdoc
|
54
53
|
- test/data_migrator_test.rb
|
data/lib/version.rb
DELETED