dump 1.0.4 → 1.0.5
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +8 -8
- data/.rubocop.yml +53 -0
- data/.rubocop_todo.yml +33 -0
- data/.travis.yml +12 -3
- data/README.markdown +8 -0
- data/dump.gemspec +4 -1
- data/lib/dump.rb +1 -0
- data/lib/dump/capistrano.rb +7 -1
- data/lib/dump/capistrano/v2.rb +331 -0
- data/lib/dump/railtie.rb +1 -0
- data/lib/dump_rake.rb +17 -8
- data/lib/dump_rake/archive_tar_minitar_fix.rb +3 -1
- data/lib/dump_rake/assets.rb +1 -0
- data/lib/dump_rake/continious_timeout.rb +17 -17
- data/lib/dump_rake/dump.rb +27 -22
- data/lib/dump_rake/dump_reader.rb +62 -63
- data/lib/dump_rake/dump_writer.rb +20 -21
- data/lib/dump_rake/env.rb +12 -10
- data/lib/dump_rake/env/filter.rb +3 -0
- data/lib/dump_rake/rails_root.rb +1 -0
- data/lib/dump_rake/table_manipulation.rb +23 -20
- data/lib/generators/assets_config/assets_config_generator.rb +2 -0
- data/lib/tasks/assets.rake +1 -2
- data/lib/tasks/dump.rake +5 -6
- data/recipes/dump.rb +1 -343
- data/script/update_readme +5 -3
- data/spec/cycle_spec.rb +33 -30
- data/spec/dummy_rails_app.rb +42 -0
- data/spec/lib/dump_rake/dump_reader_spec.rb +80 -92
- data/spec/lib/dump_rake/dump_spec.rb +56 -58
- data/spec/lib/dump_rake/dump_writer_spec.rb +42 -43
- data/spec/lib/dump_rake/env/filter_spec.rb +9 -9
- data/spec/lib/dump_rake/env_spec.rb +21 -21
- data/spec/lib/dump_rake/rails_root_spec.rb +7 -7
- data/spec/lib/dump_rake/table_manipulation_spec.rb +57 -63
- data/spec/lib/dump_rake_spec.rb +76 -76
- data/spec/recipes/dump_spec.rb +241 -217
- data/spec/spec_helper.rb +1 -42
- data/spec/tasks/assets_spec.rb +18 -18
- data/spec/tasks/dump_spec.rb +18 -18
- metadata +21 -2
data/lib/dump/railtie.rb
CHANGED
data/lib/dump_rake.rb
CHANGED
@@ -11,11 +11,24 @@ require 'archive/tar/minitar'
|
|
11
11
|
require 'dump_rake/archive_tar_minitar_fix'
|
12
12
|
require 'progress'
|
13
13
|
|
14
|
+
require 'dump_rake/rails_root'
|
15
|
+
require 'dump_rake/assets'
|
16
|
+
require 'dump_rake/table_manipulation'
|
17
|
+
require 'dump_rake/dump'
|
18
|
+
require 'dump_rake/dump_reader'
|
19
|
+
require 'dump_rake/dump_writer'
|
20
|
+
require 'dump_rake/env'
|
21
|
+
|
22
|
+
# Main interface
|
14
23
|
class DumpRake
|
15
24
|
class << self
|
16
25
|
def versions(options = {})
|
17
26
|
Dump.list(options).each do |dump|
|
18
|
-
|
27
|
+
if DumpRake::Env[:show_size] || $stdout.tty?
|
28
|
+
puts "#{dump.human_size.to_s.rjust(7)}\t#{dump}"
|
29
|
+
else
|
30
|
+
puts dump
|
31
|
+
end
|
19
32
|
begin
|
20
33
|
case options[:summary].to_s.downcase[0, 1]
|
21
34
|
when *%w[1 t y]
|
@@ -47,14 +60,14 @@ class DumpRake
|
|
47
60
|
if dump
|
48
61
|
DumpReader.restore(dump.path)
|
49
62
|
else
|
50
|
-
$stderr.puts
|
63
|
+
$stderr.puts 'Avaliable versions:'
|
51
64
|
$stderr.puts Dump.list
|
52
65
|
end
|
53
66
|
end
|
54
67
|
|
55
68
|
def cleanup(options = {})
|
56
|
-
unless options[:leave].nil? || /^\d+$/
|
57
|
-
|
69
|
+
unless options[:leave].nil? || /^\d+$/ =~ options[:leave] || options[:leave].downcase == 'none'
|
70
|
+
fail 'LEAVE should be number or "none"'
|
58
71
|
end
|
59
72
|
|
60
73
|
to_delete = []
|
@@ -79,7 +92,3 @@ class DumpRake
|
|
79
92
|
end
|
80
93
|
end
|
81
94
|
end
|
82
|
-
|
83
|
-
%w[rails_root assets table_manipulation dump dump_reader dump_writer env].each do |file|
|
84
|
-
require "dump_rake/#{file}"
|
85
|
-
end
|
data/lib/dump_rake/assets.rb
CHANGED
@@ -1,38 +1,38 @@
|
|
1
1
|
# based on Timeout
|
2
2
|
|
3
|
-
|
4
|
-
|
5
|
-
|
3
|
+
class DumpRake
|
4
|
+
# Timeout if does not finish or defer in requested time
|
5
|
+
module ContiniousTimeout
|
6
|
+
class TimeoutException < ::Exception; end
|
6
7
|
|
7
|
-
|
8
|
-
end
|
8
|
+
class RestartException < ::Exception; end
|
9
9
|
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
10
|
+
# Object with defer method
|
11
|
+
class Deferer
|
12
|
+
def initialize(thread)
|
13
|
+
@thread = thread
|
14
|
+
end
|
14
15
|
|
15
|
-
|
16
|
-
|
16
|
+
def defer
|
17
|
+
@thread.raise RestartException.new
|
18
|
+
end
|
17
19
|
end
|
18
|
-
end
|
19
20
|
|
20
|
-
|
21
|
-
begin
|
21
|
+
def self.timeout(sec)
|
22
22
|
x = Thread.current
|
23
23
|
y = Thread.start do
|
24
24
|
1.times do
|
25
25
|
begin
|
26
26
|
sleep sec
|
27
|
-
rescue RestartException
|
27
|
+
rescue RestartException
|
28
28
|
retry
|
29
29
|
end
|
30
30
|
end
|
31
|
-
x.raise TimeoutException,
|
31
|
+
x.raise TimeoutException, 'execution expired' if x.alive?
|
32
32
|
end
|
33
33
|
yield Deferer.new(y)
|
34
34
|
ensure
|
35
|
-
y.kill if y
|
35
|
+
y.kill if y && y.alive?
|
36
36
|
end
|
37
37
|
end
|
38
38
|
end
|
data/lib/dump_rake/dump.rb
CHANGED
@@ -1,6 +1,7 @@
|
|
1
1
|
# encoding: UTF-8
|
2
2
|
|
3
3
|
class DumpRake
|
4
|
+
# Base class for dump
|
4
5
|
class Dump
|
5
6
|
include TableManipulation
|
6
7
|
def self.list(options = {})
|
@@ -19,7 +20,7 @@ class DumpRake
|
|
19
20
|
if path_or_options.is_a?(Hash)
|
20
21
|
options = path_or_options
|
21
22
|
|
22
|
-
name = Time.now.utc.strftime(
|
23
|
+
name = Time.now.utc.strftime('%Y%m%d%H%M%S')
|
23
24
|
|
24
25
|
description = clean_description(options[:desc])
|
25
26
|
name += "-#{description}" unless description.blank?
|
@@ -52,12 +53,12 @@ class DumpRake
|
|
52
53
|
|
53
54
|
def parts
|
54
55
|
@parts ||=
|
55
|
-
if m = name.match(/^(\d{#{4+2+2 + 2+2+2}})(-[^@]+)?((?:@[^@]+)+)?\.(tmp|tgz)$/)
|
56
|
+
if (m = name.match(/^(\d{#{4 + 2 + 2 + 2 + 2 + 2}})(-[^@]+)?((?:@[^@]+)+)?\.(tmp|tgz)$/))
|
56
57
|
{
|
57
58
|
:time => m[1],
|
58
59
|
:desc => m[2] && m[2][1, m[2].length],
|
59
60
|
:tags => m[3] && m[3][1, m[3].length],
|
60
|
-
:ext => m[4]
|
61
|
+
:ext => m[4],
|
61
62
|
}
|
62
63
|
else
|
63
64
|
{}
|
@@ -83,7 +84,7 @@ class DumpRake
|
|
83
84
|
def name
|
84
85
|
@name ||= File.basename(path)
|
85
86
|
end
|
86
|
-
|
87
|
+
alias_method :to_s, :name
|
87
88
|
|
88
89
|
def size
|
89
90
|
File.size(path) rescue nil
|
@@ -93,29 +94,30 @@ class DumpRake
|
|
93
94
|
number = size
|
94
95
|
return nil if number.nil?
|
95
96
|
degree = 0
|
96
|
-
symbols = %
|
97
|
+
symbols = %w[B K M G T]
|
97
98
|
while number >= 1000 && degree < symbols.length - 1
|
98
99
|
degree += 1
|
99
100
|
number /= 1024.0
|
100
101
|
end
|
101
|
-
|
102
|
+
format('%.2f%s', number, symbols[degree])
|
102
103
|
end
|
103
104
|
|
104
105
|
def inspect
|
105
|
-
"
|
106
|
+
"#<#{self.class}:0x#{object_id} #{path}>"
|
106
107
|
end
|
107
108
|
|
108
109
|
def lock
|
109
|
-
|
110
|
-
|
111
|
-
|
112
|
-
|
113
|
-
end
|
114
|
-
ensure
|
115
|
-
lock.flock(File::LOCK_UN)
|
116
|
-
lock.close
|
110
|
+
lock = File.open(path, 'r')
|
111
|
+
begin
|
112
|
+
if lock.flock(File::LOCK_EX | File::LOCK_NB)
|
113
|
+
yield
|
117
114
|
end
|
115
|
+
ensure
|
116
|
+
lock.flock(File::LOCK_UN)
|
117
|
+
lock.close
|
118
118
|
end
|
119
|
+
rescue Errno::ENOENT
|
120
|
+
nil
|
119
121
|
end
|
120
122
|
|
121
123
|
def silence(&block)
|
@@ -142,32 +144,35 @@ class DumpRake
|
|
142
144
|
Pathname(path.to_s.sub(/#{parts[:ext]}$/, ext))
|
143
145
|
end
|
144
146
|
|
147
|
+
# Cleanup name of dump
|
145
148
|
module CleanNParse
|
146
149
|
def clean_str(str, additional = nil)
|
147
150
|
str.to_s.strip.gsub(/\s+/, ' ').gsub(/[^A-Za-z0-9 \-_#{Regexp.escape(additional.to_s) if additional}]+/, '_')
|
148
151
|
end
|
152
|
+
|
149
153
|
def clean_description(description)
|
150
154
|
clean_str(description, '()#')[0, 50].strip
|
151
155
|
end
|
156
|
+
|
152
157
|
def clean_tag(tag)
|
153
158
|
clean_str(tag).downcase.sub(/^\-+/, '')[0, 20].strip
|
154
159
|
end
|
160
|
+
|
155
161
|
def clean_tags(tags)
|
156
162
|
tags.to_s.split(',').map{ |tag| clean_tag(tag) }.uniq.reject(&:blank?).sort
|
157
163
|
end
|
164
|
+
|
158
165
|
def get_filter_tags(tags)
|
159
166
|
groups = Hash.new{ |hash, key| hash[key] = SortedSet.new }
|
160
167
|
tags.to_s.split(',').each do |tag|
|
161
|
-
|
162
|
-
|
163
|
-
|
164
|
-
|
165
|
-
end
|
166
|
-
end
|
168
|
+
next unless (m = tag.strip.match(/^(\-|\+)?(.*)$/))
|
169
|
+
type = {'+' => :mandatory, '-' => :forbidden}[m[1]] || :simple
|
170
|
+
next unless (cleaned_tag = clean_tag(m[2])).present?
|
171
|
+
groups[type] << cleaned_tag
|
167
172
|
end
|
168
173
|
[:simple, :mandatory].each do |type|
|
169
174
|
if (clashing = (groups[type] & groups[:forbidden])).present?
|
170
|
-
|
175
|
+
fail "#{type} tags clashes with forbidden ones: #{clashing}"
|
171
176
|
end
|
172
177
|
end
|
173
178
|
groups.each_with_object({}){ |(key, value), hsh| hsh[key] = value.to_a }
|
@@ -1,4 +1,5 @@
|
|
1
1
|
class DumpRake
|
2
|
+
# Reading dump
|
2
3
|
class DumpReader < Dump
|
3
4
|
attr_reader :stream, :config
|
4
5
|
|
@@ -15,6 +16,7 @@ class DumpRake
|
|
15
16
|
end
|
16
17
|
end
|
17
18
|
|
19
|
+
# Helper class for building summary of dump
|
18
20
|
class Summary
|
19
21
|
attr_reader :text
|
20
22
|
alias_method :to_s, :text
|
@@ -54,11 +56,11 @@ class DumpRake
|
|
54
56
|
if assets.present?
|
55
57
|
sum.header 'Assets'
|
56
58
|
sum.data assets.sort.map{ |entry|
|
57
|
-
if String
|
59
|
+
if entry.is_a?(String)
|
58
60
|
entry
|
59
61
|
else
|
60
62
|
asset, paths = entry
|
61
|
-
if Hash
|
63
|
+
if paths.is_a?(Hash)
|
62
64
|
"#{asset}: #{Summary.pluralize paths[:files], 'file'} (#{Summary.pluralize paths[:total], 'entry'} total)"
|
63
65
|
else
|
64
66
|
"#{asset}: #{Summary.pluralize paths, 'entry'}"
|
@@ -87,7 +89,7 @@ class DumpRake
|
|
87
89
|
|
88
90
|
def find_entry(matcher)
|
89
91
|
stream.each do |entry|
|
90
|
-
if
|
92
|
+
if entry.full_name.match(matcher)
|
91
93
|
# we can not return entry - after exiting stream.each the entry will be invalid and will read from tar start
|
92
94
|
return yield(entry)
|
93
95
|
end
|
@@ -120,27 +122,26 @@ class DumpRake
|
|
120
122
|
Rake::Task['db:drop'].invoke
|
121
123
|
Rake::Task['db:create'].invoke
|
122
124
|
when !DumpRake::Env.no?(:migrate_down)
|
123
|
-
|
124
|
-
|
125
|
-
|
126
|
-
|
127
|
-
|
128
|
-
|
129
|
-
|
130
|
-
|
131
|
-
|
132
|
-
|
133
|
-
|
134
|
-
|
135
|
-
|
136
|
-
|
137
|
-
|
138
|
-
|
139
|
-
|
140
|
-
|
141
|
-
end
|
142
|
-
task.reenable
|
125
|
+
return unless avaliable_tables.include?('schema_migrations')
|
126
|
+
find_entry('schema_migrations.dump') do |entry|
|
127
|
+
migrated = table_rows('schema_migrations').map{ |row| row['version'] }
|
128
|
+
|
129
|
+
dump_migrations = []
|
130
|
+
Marshal.load(entry) # skip header
|
131
|
+
dump_migrations << Marshal.load(entry).first until entry.eof?
|
132
|
+
|
133
|
+
migrate_down = (migrated - dump_migrations)
|
134
|
+
|
135
|
+
unless migrate_down.empty?
|
136
|
+
migrate_down.reverse.with_progress('Migrating down') do |version|
|
137
|
+
DumpRake::Env.with_env('VERSION' => version) do
|
138
|
+
Rake::Task['db:migrate:down'].tap do |task|
|
139
|
+
begin
|
140
|
+
task.invoke
|
141
|
+
rescue ActiveRecord::IrreversibleMigration
|
142
|
+
$stderr.puts "Irreversible migration: #{version}"
|
143
143
|
end
|
144
|
+
task.reenable
|
144
145
|
end
|
145
146
|
end
|
146
147
|
end
|
@@ -154,13 +155,12 @@ class DumpRake
|
|
154
155
|
end
|
155
156
|
|
156
157
|
def read_schema
|
157
|
-
|
158
|
-
|
159
|
-
|
160
|
-
|
161
|
-
end
|
162
|
-
Rake::Task['db:schema:dump'].invoke
|
158
|
+
return unless restore_schema?
|
159
|
+
read_entry_to_file('schema.rb') do |f|
|
160
|
+
DumpRake::Env.with_env('SCHEMA' => f.path) do
|
161
|
+
Rake::Task['db:schema:load'].invoke
|
163
162
|
end
|
163
|
+
Rake::Task['db:schema:dump'].invoke
|
164
164
|
end
|
165
165
|
end
|
166
166
|
|
@@ -209,42 +209,41 @@ class DumpRake
|
|
209
209
|
|
210
210
|
def read_assets
|
211
211
|
return if DumpRake::Env[:restore_assets] && DumpRake::Env[:restore_assets].empty?
|
212
|
-
|
213
|
-
|
214
|
-
|
215
|
-
|
216
|
-
|
217
|
-
|
218
|
-
|
219
|
-
|
212
|
+
return if config[:assets].blank?
|
213
|
+
|
214
|
+
assets = config[:assets]
|
215
|
+
if assets.is_a?(Hash)
|
216
|
+
assets_count = assets.values.sum{ |value| value.is_a?(Hash) ? value[:total] : value }
|
217
|
+
assets_paths = assets.keys
|
218
|
+
else
|
219
|
+
assets_count, assets_paths = nil, assets
|
220
|
+
end
|
220
221
|
|
221
|
-
|
222
|
-
|
223
|
-
|
224
|
-
|
225
|
-
|
226
|
-
|
227
|
-
|
228
|
-
|
229
|
-
|
230
|
-
|
231
|
-
|
232
|
-
|
233
|
-
end
|
234
|
-
end
|
222
|
+
if DumpRake::Env[:restore_assets]
|
223
|
+
assets_paths.each do |asset|
|
224
|
+
DumpRake::Assets.glob_asset_children(asset, '**/*').reverse.each do |child|
|
225
|
+
next unless read_asset?(child, DumpRake::RailsRoot)
|
226
|
+
case
|
227
|
+
when File.file?(child)
|
228
|
+
File.unlink(child)
|
229
|
+
when File.directory?(child)
|
230
|
+
begin
|
231
|
+
Dir.unlink(child)
|
232
|
+
rescue Errno::ENOTEMPTY
|
233
|
+
nil
|
235
234
|
end
|
236
235
|
end
|
237
236
|
end
|
238
|
-
else
|
239
|
-
DumpRake::Env.with_env(:assets => assets_paths.join(':')) do
|
240
|
-
Rake::Task['assets:delete'].invoke
|
241
|
-
end
|
242
237
|
end
|
238
|
+
else
|
239
|
+
DumpRake::Env.with_env(:assets => assets_paths.join(':')) do
|
240
|
+
Rake::Task['assets:delete'].invoke
|
241
|
+
end
|
242
|
+
end
|
243
243
|
|
244
|
-
|
245
|
-
|
246
|
-
|
247
|
-
end
|
244
|
+
read_assets_entries(assets_paths, assets_count) do |stream, root, entry, prefix|
|
245
|
+
if !DumpRake::Env[:restore_assets] || read_asset?(entry.full_name, prefix)
|
246
|
+
stream.extract_entry(root, entry)
|
248
247
|
end
|
249
248
|
end
|
250
249
|
end
|
@@ -252,14 +251,14 @@ class DumpRake
|
|
252
251
|
def read_asset?(path, prefix)
|
253
252
|
DumpRake::Env.filter(:restore_assets, DumpRake::Assets::SPLITTER).custom_pass? do |value|
|
254
253
|
File.fnmatch(File.join(prefix, value), path) ||
|
255
|
-
|
254
|
+
File.fnmatch(File.join(prefix, value, '**'), path)
|
256
255
|
end
|
257
256
|
end
|
258
257
|
|
259
|
-
def read_assets_entries(
|
258
|
+
def read_assets_entries(_assets_paths, assets_count)
|
260
259
|
Progress.start('Assets', assets_count || 1) do
|
261
260
|
found_assets = false
|
262
|
-
# old style
|
261
|
+
# old style - in separate tar
|
263
262
|
find_entry('assets.tar') do |assets_tar|
|
264
263
|
def assets_tar.rewind
|
265
264
|
# rewind will fail - it must go to center of gzip
|
@@ -275,7 +274,7 @@ class DumpRake
|
|
275
274
|
end
|
276
275
|
|
277
276
|
unless found_assets
|
278
|
-
# new style
|
277
|
+
# new style - in same tar
|
279
278
|
assets_root_link do |tmpdir, prefix|
|
280
279
|
stream.each do |entry|
|
281
280
|
if entry.full_name.starts_with?("#{prefix}/")
|