dump 1.0.4 → 1.0.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +8 -8
- data/.rubocop.yml +53 -0
- data/.rubocop_todo.yml +33 -0
- data/.travis.yml +12 -3
- data/README.markdown +8 -0
- data/dump.gemspec +4 -1
- data/lib/dump.rb +1 -0
- data/lib/dump/capistrano.rb +7 -1
- data/lib/dump/capistrano/v2.rb +331 -0
- data/lib/dump/railtie.rb +1 -0
- data/lib/dump_rake.rb +17 -8
- data/lib/dump_rake/archive_tar_minitar_fix.rb +3 -1
- data/lib/dump_rake/assets.rb +1 -0
- data/lib/dump_rake/continious_timeout.rb +17 -17
- data/lib/dump_rake/dump.rb +27 -22
- data/lib/dump_rake/dump_reader.rb +62 -63
- data/lib/dump_rake/dump_writer.rb +20 -21
- data/lib/dump_rake/env.rb +12 -10
- data/lib/dump_rake/env/filter.rb +3 -0
- data/lib/dump_rake/rails_root.rb +1 -0
- data/lib/dump_rake/table_manipulation.rb +23 -20
- data/lib/generators/assets_config/assets_config_generator.rb +2 -0
- data/lib/tasks/assets.rake +1 -2
- data/lib/tasks/dump.rake +5 -6
- data/recipes/dump.rb +1 -343
- data/script/update_readme +5 -3
- data/spec/cycle_spec.rb +33 -30
- data/spec/dummy_rails_app.rb +42 -0
- data/spec/lib/dump_rake/dump_reader_spec.rb +80 -92
- data/spec/lib/dump_rake/dump_spec.rb +56 -58
- data/spec/lib/dump_rake/dump_writer_spec.rb +42 -43
- data/spec/lib/dump_rake/env/filter_spec.rb +9 -9
- data/spec/lib/dump_rake/env_spec.rb +21 -21
- data/spec/lib/dump_rake/rails_root_spec.rb +7 -7
- data/spec/lib/dump_rake/table_manipulation_spec.rb +57 -63
- data/spec/lib/dump_rake_spec.rb +76 -76
- data/spec/recipes/dump_spec.rb +241 -217
- data/spec/spec_helper.rb +1 -42
- data/spec/tasks/assets_spec.rb +18 -18
- data/spec/tasks/dump_spec.rb +18 -18
- metadata +21 -2
data/lib/dump/railtie.rb
CHANGED
data/lib/dump_rake.rb
CHANGED
@@ -11,11 +11,24 @@ require 'archive/tar/minitar'
|
|
11
11
|
require 'dump_rake/archive_tar_minitar_fix'
|
12
12
|
require 'progress'
|
13
13
|
|
14
|
+
require 'dump_rake/rails_root'
|
15
|
+
require 'dump_rake/assets'
|
16
|
+
require 'dump_rake/table_manipulation'
|
17
|
+
require 'dump_rake/dump'
|
18
|
+
require 'dump_rake/dump_reader'
|
19
|
+
require 'dump_rake/dump_writer'
|
20
|
+
require 'dump_rake/env'
|
21
|
+
|
22
|
+
# Main interface
|
14
23
|
class DumpRake
|
15
24
|
class << self
|
16
25
|
def versions(options = {})
|
17
26
|
Dump.list(options).each do |dump|
|
18
|
-
|
27
|
+
if DumpRake::Env[:show_size] || $stdout.tty?
|
28
|
+
puts "#{dump.human_size.to_s.rjust(7)}\t#{dump}"
|
29
|
+
else
|
30
|
+
puts dump
|
31
|
+
end
|
19
32
|
begin
|
20
33
|
case options[:summary].to_s.downcase[0, 1]
|
21
34
|
when *%w[1 t y]
|
@@ -47,14 +60,14 @@ class DumpRake
|
|
47
60
|
if dump
|
48
61
|
DumpReader.restore(dump.path)
|
49
62
|
else
|
50
|
-
$stderr.puts
|
63
|
+
$stderr.puts 'Avaliable versions:'
|
51
64
|
$stderr.puts Dump.list
|
52
65
|
end
|
53
66
|
end
|
54
67
|
|
55
68
|
def cleanup(options = {})
|
56
|
-
unless options[:leave].nil? || /^\d+$/
|
57
|
-
|
69
|
+
unless options[:leave].nil? || /^\d+$/ =~ options[:leave] || options[:leave].downcase == 'none'
|
70
|
+
fail 'LEAVE should be number or "none"'
|
58
71
|
end
|
59
72
|
|
60
73
|
to_delete = []
|
@@ -79,7 +92,3 @@ class DumpRake
|
|
79
92
|
end
|
80
93
|
end
|
81
94
|
end
|
82
|
-
|
83
|
-
%w[rails_root assets table_manipulation dump dump_reader dump_writer env].each do |file|
|
84
|
-
require "dump_rake/#{file}"
|
85
|
-
end
|
data/lib/dump_rake/assets.rb
CHANGED
@@ -1,38 +1,38 @@
|
|
1
1
|
# based on Timeout
|
2
2
|
|
3
|
-
|
4
|
-
|
5
|
-
|
3
|
+
class DumpRake
|
4
|
+
# Timeout if does not finish or defer in requested time
|
5
|
+
module ContiniousTimeout
|
6
|
+
class TimeoutException < ::Exception; end
|
6
7
|
|
7
|
-
|
8
|
-
end
|
8
|
+
class RestartException < ::Exception; end
|
9
9
|
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
10
|
+
# Object with defer method
|
11
|
+
class Deferer
|
12
|
+
def initialize(thread)
|
13
|
+
@thread = thread
|
14
|
+
end
|
14
15
|
|
15
|
-
|
16
|
-
|
16
|
+
def defer
|
17
|
+
@thread.raise RestartException.new
|
18
|
+
end
|
17
19
|
end
|
18
|
-
end
|
19
20
|
|
20
|
-
|
21
|
-
begin
|
21
|
+
def self.timeout(sec)
|
22
22
|
x = Thread.current
|
23
23
|
y = Thread.start do
|
24
24
|
1.times do
|
25
25
|
begin
|
26
26
|
sleep sec
|
27
|
-
rescue RestartException
|
27
|
+
rescue RestartException
|
28
28
|
retry
|
29
29
|
end
|
30
30
|
end
|
31
|
-
x.raise TimeoutException,
|
31
|
+
x.raise TimeoutException, 'execution expired' if x.alive?
|
32
32
|
end
|
33
33
|
yield Deferer.new(y)
|
34
34
|
ensure
|
35
|
-
y.kill if y
|
35
|
+
y.kill if y && y.alive?
|
36
36
|
end
|
37
37
|
end
|
38
38
|
end
|
data/lib/dump_rake/dump.rb
CHANGED
@@ -1,6 +1,7 @@
|
|
1
1
|
# encoding: UTF-8
|
2
2
|
|
3
3
|
class DumpRake
|
4
|
+
# Base class for dump
|
4
5
|
class Dump
|
5
6
|
include TableManipulation
|
6
7
|
def self.list(options = {})
|
@@ -19,7 +20,7 @@ class DumpRake
|
|
19
20
|
if path_or_options.is_a?(Hash)
|
20
21
|
options = path_or_options
|
21
22
|
|
22
|
-
name = Time.now.utc.strftime(
|
23
|
+
name = Time.now.utc.strftime('%Y%m%d%H%M%S')
|
23
24
|
|
24
25
|
description = clean_description(options[:desc])
|
25
26
|
name += "-#{description}" unless description.blank?
|
@@ -52,12 +53,12 @@ class DumpRake
|
|
52
53
|
|
53
54
|
def parts
|
54
55
|
@parts ||=
|
55
|
-
if m = name.match(/^(\d{#{4+2+2 + 2+2+2}})(-[^@]+)?((?:@[^@]+)+)?\.(tmp|tgz)$/)
|
56
|
+
if (m = name.match(/^(\d{#{4 + 2 + 2 + 2 + 2 + 2}})(-[^@]+)?((?:@[^@]+)+)?\.(tmp|tgz)$/))
|
56
57
|
{
|
57
58
|
:time => m[1],
|
58
59
|
:desc => m[2] && m[2][1, m[2].length],
|
59
60
|
:tags => m[3] && m[3][1, m[3].length],
|
60
|
-
:ext => m[4]
|
61
|
+
:ext => m[4],
|
61
62
|
}
|
62
63
|
else
|
63
64
|
{}
|
@@ -83,7 +84,7 @@ class DumpRake
|
|
83
84
|
def name
|
84
85
|
@name ||= File.basename(path)
|
85
86
|
end
|
86
|
-
|
87
|
+
alias_method :to_s, :name
|
87
88
|
|
88
89
|
def size
|
89
90
|
File.size(path) rescue nil
|
@@ -93,29 +94,30 @@ class DumpRake
|
|
93
94
|
number = size
|
94
95
|
return nil if number.nil?
|
95
96
|
degree = 0
|
96
|
-
symbols = %
|
97
|
+
symbols = %w[B K M G T]
|
97
98
|
while number >= 1000 && degree < symbols.length - 1
|
98
99
|
degree += 1
|
99
100
|
number /= 1024.0
|
100
101
|
end
|
101
|
-
|
102
|
+
format('%.2f%s', number, symbols[degree])
|
102
103
|
end
|
103
104
|
|
104
105
|
def inspect
|
105
|
-
"
|
106
|
+
"#<#{self.class}:0x#{object_id} #{path}>"
|
106
107
|
end
|
107
108
|
|
108
109
|
def lock
|
109
|
-
|
110
|
-
|
111
|
-
|
112
|
-
|
113
|
-
end
|
114
|
-
ensure
|
115
|
-
lock.flock(File::LOCK_UN)
|
116
|
-
lock.close
|
110
|
+
lock = File.open(path, 'r')
|
111
|
+
begin
|
112
|
+
if lock.flock(File::LOCK_EX | File::LOCK_NB)
|
113
|
+
yield
|
117
114
|
end
|
115
|
+
ensure
|
116
|
+
lock.flock(File::LOCK_UN)
|
117
|
+
lock.close
|
118
118
|
end
|
119
|
+
rescue Errno::ENOENT
|
120
|
+
nil
|
119
121
|
end
|
120
122
|
|
121
123
|
def silence(&block)
|
@@ -142,32 +144,35 @@ class DumpRake
|
|
142
144
|
Pathname(path.to_s.sub(/#{parts[:ext]}$/, ext))
|
143
145
|
end
|
144
146
|
|
147
|
+
# Cleanup name of dump
|
145
148
|
module CleanNParse
|
146
149
|
def clean_str(str, additional = nil)
|
147
150
|
str.to_s.strip.gsub(/\s+/, ' ').gsub(/[^A-Za-z0-9 \-_#{Regexp.escape(additional.to_s) if additional}]+/, '_')
|
148
151
|
end
|
152
|
+
|
149
153
|
def clean_description(description)
|
150
154
|
clean_str(description, '()#')[0, 50].strip
|
151
155
|
end
|
156
|
+
|
152
157
|
def clean_tag(tag)
|
153
158
|
clean_str(tag).downcase.sub(/^\-+/, '')[0, 20].strip
|
154
159
|
end
|
160
|
+
|
155
161
|
def clean_tags(tags)
|
156
162
|
tags.to_s.split(',').map{ |tag| clean_tag(tag) }.uniq.reject(&:blank?).sort
|
157
163
|
end
|
164
|
+
|
158
165
|
def get_filter_tags(tags)
|
159
166
|
groups = Hash.new{ |hash, key| hash[key] = SortedSet.new }
|
160
167
|
tags.to_s.split(',').each do |tag|
|
161
|
-
|
162
|
-
|
163
|
-
|
164
|
-
|
165
|
-
end
|
166
|
-
end
|
168
|
+
next unless (m = tag.strip.match(/^(\-|\+)?(.*)$/))
|
169
|
+
type = {'+' => :mandatory, '-' => :forbidden}[m[1]] || :simple
|
170
|
+
next unless (cleaned_tag = clean_tag(m[2])).present?
|
171
|
+
groups[type] << cleaned_tag
|
167
172
|
end
|
168
173
|
[:simple, :mandatory].each do |type|
|
169
174
|
if (clashing = (groups[type] & groups[:forbidden])).present?
|
170
|
-
|
175
|
+
fail "#{type} tags clashes with forbidden ones: #{clashing}"
|
171
176
|
end
|
172
177
|
end
|
173
178
|
groups.each_with_object({}){ |(key, value), hsh| hsh[key] = value.to_a }
|
@@ -1,4 +1,5 @@
|
|
1
1
|
class DumpRake
|
2
|
+
# Reading dump
|
2
3
|
class DumpReader < Dump
|
3
4
|
attr_reader :stream, :config
|
4
5
|
|
@@ -15,6 +16,7 @@ class DumpRake
|
|
15
16
|
end
|
16
17
|
end
|
17
18
|
|
19
|
+
# Helper class for building summary of dump
|
18
20
|
class Summary
|
19
21
|
attr_reader :text
|
20
22
|
alias_method :to_s, :text
|
@@ -54,11 +56,11 @@ class DumpRake
|
|
54
56
|
if assets.present?
|
55
57
|
sum.header 'Assets'
|
56
58
|
sum.data assets.sort.map{ |entry|
|
57
|
-
if String
|
59
|
+
if entry.is_a?(String)
|
58
60
|
entry
|
59
61
|
else
|
60
62
|
asset, paths = entry
|
61
|
-
if Hash
|
63
|
+
if paths.is_a?(Hash)
|
62
64
|
"#{asset}: #{Summary.pluralize paths[:files], 'file'} (#{Summary.pluralize paths[:total], 'entry'} total)"
|
63
65
|
else
|
64
66
|
"#{asset}: #{Summary.pluralize paths, 'entry'}"
|
@@ -87,7 +89,7 @@ class DumpRake
|
|
87
89
|
|
88
90
|
def find_entry(matcher)
|
89
91
|
stream.each do |entry|
|
90
|
-
if
|
92
|
+
if entry.full_name.match(matcher)
|
91
93
|
# we can not return entry - after exiting stream.each the entry will be invalid and will read from tar start
|
92
94
|
return yield(entry)
|
93
95
|
end
|
@@ -120,27 +122,26 @@ class DumpRake
|
|
120
122
|
Rake::Task['db:drop'].invoke
|
121
123
|
Rake::Task['db:create'].invoke
|
122
124
|
when !DumpRake::Env.no?(:migrate_down)
|
123
|
-
|
124
|
-
|
125
|
-
|
126
|
-
|
127
|
-
|
128
|
-
|
129
|
-
|
130
|
-
|
131
|
-
|
132
|
-
|
133
|
-
|
134
|
-
|
135
|
-
|
136
|
-
|
137
|
-
|
138
|
-
|
139
|
-
|
140
|
-
|
141
|
-
end
|
142
|
-
task.reenable
|
125
|
+
return unless avaliable_tables.include?('schema_migrations')
|
126
|
+
find_entry('schema_migrations.dump') do |entry|
|
127
|
+
migrated = table_rows('schema_migrations').map{ |row| row['version'] }
|
128
|
+
|
129
|
+
dump_migrations = []
|
130
|
+
Marshal.load(entry) # skip header
|
131
|
+
dump_migrations << Marshal.load(entry).first until entry.eof?
|
132
|
+
|
133
|
+
migrate_down = (migrated - dump_migrations)
|
134
|
+
|
135
|
+
unless migrate_down.empty?
|
136
|
+
migrate_down.reverse.with_progress('Migrating down') do |version|
|
137
|
+
DumpRake::Env.with_env('VERSION' => version) do
|
138
|
+
Rake::Task['db:migrate:down'].tap do |task|
|
139
|
+
begin
|
140
|
+
task.invoke
|
141
|
+
rescue ActiveRecord::IrreversibleMigration
|
142
|
+
$stderr.puts "Irreversible migration: #{version}"
|
143
143
|
end
|
144
|
+
task.reenable
|
144
145
|
end
|
145
146
|
end
|
146
147
|
end
|
@@ -154,13 +155,12 @@ class DumpRake
|
|
154
155
|
end
|
155
156
|
|
156
157
|
def read_schema
|
157
|
-
|
158
|
-
|
159
|
-
|
160
|
-
|
161
|
-
end
|
162
|
-
Rake::Task['db:schema:dump'].invoke
|
158
|
+
return unless restore_schema?
|
159
|
+
read_entry_to_file('schema.rb') do |f|
|
160
|
+
DumpRake::Env.with_env('SCHEMA' => f.path) do
|
161
|
+
Rake::Task['db:schema:load'].invoke
|
163
162
|
end
|
163
|
+
Rake::Task['db:schema:dump'].invoke
|
164
164
|
end
|
165
165
|
end
|
166
166
|
|
@@ -209,42 +209,41 @@ class DumpRake
|
|
209
209
|
|
210
210
|
def read_assets
|
211
211
|
return if DumpRake::Env[:restore_assets] && DumpRake::Env[:restore_assets].empty?
|
212
|
-
|
213
|
-
|
214
|
-
|
215
|
-
|
216
|
-
|
217
|
-
|
218
|
-
|
219
|
-
|
212
|
+
return if config[:assets].blank?
|
213
|
+
|
214
|
+
assets = config[:assets]
|
215
|
+
if assets.is_a?(Hash)
|
216
|
+
assets_count = assets.values.sum{ |value| value.is_a?(Hash) ? value[:total] : value }
|
217
|
+
assets_paths = assets.keys
|
218
|
+
else
|
219
|
+
assets_count, assets_paths = nil, assets
|
220
|
+
end
|
220
221
|
|
221
|
-
|
222
|
-
|
223
|
-
|
224
|
-
|
225
|
-
|
226
|
-
|
227
|
-
|
228
|
-
|
229
|
-
|
230
|
-
|
231
|
-
|
232
|
-
|
233
|
-
end
|
234
|
-
end
|
222
|
+
if DumpRake::Env[:restore_assets]
|
223
|
+
assets_paths.each do |asset|
|
224
|
+
DumpRake::Assets.glob_asset_children(asset, '**/*').reverse.each do |child|
|
225
|
+
next unless read_asset?(child, DumpRake::RailsRoot)
|
226
|
+
case
|
227
|
+
when File.file?(child)
|
228
|
+
File.unlink(child)
|
229
|
+
when File.directory?(child)
|
230
|
+
begin
|
231
|
+
Dir.unlink(child)
|
232
|
+
rescue Errno::ENOTEMPTY
|
233
|
+
nil
|
235
234
|
end
|
236
235
|
end
|
237
236
|
end
|
238
|
-
else
|
239
|
-
DumpRake::Env.with_env(:assets => assets_paths.join(':')) do
|
240
|
-
Rake::Task['assets:delete'].invoke
|
241
|
-
end
|
242
237
|
end
|
238
|
+
else
|
239
|
+
DumpRake::Env.with_env(:assets => assets_paths.join(':')) do
|
240
|
+
Rake::Task['assets:delete'].invoke
|
241
|
+
end
|
242
|
+
end
|
243
243
|
|
244
|
-
|
245
|
-
|
246
|
-
|
247
|
-
end
|
244
|
+
read_assets_entries(assets_paths, assets_count) do |stream, root, entry, prefix|
|
245
|
+
if !DumpRake::Env[:restore_assets] || read_asset?(entry.full_name, prefix)
|
246
|
+
stream.extract_entry(root, entry)
|
248
247
|
end
|
249
248
|
end
|
250
249
|
end
|
@@ -252,14 +251,14 @@ class DumpRake
|
|
252
251
|
def read_asset?(path, prefix)
|
253
252
|
DumpRake::Env.filter(:restore_assets, DumpRake::Assets::SPLITTER).custom_pass? do |value|
|
254
253
|
File.fnmatch(File.join(prefix, value), path) ||
|
255
|
-
|
254
|
+
File.fnmatch(File.join(prefix, value, '**'), path)
|
256
255
|
end
|
257
256
|
end
|
258
257
|
|
259
|
-
def read_assets_entries(
|
258
|
+
def read_assets_entries(_assets_paths, assets_count)
|
260
259
|
Progress.start('Assets', assets_count || 1) do
|
261
260
|
found_assets = false
|
262
|
-
# old style
|
261
|
+
# old style - in separate tar
|
263
262
|
find_entry('assets.tar') do |assets_tar|
|
264
263
|
def assets_tar.rewind
|
265
264
|
# rewind will fail - it must go to center of gzip
|
@@ -275,7 +274,7 @@ class DumpRake
|
|
275
274
|
end
|
276
275
|
|
277
276
|
unless found_assets
|
278
|
-
# new style
|
277
|
+
# new style - in same tar
|
279
278
|
assets_root_link do |tmpdir, prefix|
|
280
279
|
stream.each do |entry|
|
281
280
|
if entry.full_name.starts_with?("#{prefix}/")
|