db_sucker 3.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.gitignore +16 -0
- data/CHANGELOG.md +45 -0
- data/Gemfile +4 -0
- data/LICENSE.txt +22 -0
- data/README.md +193 -0
- data/Rakefile +1 -0
- data/VERSION +1 -0
- data/bin/db_sucker +12 -0
- data/bin/db_sucker.sh +14 -0
- data/db_sucker.gemspec +29 -0
- data/doc/config_example.rb +53 -0
- data/doc/container_example.yml +150 -0
- data/lib/db_sucker/adapters/mysql2.rb +103 -0
- data/lib/db_sucker/application/colorize.rb +28 -0
- data/lib/db_sucker/application/container/accessors.rb +60 -0
- data/lib/db_sucker/application/container/ssh.rb +225 -0
- data/lib/db_sucker/application/container/validations.rb +53 -0
- data/lib/db_sucker/application/container/variation/accessors.rb +45 -0
- data/lib/db_sucker/application/container/variation/helpers.rb +21 -0
- data/lib/db_sucker/application/container/variation/worker_api.rb +65 -0
- data/lib/db_sucker/application/container/variation.rb +60 -0
- data/lib/db_sucker/application/container.rb +70 -0
- data/lib/db_sucker/application/container_collection.rb +47 -0
- data/lib/db_sucker/application/core.rb +222 -0
- data/lib/db_sucker/application/dispatch.rb +364 -0
- data/lib/db_sucker/application/evented_resultset.rb +149 -0
- data/lib/db_sucker/application/fake_channel.rb +22 -0
- data/lib/db_sucker/application/output_helper.rb +197 -0
- data/lib/db_sucker/application/sklaven_treiber/log_spool.rb +57 -0
- data/lib/db_sucker/application/sklaven_treiber/worker/accessors.rb +105 -0
- data/lib/db_sucker/application/sklaven_treiber/worker/core.rb +168 -0
- data/lib/db_sucker/application/sklaven_treiber/worker/helpers.rb +144 -0
- data/lib/db_sucker/application/sklaven_treiber/worker/io/base.rb +240 -0
- data/lib/db_sucker/application/sklaven_treiber/worker/io/file_copy.rb +81 -0
- data/lib/db_sucker/application/sklaven_treiber/worker/io/file_gunzip.rb +58 -0
- data/lib/db_sucker/application/sklaven_treiber/worker/io/file_import_sql.rb +80 -0
- data/lib/db_sucker/application/sklaven_treiber/worker/io/file_shasum.rb +49 -0
- data/lib/db_sucker/application/sklaven_treiber/worker/io/pv_wrapper.rb +73 -0
- data/lib/db_sucker/application/sklaven_treiber/worker/io/sftp_download.rb +57 -0
- data/lib/db_sucker/application/sklaven_treiber/worker/io/throughput.rb +219 -0
- data/lib/db_sucker/application/sklaven_treiber/worker/routines.rb +313 -0
- data/lib/db_sucker/application/sklaven_treiber/worker.rb +48 -0
- data/lib/db_sucker/application/sklaven_treiber.rb +281 -0
- data/lib/db_sucker/application/slot_pool.rb +137 -0
- data/lib/db_sucker/application/tie.rb +25 -0
- data/lib/db_sucker/application/window/core.rb +185 -0
- data/lib/db_sucker/application/window/dialog.rb +142 -0
- data/lib/db_sucker/application/window/keypad/core.rb +85 -0
- data/lib/db_sucker/application/window/keypad.rb +174 -0
- data/lib/db_sucker/application/window/prompt.rb +124 -0
- data/lib/db_sucker/application/window.rb +329 -0
- data/lib/db_sucker/application.rb +168 -0
- data/lib/db_sucker/patches/beta-warning.rb +374 -0
- data/lib/db_sucker/patches/developer.rb +29 -0
- data/lib/db_sucker/patches/net-sftp.rb +20 -0
- data/lib/db_sucker/patches/thread-count.rb +30 -0
- data/lib/db_sucker/version.rb +4 -0
- data/lib/db_sucker.rb +81 -0
- metadata +217 -0
@@ -0,0 +1,144 @@
|
|
1
|
+
module DbSucker
|
2
|
+
class Application
|
3
|
+
class SklavenTreiber
|
4
|
+
class Worker
|
5
|
+
module Helpers
|
6
|
+
def runtime
|
7
|
+
if @started
|
8
|
+
human_seconds ((@ended || Time.current) - @started).to_i
|
9
|
+
end
|
10
|
+
end
|
11
|
+
|
12
|
+
def sftp_download *args, &block
|
13
|
+
IO::SftpDownload.new(self, *args).tap do |op|
|
14
|
+
block.try(:call, op)
|
15
|
+
end
|
16
|
+
end
|
17
|
+
|
18
|
+
def file_copy *args, &block
|
19
|
+
IO::FileCopy.new(self, *args).tap do |op|
|
20
|
+
block.try(:call, op)
|
21
|
+
end
|
22
|
+
end
|
23
|
+
|
24
|
+
def file_gunzip *args, &block
|
25
|
+
IO::FileGunzip.new(self, *args).tap do |op|
|
26
|
+
block.try(:call, op)
|
27
|
+
end
|
28
|
+
end
|
29
|
+
|
30
|
+
def file_shasum *args, &block
|
31
|
+
IO::Shasum.new(self, *args).tap do |op|
|
32
|
+
block.try(:call, op)
|
33
|
+
end
|
34
|
+
end
|
35
|
+
|
36
|
+
def pv_wrap *args, &block
|
37
|
+
IO::PvWrapper.new(self, *args).tap do |op|
|
38
|
+
block.try(:call, op)
|
39
|
+
end
|
40
|
+
end
|
41
|
+
|
42
|
+
def file_import_sql *args, &block
|
43
|
+
IO::FileImportSql.new(self, *args).tap do |op|
|
44
|
+
block.try(:call, op)
|
45
|
+
end
|
46
|
+
end
|
47
|
+
|
48
|
+
def aquire_slots *which, &block
|
49
|
+
target_thread = Thread.current
|
50
|
+
aquired = []
|
51
|
+
which.each_with_index do |wh, i|
|
52
|
+
if pool = sklaventreiber.slot_pools[wh]
|
53
|
+
waitlock = Queue.new
|
54
|
+
channel = app.channelfy_thread app.spawn_thread(:sklaventreiber_worker_slot_progress) {|thr|
|
55
|
+
thr[:current_task] = target_thread[:current_task] if target_thread[:current_task]
|
56
|
+
thr[:slot_pool_qindex] = Proc.new { pool.qindex(target_thread) }
|
57
|
+
waitlock.pop
|
58
|
+
pool.aquire(target_thread)
|
59
|
+
}
|
60
|
+
waitlock << true
|
61
|
+
target_thread.wait
|
62
|
+
|
63
|
+
label = "aquiring slot #{i+1}/#{which.length} `#{pool.name}' :slot_pool_qindex(– #%s in queue )(:seconds)..."
|
64
|
+
second_progress(channel, label, :blue).tap{ pool.wait_aquired(target_thread) }.join
|
65
|
+
if pool.aquired?(target_thread)
|
66
|
+
aquired << wh
|
67
|
+
else
|
68
|
+
break
|
69
|
+
end
|
70
|
+
else
|
71
|
+
raise SlotPoolNotInitializedError, "slot pool `#{wh}' was never initialized, can't aquire slot"
|
72
|
+
end
|
73
|
+
end
|
74
|
+
block.call if block && (which - aquired).empty?
|
75
|
+
ensure
|
76
|
+
release_slots(*which)
|
77
|
+
end
|
78
|
+
|
79
|
+
def release_slots *which
|
80
|
+
which.each_with_index do |wh, i|
|
81
|
+
if pool = sklaventreiber.slot_pools[wh]
|
82
|
+
pool.release(Thread.current)
|
83
|
+
else
|
84
|
+
raise SlotPoolNotInitializedError, "slot pool `#{wh}' was never initialized, can't release slot (was most likely never aquired)"
|
85
|
+
end
|
86
|
+
end
|
87
|
+
end
|
88
|
+
|
89
|
+
def wait_defer_ready label = nil
|
90
|
+
channel = app.fake_channel {|c| c[:slot_pool_qindex].call.zero? || @should_cancel }
|
91
|
+
channel[:slot_pool_qindex] = Proc.new { sklaventreiber.sync { sklaventreiber.workers.count{|w| !w.done? && !w.deferred? } } }
|
92
|
+
|
93
|
+
label = "deferred import: #{human_bytes(File.size(@local_file_raw))} raw SQL :slot_pool_qindex(– waiting for %s workers )(:seconds)"
|
94
|
+
second_progress(channel, label, :blue).join
|
95
|
+
end
|
96
|
+
|
97
|
+
def second_progress channel, status, color = :yellow
|
98
|
+
target_thread = Thread.current
|
99
|
+
app.spawn_thread(:sklaventreiber_worker_second_progress) do |thr|
|
100
|
+
thr[:iteration] = 0
|
101
|
+
thr[:started_at] = Time.current
|
102
|
+
thr[:current_task] = target_thread[:current_task] if target_thread[:current_task]
|
103
|
+
channel[:handler] = thr if channel.respond_to?(:[]=)
|
104
|
+
loop do
|
105
|
+
if @should_cancel && !thr[:canceled]
|
106
|
+
if channel.is_a?(Net::SSH::Connection::Channel)
|
107
|
+
if channel[:pty]
|
108
|
+
channel.send_data("\C-c") rescue false
|
109
|
+
elsif channel[:pid]
|
110
|
+
@ctn.kill_remote_process(channel[:pid])
|
111
|
+
end
|
112
|
+
end
|
113
|
+
channel.try(:close) rescue false
|
114
|
+
Process.kill(:SIGINT, channel[:ipc_thread].pid) if channel[:ipc_thread]
|
115
|
+
thr[:canceled] = true
|
116
|
+
end
|
117
|
+
stat = status.gsub(":seconds", human_seconds(Time.current - thr[:started_at]))
|
118
|
+
if channel[:slot_pool_qindex].respond_to?(:call)
|
119
|
+
qi = channel[:slot_pool_qindex].call
|
120
|
+
re = /:slot_pool_qindex\(([^\)]+)\)/
|
121
|
+
if stat[re]
|
122
|
+
stat[re] = qi ? stat[re].match(/\(([^\)]+)\)/)[1].gsub("%s", qi.to_s) : ""
|
123
|
+
end
|
124
|
+
end
|
125
|
+
if channel[:error_message]
|
126
|
+
@status = ["[ERROR] #{channel[:error_message]}", :red]
|
127
|
+
elsif !channel.active?
|
128
|
+
@status = ["[CLOSED] #{stat}", :red]
|
129
|
+
elsif channel.closing?
|
130
|
+
@status = ["[CLOSING] #{stat}", :red]
|
131
|
+
else
|
132
|
+
@status = [stat, color]
|
133
|
+
end
|
134
|
+
break unless channel.active?
|
135
|
+
thr.wait(0.1)
|
136
|
+
thr[:iteration] += 1
|
137
|
+
end
|
138
|
+
end
|
139
|
+
end
|
140
|
+
end
|
141
|
+
end
|
142
|
+
end
|
143
|
+
end
|
144
|
+
end
|
@@ -0,0 +1,240 @@
|
|
1
|
+
module DbSucker
|
2
|
+
class Application
|
3
|
+
class SklavenTreiber
|
4
|
+
class Worker
|
5
|
+
module IO
|
6
|
+
class Base
|
7
|
+
UnknownFormatterError = Class.new(::ArgumentError)
|
8
|
+
DataIntegrityError = Class.new(::RuntimeError)
|
9
|
+
STATUS_FORMATTERS = [:none, :minimal, :full]
|
10
|
+
attr_reader :status_format, :state, :operror, :offset, :closing, :local, :remote, :ctn, :verify_handle
|
11
|
+
attr_accessor :read_size, :label, :entity, :filesize, :throughput, :mode
|
12
|
+
OutputHelper.hook(self)
|
13
|
+
|
14
|
+
def initialize worker, ctn, fd
|
15
|
+
# references
|
16
|
+
@worker = worker
|
17
|
+
@ctn = ctn
|
18
|
+
@thread = Thread.current
|
19
|
+
|
20
|
+
# remote & local
|
21
|
+
@remote = fd.is_a?(Hash) ? fd.keys[0] : fd
|
22
|
+
@local = fd.values[0] if fd.is_a?(Hash)
|
23
|
+
|
24
|
+
# defaults
|
25
|
+
@label ||= "working"
|
26
|
+
@entity ||= "task"
|
27
|
+
@status_format = :off
|
28
|
+
@mode = :fs
|
29
|
+
@throughput = worker.sklaventreiber.throughput.register(self)
|
30
|
+
@read_size = 128 * 1024 # 128kb
|
31
|
+
@filesize = 0
|
32
|
+
|
33
|
+
# callbacks
|
34
|
+
@abort_if = Proc.new { false }
|
35
|
+
@on_error = Proc.new {}
|
36
|
+
@on_complete = Proc.new {}
|
37
|
+
@on_success = Proc.new {}
|
38
|
+
init
|
39
|
+
reset_state
|
40
|
+
end
|
41
|
+
|
42
|
+
def init
|
43
|
+
end
|
44
|
+
|
45
|
+
def reset_state
|
46
|
+
@operror = nil
|
47
|
+
@closing = false
|
48
|
+
@state = :idle
|
49
|
+
@offset = 0
|
50
|
+
@throughput.reset_stats
|
51
|
+
end
|
52
|
+
|
53
|
+
def abort_if &block
|
54
|
+
@abort_if = block
|
55
|
+
end
|
56
|
+
|
57
|
+
def on_error &block
|
58
|
+
@on_error = block
|
59
|
+
end
|
60
|
+
|
61
|
+
def on_complete &block
|
62
|
+
@on_complete = block
|
63
|
+
end
|
64
|
+
|
65
|
+
def on_success &block
|
66
|
+
@on_success = block
|
67
|
+
end
|
68
|
+
|
69
|
+
def status_format= which
|
70
|
+
which = which.to_sym
|
71
|
+
raise UnknownFormatterError, "unknown status format `#{which}', available options: #{STATUS_FORMATTERS * ", "}" unless STATUS_FORMATTERS.include?(which)
|
72
|
+
@status_format = which
|
73
|
+
end
|
74
|
+
|
75
|
+
def prepare_local_destination
|
76
|
+
FileUtils.mkdir_p(File.dirname(@local))
|
77
|
+
end
|
78
|
+
|
79
|
+
def handle_pause
|
80
|
+
@thread[:paused] ? throughput.pause! : throughput.unpause!
|
81
|
+
end
|
82
|
+
|
83
|
+
def execute opts = {}, &block
|
84
|
+
opts = opts.reverse_merge(tries: 1, sleep_error: 0)
|
85
|
+
try = 1
|
86
|
+
begin
|
87
|
+
reset_state
|
88
|
+
throughput.measure(&block)
|
89
|
+
if !@closing && !@worker.should_cancel
|
90
|
+
@state = :done
|
91
|
+
@on_success.call(self)
|
92
|
+
end
|
93
|
+
rescue StandardError => ex
|
94
|
+
@operror = "##{try} #{ex.class}: #{ex.message}"
|
95
|
+
@on_error.call(self, ex, @operror)
|
96
|
+
try += 1
|
97
|
+
if try > opts[:tries]
|
98
|
+
raise ex
|
99
|
+
else
|
100
|
+
Thread.current.wait(opts[:sleep_error])
|
101
|
+
retry
|
102
|
+
end
|
103
|
+
ensure
|
104
|
+
@on_complete.call(self)
|
105
|
+
end
|
106
|
+
ensure
|
107
|
+
@throughput.unregister
|
108
|
+
end
|
109
|
+
|
110
|
+
def to_s
|
111
|
+
handle_pause
|
112
|
+
return @operror if @operror
|
113
|
+
tp = @throughput
|
114
|
+
|
115
|
+
[].tap do |r|
|
116
|
+
r << "[CLOSING]" if @closing
|
117
|
+
if @status_format == :none
|
118
|
+
r << "#{@label}"
|
119
|
+
break
|
120
|
+
end
|
121
|
+
case @state
|
122
|
+
when :idle, :init
|
123
|
+
r << "#{@label}:"
|
124
|
+
r << " initiating..."
|
125
|
+
when :finishing
|
126
|
+
r << "#{@label}:"
|
127
|
+
r << " finishing..."
|
128
|
+
when :verifying
|
129
|
+
r << "#{@label}:"
|
130
|
+
r << " verifying..."
|
131
|
+
r << " #{@verify_handle.throughput.f_done_percentage}" if @verify_handle.respond_to?(:throughput)
|
132
|
+
when :done
|
133
|
+
if @mode == :nofs
|
134
|
+
r << "#{@entity || @label} complete(?): #{tp.f_offset}"
|
135
|
+
else
|
136
|
+
r << "#{@entity || @label} #{@offset == @filesize ? "complete" : "INCOMPLETE"}: #{tp.f_done_percentage} – #{tp.f_byte_progress}"
|
137
|
+
end
|
138
|
+
when :downloading, :copying, :decompressing, :working
|
139
|
+
r << "#{@label}:"
|
140
|
+
if @mode == :nofs
|
141
|
+
r << "[#{tp.f_offset} – #{tp.f_bps}/s]"
|
142
|
+
else
|
143
|
+
r << tp.f_done_percentage.rjust(7, " ")
|
144
|
+
if @status_format == :minimal
|
145
|
+
r << "[#{tp.f_eta}]"
|
146
|
+
elsif @status_format == :full
|
147
|
+
r << "[#{tp.f_eta} – #{tp.f_bps.rjust(9, " ")}/s]"
|
148
|
+
end
|
149
|
+
|
150
|
+
if @status_format == :full
|
151
|
+
f_has, f_tot = tp.f_offset, tp.f_filesize
|
152
|
+
r << "[#{f_has.rjust(f_tot.length, "0")}/#{f_tot}]"
|
153
|
+
end
|
154
|
+
end
|
155
|
+
end
|
156
|
+
end * " "
|
157
|
+
end
|
158
|
+
|
159
|
+
def to_curses target
|
160
|
+
handle_pause
|
161
|
+
_this = self
|
162
|
+
tp = @throughput
|
163
|
+
target.instance_eval do
|
164
|
+
if _this.operror
|
165
|
+
red "#{_this.operror}"
|
166
|
+
return
|
167
|
+
end
|
168
|
+
|
169
|
+
if _this.closing
|
170
|
+
red "[CLOSING] "
|
171
|
+
end
|
172
|
+
|
173
|
+
if _this.status_format == :none
|
174
|
+
blue "#{_this.label}"
|
175
|
+
break
|
176
|
+
end
|
177
|
+
|
178
|
+
case _this.state
|
179
|
+
when :idle, :init
|
180
|
+
yellow "#{_this.label}: "
|
181
|
+
gray " initiating..."
|
182
|
+
when :finishing
|
183
|
+
yellow "#{_this.label}: "
|
184
|
+
gray " finishing..."
|
185
|
+
when :verifying
|
186
|
+
yellow "#{_this.label}: "
|
187
|
+
gray " verifying..."
|
188
|
+
gray " #{_this.verify_handle.throughput.f_done_percentage}" if _this.verify_handle.respond_to?(:throughput)
|
189
|
+
when :done
|
190
|
+
if _this.mode == :nofs
|
191
|
+
green "#{_this.entity || _this.label} complete(?): "
|
192
|
+
cyan "#{human_bytes _this.offset}"
|
193
|
+
else
|
194
|
+
if _this.offset == _this.filesize
|
195
|
+
green "#{_this.entity || _this.label} complete: #{tp.f_done_percentage}"
|
196
|
+
yellow " – "
|
197
|
+
cyan "#{human_bytes _this.offset}"
|
198
|
+
else
|
199
|
+
red "#{_this.entity || _this.label} INCOMPLETE: #{tp.f_done_percentage}"
|
200
|
+
yellow " – "
|
201
|
+
cyan "#{tp.f_byte_progress}"
|
202
|
+
end
|
203
|
+
end
|
204
|
+
when :downloading, :copying, :decompressing, :working
|
205
|
+
yellow "#{_this.label}: "
|
206
|
+
if _this.mode == :nofs
|
207
|
+
blue "#{tp.f_offset}"
|
208
|
+
if _this.status_format == :minimal
|
209
|
+
gray " [#{tp.f_runtime}]"
|
210
|
+
elsif _this.status_format == :full
|
211
|
+
gray " [#{tp.f_bps}/s – #{tp.f_runtime}]"
|
212
|
+
end
|
213
|
+
#progress_bar(-1)
|
214
|
+
else
|
215
|
+
diffp = tp.done_percentage
|
216
|
+
color = diffp > 90 ? :green : diffp > 75 ? :blue : diffp > 50 ? :cyan : diffp > 25 ? :yellow : :red
|
217
|
+
send(color, tp.f_done_percentage.rjust(7, " ") << " ")
|
218
|
+
|
219
|
+
if _this.status_format == :minimal
|
220
|
+
yellow "[#{tp.f_eta}]"
|
221
|
+
elsif _this.status_format == :full
|
222
|
+
yellow "[#{tp.f_eta} – #{tp.f_bps.rjust(9, " ")}/s]"
|
223
|
+
end
|
224
|
+
|
225
|
+
if _this.status_format == :full
|
226
|
+
f_has, f_tot = tp.f_offset, tp.f_filesize
|
227
|
+
gray " [#{f_has.rjust(f_tot.length, "0")}/#{f_tot}]"
|
228
|
+
end
|
229
|
+
|
230
|
+
progress_bar(diffp, prog_done_color: color, prog_current_color: color)
|
231
|
+
end
|
232
|
+
end
|
233
|
+
end
|
234
|
+
end
|
235
|
+
end
|
236
|
+
end
|
237
|
+
end
|
238
|
+
end
|
239
|
+
end
|
240
|
+
end
|
@@ -0,0 +1,81 @@
|
|
1
|
+
module DbSucker
|
2
|
+
class Application
|
3
|
+
class SklavenTreiber
|
4
|
+
class Worker
|
5
|
+
module IO
|
6
|
+
class FileCopy < Base
|
7
|
+
attr_accessor :use_tmp, :integrity
|
8
|
+
|
9
|
+
def init
|
10
|
+
@label = "copying"
|
11
|
+
@entity = "copy"
|
12
|
+
@use_tmp = true
|
13
|
+
@integrity = true
|
14
|
+
@throughput.categories << :io << :io_file_copy
|
15
|
+
end
|
16
|
+
|
17
|
+
def copy! opts = {}
|
18
|
+
opts = opts.reverse_merge(tries: 1, read_size: @read_size)
|
19
|
+
prepare_local_destination
|
20
|
+
|
21
|
+
execute(opts.slice(:tries).merge(sleep_error: 3)) do
|
22
|
+
@tmploc = @use_tmp ? "#{@local}.tmp" : @local
|
23
|
+
@in_file = File.new(@remote, "rb")
|
24
|
+
@out_file = File.new(@tmploc, "wb")
|
25
|
+
@filesize = @in_file.size
|
26
|
+
|
27
|
+
buf = ""
|
28
|
+
@state = :copying
|
29
|
+
begin
|
30
|
+
while @in_file.sysread(opts[:read_size], buf)
|
31
|
+
if !@closing && @abort_if.call(self)
|
32
|
+
@closing = true
|
33
|
+
break
|
34
|
+
end
|
35
|
+
|
36
|
+
@offset += buf.bytesize
|
37
|
+
@out_file.syswrite(buf)
|
38
|
+
GC.start if @offset % GC_FORCE_RATE == 0
|
39
|
+
end
|
40
|
+
rescue EOFError
|
41
|
+
ensure
|
42
|
+
@state = :finishing
|
43
|
+
@in_file.close
|
44
|
+
@out_file.close
|
45
|
+
end
|
46
|
+
|
47
|
+
FileUtils.mv(@tmploc, @local) if @use_tmp
|
48
|
+
|
49
|
+
@state = :verifying
|
50
|
+
src_hash = verify_file(@remote, 0)
|
51
|
+
dst_hash = verify_file(@local, 1)
|
52
|
+
if src_hash != dst_hash
|
53
|
+
raise DataIntegrityError, "Integrity check failed! [SRC](#{src_hash}) != [DST](#{dst_hash})"
|
54
|
+
end
|
55
|
+
end
|
56
|
+
end
|
57
|
+
|
58
|
+
def verify_file file, index = 0
|
59
|
+
result = false
|
60
|
+
@worker.file_shasum(@ctn, file) do |fc|
|
61
|
+
fc.sha = @ctn.integrity_sha
|
62
|
+
fc.status_format = :none
|
63
|
+
fc.throughput.sopts[:perc_modifier] = 0.5
|
64
|
+
fc.throughput.sopts[:perc_base] = index * 50
|
65
|
+
@verify_handle = fc
|
66
|
+
|
67
|
+
fc.abort_if { @should_cancel }
|
68
|
+
fc.on_success do
|
69
|
+
result = fc.result
|
70
|
+
end
|
71
|
+
fc.verify!
|
72
|
+
end
|
73
|
+
@verify_handle = false
|
74
|
+
return result
|
75
|
+
end
|
76
|
+
end
|
77
|
+
end
|
78
|
+
end
|
79
|
+
end
|
80
|
+
end
|
81
|
+
end
|
@@ -0,0 +1,58 @@
|
|
1
|
+
module DbSucker
|
2
|
+
class Application
|
3
|
+
class SklavenTreiber
|
4
|
+
class Worker
|
5
|
+
module IO
|
6
|
+
class FileGunzip < Base
|
7
|
+
attr_accessor :use_tmp
|
8
|
+
attr_accessor :preserve_original
|
9
|
+
|
10
|
+
def init
|
11
|
+
@label = "decompressing"
|
12
|
+
@entity = "decompress"
|
13
|
+
@use_tmp = true
|
14
|
+
@preserve_original = false
|
15
|
+
@local ||= "#{File.dirname(@remote)}/#{File.basename(@remote, ".gz")}"
|
16
|
+
@throughput.categories << :io << :io_gunzip
|
17
|
+
end
|
18
|
+
|
19
|
+
def gunzip! opts = {}
|
20
|
+
opts = opts.reverse_merge(tries: 1, read_size: @read_size)
|
21
|
+
prepare_local_destination
|
22
|
+
|
23
|
+
execute(opts.slice(:tries).merge(sleep_error: 3)) do
|
24
|
+
@tmploc = @use_tmp ? "#{@local}.tmp" : @local
|
25
|
+
@in_file = File.new(@remote, "rb")
|
26
|
+
@out_file = File.new(@tmploc, "wb")
|
27
|
+
@filesize = @in_file.size if @filesize.zero?
|
28
|
+
|
29
|
+
@state = :decompressing
|
30
|
+
gz = Zlib::GzipReader.new(@in_file)
|
31
|
+
begin
|
32
|
+
while buf = gz.read(opts[:read_size])
|
33
|
+
if !@closing && @abort_if.call(self)
|
34
|
+
@closing = true
|
35
|
+
break
|
36
|
+
end
|
37
|
+
|
38
|
+
@offset += [opts[:read_size], @filesize - @offset].min
|
39
|
+
@out_file.syswrite(buf)
|
40
|
+
GC.start if @offset % GC_FORCE_RATE == 0
|
41
|
+
end
|
42
|
+
ensure
|
43
|
+
@state = :finishing
|
44
|
+
gz.close
|
45
|
+
@in_file.close
|
46
|
+
@out_file.close
|
47
|
+
end
|
48
|
+
|
49
|
+
FileUtils.mv(@tmploc, @local) if @use_tmp
|
50
|
+
File.unlink(@remote) unless @preserve_original
|
51
|
+
end
|
52
|
+
end
|
53
|
+
end
|
54
|
+
end
|
55
|
+
end
|
56
|
+
end
|
57
|
+
end
|
58
|
+
end
|
@@ -0,0 +1,80 @@
|
|
1
|
+
module DbSucker
|
2
|
+
class Application
|
3
|
+
class SklavenTreiber
|
4
|
+
class Worker
|
5
|
+
module IO
|
6
|
+
class FileImportSql < Base
|
7
|
+
MissingInstructionError = Class.new(::ArgumentError)
|
8
|
+
InvalidInstructionError = Class.new(::ArgumentError)
|
9
|
+
ImportError = Class.new(::RuntimeError)
|
10
|
+
attr_accessor :instruction
|
11
|
+
|
12
|
+
def init
|
13
|
+
@label = "importing table"
|
14
|
+
@entity = "table"
|
15
|
+
@instruction = false
|
16
|
+
@throughput.categories << :io << :io_import
|
17
|
+
end
|
18
|
+
|
19
|
+
def import! opts = {}
|
20
|
+
raise MissingInstructionError, "no instruction given for import" unless @instruction.is_a?(Hash)
|
21
|
+
raise InvalidInstructionError, "import instruction must at least contain :bin and :file keys" if [:bin, :file].any?{|w| !@instruction.key?(w) }
|
22
|
+
opts = opts.reverse_merge(tries: 1, read_size: @read_size)
|
23
|
+
|
24
|
+
execute(opts.slice(:tries).merge(sleep_error: 3)) do
|
25
|
+
@in_file = File.new(@instruction[:file], "rb")
|
26
|
+
@filesize = @in_file.size if @filesize.zero?
|
27
|
+
|
28
|
+
buf = ""
|
29
|
+
@state = :working
|
30
|
+
|
31
|
+
begin
|
32
|
+
debug "Opening process `#{@instruction[:bin]}'"
|
33
|
+
Open3.popen2e(@instruction[:bin], pgroup: true) do |_stdin, _stdouterr, _thread|
|
34
|
+
begin
|
35
|
+
# file prepend
|
36
|
+
_stdin.puts @instruction[:file_prepend] if @instruction[:file_prepend]
|
37
|
+
|
38
|
+
# file contents
|
39
|
+
begin
|
40
|
+
while @in_file.sysread(opts[:read_size], buf)
|
41
|
+
if !@closing && @abort_if.call(self)
|
42
|
+
@closing = true
|
43
|
+
break
|
44
|
+
end
|
45
|
+
|
46
|
+
@offset += buf.bytesize
|
47
|
+
_stdin.write(buf)
|
48
|
+
GC.start if @offset % GC_FORCE_RATE == 0
|
49
|
+
end
|
50
|
+
rescue EOFError
|
51
|
+
ensure
|
52
|
+
@in_file.close
|
53
|
+
end
|
54
|
+
|
55
|
+
# file append
|
56
|
+
_stdin.puts @instruction[:file_append] if @instruction[:file_append]
|
57
|
+
rescue Errno::EPIPE => ex
|
58
|
+
raise ImportError, "#{ex.message} (#{_stdouterr.read.chomp})"
|
59
|
+
end
|
60
|
+
|
61
|
+
# close & exit status
|
62
|
+
_stdin.close_write
|
63
|
+
exit_status = _thread.value
|
64
|
+
if exit_status == 0
|
65
|
+
debug "Process exited (#{exit_status}) `#{@instruction[:bin]}'"
|
66
|
+
else
|
67
|
+
warning "Process exited (#{exit_status}) `#{@instruction[:bin]}'"
|
68
|
+
end
|
69
|
+
end
|
70
|
+
ensure
|
71
|
+
@state = :finishing
|
72
|
+
end
|
73
|
+
end
|
74
|
+
end
|
75
|
+
end
|
76
|
+
end
|
77
|
+
end
|
78
|
+
end
|
79
|
+
end
|
80
|
+
end
|
@@ -0,0 +1,49 @@
|
|
1
|
+
module DbSucker
|
2
|
+
class Application
|
3
|
+
class SklavenTreiber
|
4
|
+
class Worker
|
5
|
+
module IO
|
6
|
+
class Shasum < Base
|
7
|
+
attr_accessor :sha, :result
|
8
|
+
|
9
|
+
def init
|
10
|
+
@label = "verifying"
|
11
|
+
@entity = "verification"
|
12
|
+
@sha ||= 1
|
13
|
+
@throughput.categories.clear # IO read
|
14
|
+
end
|
15
|
+
|
16
|
+
def verify! opts = {}
|
17
|
+
opts = opts.reverse_merge(tries: 1, read_size: @read_size)
|
18
|
+
|
19
|
+
execute(opts.slice(:tries).merge(sleep_error: 3)) do
|
20
|
+
@in_file = File.new(@remote, "rb")
|
21
|
+
@filesize = @in_file.size
|
22
|
+
|
23
|
+
@state = :working
|
24
|
+
sha = "Digest::SHA#{@sha}".constantize.new
|
25
|
+
buf = ""
|
26
|
+
begin
|
27
|
+
while @in_file.read(opts[:read_size], buf)
|
28
|
+
if !@closing && @abort_if.call(self)
|
29
|
+
@closing = true
|
30
|
+
break
|
31
|
+
end
|
32
|
+
|
33
|
+
@offset += buf.bytesize
|
34
|
+
sha << buf
|
35
|
+
GC.start if @offset % GC_FORCE_RATE == 0
|
36
|
+
end
|
37
|
+
ensure
|
38
|
+
@state = :finishing
|
39
|
+
@in_file.close
|
40
|
+
@result = sha.hexdigest
|
41
|
+
end
|
42
|
+
end
|
43
|
+
end
|
44
|
+
end
|
45
|
+
end
|
46
|
+
end
|
47
|
+
end
|
48
|
+
end
|
49
|
+
end
|