db_sucker 3.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (60) hide show
  1. checksums.yaml +7 -0
  2. data/.gitignore +16 -0
  3. data/CHANGELOG.md +45 -0
  4. data/Gemfile +4 -0
  5. data/LICENSE.txt +22 -0
  6. data/README.md +193 -0
  7. data/Rakefile +1 -0
  8. data/VERSION +1 -0
  9. data/bin/db_sucker +12 -0
  10. data/bin/db_sucker.sh +14 -0
  11. data/db_sucker.gemspec +29 -0
  12. data/doc/config_example.rb +53 -0
  13. data/doc/container_example.yml +150 -0
  14. data/lib/db_sucker/adapters/mysql2.rb +103 -0
  15. data/lib/db_sucker/application/colorize.rb +28 -0
  16. data/lib/db_sucker/application/container/accessors.rb +60 -0
  17. data/lib/db_sucker/application/container/ssh.rb +225 -0
  18. data/lib/db_sucker/application/container/validations.rb +53 -0
  19. data/lib/db_sucker/application/container/variation/accessors.rb +45 -0
  20. data/lib/db_sucker/application/container/variation/helpers.rb +21 -0
  21. data/lib/db_sucker/application/container/variation/worker_api.rb +65 -0
  22. data/lib/db_sucker/application/container/variation.rb +60 -0
  23. data/lib/db_sucker/application/container.rb +70 -0
  24. data/lib/db_sucker/application/container_collection.rb +47 -0
  25. data/lib/db_sucker/application/core.rb +222 -0
  26. data/lib/db_sucker/application/dispatch.rb +364 -0
  27. data/lib/db_sucker/application/evented_resultset.rb +149 -0
  28. data/lib/db_sucker/application/fake_channel.rb +22 -0
  29. data/lib/db_sucker/application/output_helper.rb +197 -0
  30. data/lib/db_sucker/application/sklaven_treiber/log_spool.rb +57 -0
  31. data/lib/db_sucker/application/sklaven_treiber/worker/accessors.rb +105 -0
  32. data/lib/db_sucker/application/sklaven_treiber/worker/core.rb +168 -0
  33. data/lib/db_sucker/application/sklaven_treiber/worker/helpers.rb +144 -0
  34. data/lib/db_sucker/application/sklaven_treiber/worker/io/base.rb +240 -0
  35. data/lib/db_sucker/application/sklaven_treiber/worker/io/file_copy.rb +81 -0
  36. data/lib/db_sucker/application/sklaven_treiber/worker/io/file_gunzip.rb +58 -0
  37. data/lib/db_sucker/application/sklaven_treiber/worker/io/file_import_sql.rb +80 -0
  38. data/lib/db_sucker/application/sklaven_treiber/worker/io/file_shasum.rb +49 -0
  39. data/lib/db_sucker/application/sklaven_treiber/worker/io/pv_wrapper.rb +73 -0
  40. data/lib/db_sucker/application/sklaven_treiber/worker/io/sftp_download.rb +57 -0
  41. data/lib/db_sucker/application/sklaven_treiber/worker/io/throughput.rb +219 -0
  42. data/lib/db_sucker/application/sklaven_treiber/worker/routines.rb +313 -0
  43. data/lib/db_sucker/application/sklaven_treiber/worker.rb +48 -0
  44. data/lib/db_sucker/application/sklaven_treiber.rb +281 -0
  45. data/lib/db_sucker/application/slot_pool.rb +137 -0
  46. data/lib/db_sucker/application/tie.rb +25 -0
  47. data/lib/db_sucker/application/window/core.rb +185 -0
  48. data/lib/db_sucker/application/window/dialog.rb +142 -0
  49. data/lib/db_sucker/application/window/keypad/core.rb +85 -0
  50. data/lib/db_sucker/application/window/keypad.rb +174 -0
  51. data/lib/db_sucker/application/window/prompt.rb +124 -0
  52. data/lib/db_sucker/application/window.rb +329 -0
  53. data/lib/db_sucker/application.rb +168 -0
  54. data/lib/db_sucker/patches/beta-warning.rb +374 -0
  55. data/lib/db_sucker/patches/developer.rb +29 -0
  56. data/lib/db_sucker/patches/net-sftp.rb +20 -0
  57. data/lib/db_sucker/patches/thread-count.rb +30 -0
  58. data/lib/db_sucker/version.rb +4 -0
  59. data/lib/db_sucker.rb +81 -0
  60. metadata +217 -0
@@ -0,0 +1,144 @@
1
+ module DbSucker
2
+ class Application
3
+ class SklavenTreiber
4
+ class Worker
5
+ module Helpers
6
+ def runtime
7
+ if @started
8
+ human_seconds ((@ended || Time.current) - @started).to_i
9
+ end
10
+ end
11
+
12
+ def sftp_download *args, &block
13
+ IO::SftpDownload.new(self, *args).tap do |op|
14
+ block.try(:call, op)
15
+ end
16
+ end
17
+
18
+ def file_copy *args, &block
19
+ IO::FileCopy.new(self, *args).tap do |op|
20
+ block.try(:call, op)
21
+ end
22
+ end
23
+
24
+ def file_gunzip *args, &block
25
+ IO::FileGunzip.new(self, *args).tap do |op|
26
+ block.try(:call, op)
27
+ end
28
+ end
29
+
30
+ def file_shasum *args, &block
31
+ IO::Shasum.new(self, *args).tap do |op|
32
+ block.try(:call, op)
33
+ end
34
+ end
35
+
36
+ def pv_wrap *args, &block
37
+ IO::PvWrapper.new(self, *args).tap do |op|
38
+ block.try(:call, op)
39
+ end
40
+ end
41
+
42
+ def file_import_sql *args, &block
43
+ IO::FileImportSql.new(self, *args).tap do |op|
44
+ block.try(:call, op)
45
+ end
46
+ end
47
+
48
+ def aquire_slots *which, &block
49
+ target_thread = Thread.current
50
+ aquired = []
51
+ which.each_with_index do |wh, i|
52
+ if pool = sklaventreiber.slot_pools[wh]
53
+ waitlock = Queue.new
54
+ channel = app.channelfy_thread app.spawn_thread(:sklaventreiber_worker_slot_progress) {|thr|
55
+ thr[:current_task] = target_thread[:current_task] if target_thread[:current_task]
56
+ thr[:slot_pool_qindex] = Proc.new { pool.qindex(target_thread) }
57
+ waitlock.pop
58
+ pool.aquire(target_thread)
59
+ }
60
+ waitlock << true
61
+ target_thread.wait
62
+
63
+ label = "aquiring slot #{i+1}/#{which.length} `#{pool.name}' :slot_pool_qindex(– #%s in queue )(:seconds)..."
64
+ second_progress(channel, label, :blue).tap{ pool.wait_aquired(target_thread) }.join
65
+ if pool.aquired?(target_thread)
66
+ aquired << wh
67
+ else
68
+ break
69
+ end
70
+ else
71
+ raise SlotPoolNotInitializedError, "slot pool `#{wh}' was never initialized, can't aquire slot"
72
+ end
73
+ end
74
+ block.call if block && (which - aquired).empty?
75
+ ensure
76
+ release_slots(*which)
77
+ end
78
+
79
+ def release_slots *which
80
+ which.each_with_index do |wh, i|
81
+ if pool = sklaventreiber.slot_pools[wh]
82
+ pool.release(Thread.current)
83
+ else
84
+ raise SlotPoolNotInitializedError, "slot pool `#{wh}' was never initialized, can't release slot (was most likely never aquired)"
85
+ end
86
+ end
87
+ end
88
+
89
+ def wait_defer_ready label = nil
90
+ channel = app.fake_channel {|c| c[:slot_pool_qindex].call.zero? || @should_cancel }
91
+ channel[:slot_pool_qindex] = Proc.new { sklaventreiber.sync { sklaventreiber.workers.count{|w| !w.done? && !w.deferred? } } }
92
+
93
+ label = "deferred import: #{human_bytes(File.size(@local_file_raw))} raw SQL :slot_pool_qindex(– waiting for %s workers )(:seconds)"
94
+ second_progress(channel, label, :blue).join
95
+ end
96
+
97
+ def second_progress channel, status, color = :yellow
98
+ target_thread = Thread.current
99
+ app.spawn_thread(:sklaventreiber_worker_second_progress) do |thr|
100
+ thr[:iteration] = 0
101
+ thr[:started_at] = Time.current
102
+ thr[:current_task] = target_thread[:current_task] if target_thread[:current_task]
103
+ channel[:handler] = thr if channel.respond_to?(:[]=)
104
+ loop do
105
+ if @should_cancel && !thr[:canceled]
106
+ if channel.is_a?(Net::SSH::Connection::Channel)
107
+ if channel[:pty]
108
+ channel.send_data("\C-c") rescue false
109
+ elsif channel[:pid]
110
+ @ctn.kill_remote_process(channel[:pid])
111
+ end
112
+ end
113
+ channel.try(:close) rescue false
114
+ Process.kill(:SIGINT, channel[:ipc_thread].pid) if channel[:ipc_thread]
115
+ thr[:canceled] = true
116
+ end
117
+ stat = status.gsub(":seconds", human_seconds(Time.current - thr[:started_at]))
118
+ if channel[:slot_pool_qindex].respond_to?(:call)
119
+ qi = channel[:slot_pool_qindex].call
120
+ re = /:slot_pool_qindex\(([^\)]+)\)/
121
+ if stat[re]
122
+ stat[re] = qi ? stat[re].match(/\(([^\)]+)\)/)[1].gsub("%s", qi.to_s) : ""
123
+ end
124
+ end
125
+ if channel[:error_message]
126
+ @status = ["[ERROR] #{channel[:error_message]}", :red]
127
+ elsif !channel.active?
128
+ @status = ["[CLOSED] #{stat}", :red]
129
+ elsif channel.closing?
130
+ @status = ["[CLOSING] #{stat}", :red]
131
+ else
132
+ @status = [stat, color]
133
+ end
134
+ break unless channel.active?
135
+ thr.wait(0.1)
136
+ thr[:iteration] += 1
137
+ end
138
+ end
139
+ end
140
+ end
141
+ end
142
+ end
143
+ end
144
+ end
@@ -0,0 +1,240 @@
1
+ module DbSucker
2
+ class Application
3
+ class SklavenTreiber
4
+ class Worker
5
+ module IO
6
+ class Base
7
+ UnknownFormatterError = Class.new(::ArgumentError)
8
+ DataIntegrityError = Class.new(::RuntimeError)
9
+ STATUS_FORMATTERS = [:none, :minimal, :full]
10
+ attr_reader :status_format, :state, :operror, :offset, :closing, :local, :remote, :ctn, :verify_handle
11
+ attr_accessor :read_size, :label, :entity, :filesize, :throughput, :mode
12
+ OutputHelper.hook(self)
13
+
14
+ def initialize worker, ctn, fd
15
+ # references
16
+ @worker = worker
17
+ @ctn = ctn
18
+ @thread = Thread.current
19
+
20
+ # remote & local
21
+ @remote = fd.is_a?(Hash) ? fd.keys[0] : fd
22
+ @local = fd.values[0] if fd.is_a?(Hash)
23
+
24
+ # defaults
25
+ @label ||= "working"
26
+ @entity ||= "task"
27
+ @status_format = :off
28
+ @mode = :fs
29
+ @throughput = worker.sklaventreiber.throughput.register(self)
30
+ @read_size = 128 * 1024 # 128kb
31
+ @filesize = 0
32
+
33
+ # callbacks
34
+ @abort_if = Proc.new { false }
35
+ @on_error = Proc.new {}
36
+ @on_complete = Proc.new {}
37
+ @on_success = Proc.new {}
38
+ init
39
+ reset_state
40
+ end
41
+
42
+ def init
43
+ end
44
+
45
+ def reset_state
46
+ @operror = nil
47
+ @closing = false
48
+ @state = :idle
49
+ @offset = 0
50
+ @throughput.reset_stats
51
+ end
52
+
53
+ def abort_if &block
54
+ @abort_if = block
55
+ end
56
+
57
+ def on_error &block
58
+ @on_error = block
59
+ end
60
+
61
+ def on_complete &block
62
+ @on_complete = block
63
+ end
64
+
65
+ def on_success &block
66
+ @on_success = block
67
+ end
68
+
69
+ def status_format= which
70
+ which = which.to_sym
71
+ raise UnknownFormatterError, "unknown status format `#{which}', available options: #{STATUS_FORMATTERS * ", "}" unless STATUS_FORMATTERS.include?(which)
72
+ @status_format = which
73
+ end
74
+
75
+ def prepare_local_destination
76
+ FileUtils.mkdir_p(File.dirname(@local))
77
+ end
78
+
79
+ def handle_pause
80
+ @thread[:paused] ? throughput.pause! : throughput.unpause!
81
+ end
82
+
83
+ def execute opts = {}, &block
84
+ opts = opts.reverse_merge(tries: 1, sleep_error: 0)
85
+ try = 1
86
+ begin
87
+ reset_state
88
+ throughput.measure(&block)
89
+ if !@closing && !@worker.should_cancel
90
+ @state = :done
91
+ @on_success.call(self)
92
+ end
93
+ rescue StandardError => ex
94
+ @operror = "##{try} #{ex.class}: #{ex.message}"
95
+ @on_error.call(self, ex, @operror)
96
+ try += 1
97
+ if try > opts[:tries]
98
+ raise ex
99
+ else
100
+ Thread.current.wait(opts[:sleep_error])
101
+ retry
102
+ end
103
+ ensure
104
+ @on_complete.call(self)
105
+ end
106
+ ensure
107
+ @throughput.unregister
108
+ end
109
+
110
+ def to_s
111
+ handle_pause
112
+ return @operror if @operror
113
+ tp = @throughput
114
+
115
+ [].tap do |r|
116
+ r << "[CLOSING]" if @closing
117
+ if @status_format == :none
118
+ r << "#{@label}"
119
+ break
120
+ end
121
+ case @state
122
+ when :idle, :init
123
+ r << "#{@label}:"
124
+ r << " initiating..."
125
+ when :finishing
126
+ r << "#{@label}:"
127
+ r << " finishing..."
128
+ when :verifying
129
+ r << "#{@label}:"
130
+ r << " verifying..."
131
+ r << " #{@verify_handle.throughput.f_done_percentage}" if @verify_handle.respond_to?(:throughput)
132
+ when :done
133
+ if @mode == :nofs
134
+ r << "#{@entity || @label} complete(?): #{tp.f_offset}"
135
+ else
136
+ r << "#{@entity || @label} #{@offset == @filesize ? "complete" : "INCOMPLETE"}: #{tp.f_done_percentage} – #{tp.f_byte_progress}"
137
+ end
138
+ when :downloading, :copying, :decompressing, :working
139
+ r << "#{@label}:"
140
+ if @mode == :nofs
141
+ r << "[#{tp.f_offset} – #{tp.f_bps}/s]"
142
+ else
143
+ r << tp.f_done_percentage.rjust(7, " ")
144
+ if @status_format == :minimal
145
+ r << "[#{tp.f_eta}]"
146
+ elsif @status_format == :full
147
+ r << "[#{tp.f_eta} – #{tp.f_bps.rjust(9, " ")}/s]"
148
+ end
149
+
150
+ if @status_format == :full
151
+ f_has, f_tot = tp.f_offset, tp.f_filesize
152
+ r << "[#{f_has.rjust(f_tot.length, "0")}/#{f_tot}]"
153
+ end
154
+ end
155
+ end
156
+ end * " "
157
+ end
158
+
159
+ def to_curses target
160
+ handle_pause
161
+ _this = self
162
+ tp = @throughput
163
+ target.instance_eval do
164
+ if _this.operror
165
+ red "#{_this.operror}"
166
+ return
167
+ end
168
+
169
+ if _this.closing
170
+ red "[CLOSING] "
171
+ end
172
+
173
+ if _this.status_format == :none
174
+ blue "#{_this.label}"
175
+ break
176
+ end
177
+
178
+ case _this.state
179
+ when :idle, :init
180
+ yellow "#{_this.label}: "
181
+ gray " initiating..."
182
+ when :finishing
183
+ yellow "#{_this.label}: "
184
+ gray " finishing..."
185
+ when :verifying
186
+ yellow "#{_this.label}: "
187
+ gray " verifying..."
188
+ gray " #{_this.verify_handle.throughput.f_done_percentage}" if _this.verify_handle.respond_to?(:throughput)
189
+ when :done
190
+ if _this.mode == :nofs
191
+ green "#{_this.entity || _this.label} complete(?): "
192
+ cyan "#{human_bytes _this.offset}"
193
+ else
194
+ if _this.offset == _this.filesize
195
+ green "#{_this.entity || _this.label} complete: #{tp.f_done_percentage}"
196
+ yellow " – "
197
+ cyan "#{human_bytes _this.offset}"
198
+ else
199
+ red "#{_this.entity || _this.label} INCOMPLETE: #{tp.f_done_percentage}"
200
+ yellow " – "
201
+ cyan "#{tp.f_byte_progress}"
202
+ end
203
+ end
204
+ when :downloading, :copying, :decompressing, :working
205
+ yellow "#{_this.label}: "
206
+ if _this.mode == :nofs
207
+ blue "#{tp.f_offset}"
208
+ if _this.status_format == :minimal
209
+ gray " [#{tp.f_runtime}]"
210
+ elsif _this.status_format == :full
211
+ gray " [#{tp.f_bps}/s – #{tp.f_runtime}]"
212
+ end
213
+ #progress_bar(-1)
214
+ else
215
+ diffp = tp.done_percentage
216
+ color = diffp > 90 ? :green : diffp > 75 ? :blue : diffp > 50 ? :cyan : diffp > 25 ? :yellow : :red
217
+ send(color, tp.f_done_percentage.rjust(7, " ") << " ")
218
+
219
+ if _this.status_format == :minimal
220
+ yellow "[#{tp.f_eta}]"
221
+ elsif _this.status_format == :full
222
+ yellow "[#{tp.f_eta} – #{tp.f_bps.rjust(9, " ")}/s]"
223
+ end
224
+
225
+ if _this.status_format == :full
226
+ f_has, f_tot = tp.f_offset, tp.f_filesize
227
+ gray " [#{f_has.rjust(f_tot.length, "0")}/#{f_tot}]"
228
+ end
229
+
230
+ progress_bar(diffp, prog_done_color: color, prog_current_color: color)
231
+ end
232
+ end
233
+ end
234
+ end
235
+ end
236
+ end
237
+ end
238
+ end
239
+ end
240
+ end
@@ -0,0 +1,81 @@
1
+ module DbSucker
2
+ class Application
3
+ class SklavenTreiber
4
+ class Worker
5
+ module IO
6
+ class FileCopy < Base
7
+ attr_accessor :use_tmp, :integrity
8
+
9
+ def init
10
+ @label = "copying"
11
+ @entity = "copy"
12
+ @use_tmp = true
13
+ @integrity = true
14
+ @throughput.categories << :io << :io_file_copy
15
+ end
16
+
17
+ def copy! opts = {}
18
+ opts = opts.reverse_merge(tries: 1, read_size: @read_size)
19
+ prepare_local_destination
20
+
21
+ execute(opts.slice(:tries).merge(sleep_error: 3)) do
22
+ @tmploc = @use_tmp ? "#{@local}.tmp" : @local
23
+ @in_file = File.new(@remote, "rb")
24
+ @out_file = File.new(@tmploc, "wb")
25
+ @filesize = @in_file.size
26
+
27
+ buf = ""
28
+ @state = :copying
29
+ begin
30
+ while @in_file.sysread(opts[:read_size], buf)
31
+ if !@closing && @abort_if.call(self)
32
+ @closing = true
33
+ break
34
+ end
35
+
36
+ @offset += buf.bytesize
37
+ @out_file.syswrite(buf)
38
+ GC.start if @offset % GC_FORCE_RATE == 0
39
+ end
40
+ rescue EOFError
41
+ ensure
42
+ @state = :finishing
43
+ @in_file.close
44
+ @out_file.close
45
+ end
46
+
47
+ FileUtils.mv(@tmploc, @local) if @use_tmp
48
+
49
+ @state = :verifying
50
+ src_hash = verify_file(@remote, 0)
51
+ dst_hash = verify_file(@local, 1)
52
+ if src_hash != dst_hash
53
+ raise DataIntegrityError, "Integrity check failed! [SRC](#{src_hash}) != [DST](#{dst_hash})"
54
+ end
55
+ end
56
+ end
57
+
58
+ def verify_file file, index = 0
59
+ result = false
60
+ @worker.file_shasum(@ctn, file) do |fc|
61
+ fc.sha = @ctn.integrity_sha
62
+ fc.status_format = :none
63
+ fc.throughput.sopts[:perc_modifier] = 0.5
64
+ fc.throughput.sopts[:perc_base] = index * 50
65
+ @verify_handle = fc
66
+
67
+ fc.abort_if { @should_cancel }
68
+ fc.on_success do
69
+ result = fc.result
70
+ end
71
+ fc.verify!
72
+ end
73
+ @verify_handle = false
74
+ return result
75
+ end
76
+ end
77
+ end
78
+ end
79
+ end
80
+ end
81
+ end
@@ -0,0 +1,58 @@
1
+ module DbSucker
2
+ class Application
3
+ class SklavenTreiber
4
+ class Worker
5
+ module IO
6
+ class FileGunzip < Base
7
+ attr_accessor :use_tmp
8
+ attr_accessor :preserve_original
9
+
10
+ def init
11
+ @label = "decompressing"
12
+ @entity = "decompress"
13
+ @use_tmp = true
14
+ @preserve_original = false
15
+ @local ||= "#{File.dirname(@remote)}/#{File.basename(@remote, ".gz")}"
16
+ @throughput.categories << :io << :io_gunzip
17
+ end
18
+
19
+ def gunzip! opts = {}
20
+ opts = opts.reverse_merge(tries: 1, read_size: @read_size)
21
+ prepare_local_destination
22
+
23
+ execute(opts.slice(:tries).merge(sleep_error: 3)) do
24
+ @tmploc = @use_tmp ? "#{@local}.tmp" : @local
25
+ @in_file = File.new(@remote, "rb")
26
+ @out_file = File.new(@tmploc, "wb")
27
+ @filesize = @in_file.size if @filesize.zero?
28
+
29
+ @state = :decompressing
30
+ gz = Zlib::GzipReader.new(@in_file)
31
+ begin
32
+ while buf = gz.read(opts[:read_size])
33
+ if !@closing && @abort_if.call(self)
34
+ @closing = true
35
+ break
36
+ end
37
+
38
+ @offset += [opts[:read_size], @filesize - @offset].min
39
+ @out_file.syswrite(buf)
40
+ GC.start if @offset % GC_FORCE_RATE == 0
41
+ end
42
+ ensure
43
+ @state = :finishing
44
+ gz.close
45
+ @in_file.close
46
+ @out_file.close
47
+ end
48
+
49
+ FileUtils.mv(@tmploc, @local) if @use_tmp
50
+ File.unlink(@remote) unless @preserve_original
51
+ end
52
+ end
53
+ end
54
+ end
55
+ end
56
+ end
57
+ end
58
+ end
@@ -0,0 +1,80 @@
1
+ module DbSucker
2
+ class Application
3
+ class SklavenTreiber
4
+ class Worker
5
+ module IO
6
+ class FileImportSql < Base
7
+ MissingInstructionError = Class.new(::ArgumentError)
8
+ InvalidInstructionError = Class.new(::ArgumentError)
9
+ ImportError = Class.new(::RuntimeError)
10
+ attr_accessor :instruction
11
+
12
+ def init
13
+ @label = "importing table"
14
+ @entity = "table"
15
+ @instruction = false
16
+ @throughput.categories << :io << :io_import
17
+ end
18
+
19
+ def import! opts = {}
20
+ raise MissingInstructionError, "no instruction given for import" unless @instruction.is_a?(Hash)
21
+ raise InvalidInstructionError, "import instruction must at least contain :bin and :file keys" if [:bin, :file].any?{|w| !@instruction.key?(w) }
22
+ opts = opts.reverse_merge(tries: 1, read_size: @read_size)
23
+
24
+ execute(opts.slice(:tries).merge(sleep_error: 3)) do
25
+ @in_file = File.new(@instruction[:file], "rb")
26
+ @filesize = @in_file.size if @filesize.zero?
27
+
28
+ buf = ""
29
+ @state = :working
30
+
31
+ begin
32
+ debug "Opening process `#{@instruction[:bin]}'"
33
+ Open3.popen2e(@instruction[:bin], pgroup: true) do |_stdin, _stdouterr, _thread|
34
+ begin
35
+ # file prepend
36
+ _stdin.puts @instruction[:file_prepend] if @instruction[:file_prepend]
37
+
38
+ # file contents
39
+ begin
40
+ while @in_file.sysread(opts[:read_size], buf)
41
+ if !@closing && @abort_if.call(self)
42
+ @closing = true
43
+ break
44
+ end
45
+
46
+ @offset += buf.bytesize
47
+ _stdin.write(buf)
48
+ GC.start if @offset % GC_FORCE_RATE == 0
49
+ end
50
+ rescue EOFError
51
+ ensure
52
+ @in_file.close
53
+ end
54
+
55
+ # file append
56
+ _stdin.puts @instruction[:file_append] if @instruction[:file_append]
57
+ rescue Errno::EPIPE => ex
58
+ raise ImportError, "#{ex.message} (#{_stdouterr.read.chomp})"
59
+ end
60
+
61
+ # close & exit status
62
+ _stdin.close_write
63
+ exit_status = _thread.value
64
+ if exit_status == 0
65
+ debug "Process exited (#{exit_status}) `#{@instruction[:bin]}'"
66
+ else
67
+ warning "Process exited (#{exit_status}) `#{@instruction[:bin]}'"
68
+ end
69
+ end
70
+ ensure
71
+ @state = :finishing
72
+ end
73
+ end
74
+ end
75
+ end
76
+ end
77
+ end
78
+ end
79
+ end
80
+ end
@@ -0,0 +1,49 @@
1
+ module DbSucker
2
+ class Application
3
+ class SklavenTreiber
4
+ class Worker
5
+ module IO
6
+ class Shasum < Base
7
+ attr_accessor :sha, :result
8
+
9
+ def init
10
+ @label = "verifying"
11
+ @entity = "verification"
12
+ @sha ||= 1
13
+ @throughput.categories.clear # IO read
14
+ end
15
+
16
+ def verify! opts = {}
17
+ opts = opts.reverse_merge(tries: 1, read_size: @read_size)
18
+
19
+ execute(opts.slice(:tries).merge(sleep_error: 3)) do
20
+ @in_file = File.new(@remote, "rb")
21
+ @filesize = @in_file.size
22
+
23
+ @state = :working
24
+ sha = "Digest::SHA#{@sha}".constantize.new
25
+ buf = ""
26
+ begin
27
+ while @in_file.read(opts[:read_size], buf)
28
+ if !@closing && @abort_if.call(self)
29
+ @closing = true
30
+ break
31
+ end
32
+
33
+ @offset += buf.bytesize
34
+ sha << buf
35
+ GC.start if @offset % GC_FORCE_RATE == 0
36
+ end
37
+ ensure
38
+ @state = :finishing
39
+ @in_file.close
40
+ @result = sha.hexdigest
41
+ end
42
+ end
43
+ end
44
+ end
45
+ end
46
+ end
47
+ end
48
+ end
49
+ end