db_sucker 3.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (60) hide show
  1. checksums.yaml +7 -0
  2. data/.gitignore +16 -0
  3. data/CHANGELOG.md +45 -0
  4. data/Gemfile +4 -0
  5. data/LICENSE.txt +22 -0
  6. data/README.md +193 -0
  7. data/Rakefile +1 -0
  8. data/VERSION +1 -0
  9. data/bin/db_sucker +12 -0
  10. data/bin/db_sucker.sh +14 -0
  11. data/db_sucker.gemspec +29 -0
  12. data/doc/config_example.rb +53 -0
  13. data/doc/container_example.yml +150 -0
  14. data/lib/db_sucker/adapters/mysql2.rb +103 -0
  15. data/lib/db_sucker/application/colorize.rb +28 -0
  16. data/lib/db_sucker/application/container/accessors.rb +60 -0
  17. data/lib/db_sucker/application/container/ssh.rb +225 -0
  18. data/lib/db_sucker/application/container/validations.rb +53 -0
  19. data/lib/db_sucker/application/container/variation/accessors.rb +45 -0
  20. data/lib/db_sucker/application/container/variation/helpers.rb +21 -0
  21. data/lib/db_sucker/application/container/variation/worker_api.rb +65 -0
  22. data/lib/db_sucker/application/container/variation.rb +60 -0
  23. data/lib/db_sucker/application/container.rb +70 -0
  24. data/lib/db_sucker/application/container_collection.rb +47 -0
  25. data/lib/db_sucker/application/core.rb +222 -0
  26. data/lib/db_sucker/application/dispatch.rb +364 -0
  27. data/lib/db_sucker/application/evented_resultset.rb +149 -0
  28. data/lib/db_sucker/application/fake_channel.rb +22 -0
  29. data/lib/db_sucker/application/output_helper.rb +197 -0
  30. data/lib/db_sucker/application/sklaven_treiber/log_spool.rb +57 -0
  31. data/lib/db_sucker/application/sklaven_treiber/worker/accessors.rb +105 -0
  32. data/lib/db_sucker/application/sklaven_treiber/worker/core.rb +168 -0
  33. data/lib/db_sucker/application/sklaven_treiber/worker/helpers.rb +144 -0
  34. data/lib/db_sucker/application/sklaven_treiber/worker/io/base.rb +240 -0
  35. data/lib/db_sucker/application/sklaven_treiber/worker/io/file_copy.rb +81 -0
  36. data/lib/db_sucker/application/sklaven_treiber/worker/io/file_gunzip.rb +58 -0
  37. data/lib/db_sucker/application/sklaven_treiber/worker/io/file_import_sql.rb +80 -0
  38. data/lib/db_sucker/application/sklaven_treiber/worker/io/file_shasum.rb +49 -0
  39. data/lib/db_sucker/application/sklaven_treiber/worker/io/pv_wrapper.rb +73 -0
  40. data/lib/db_sucker/application/sklaven_treiber/worker/io/sftp_download.rb +57 -0
  41. data/lib/db_sucker/application/sklaven_treiber/worker/io/throughput.rb +219 -0
  42. data/lib/db_sucker/application/sklaven_treiber/worker/routines.rb +313 -0
  43. data/lib/db_sucker/application/sklaven_treiber/worker.rb +48 -0
  44. data/lib/db_sucker/application/sklaven_treiber.rb +281 -0
  45. data/lib/db_sucker/application/slot_pool.rb +137 -0
  46. data/lib/db_sucker/application/tie.rb +25 -0
  47. data/lib/db_sucker/application/window/core.rb +185 -0
  48. data/lib/db_sucker/application/window/dialog.rb +142 -0
  49. data/lib/db_sucker/application/window/keypad/core.rb +85 -0
  50. data/lib/db_sucker/application/window/keypad.rb +174 -0
  51. data/lib/db_sucker/application/window/prompt.rb +124 -0
  52. data/lib/db_sucker/application/window.rb +329 -0
  53. data/lib/db_sucker/application.rb +168 -0
  54. data/lib/db_sucker/patches/beta-warning.rb +374 -0
  55. data/lib/db_sucker/patches/developer.rb +29 -0
  56. data/lib/db_sucker/patches/net-sftp.rb +20 -0
  57. data/lib/db_sucker/patches/thread-count.rb +30 -0
  58. data/lib/db_sucker/version.rb +4 -0
  59. data/lib/db_sucker.rb +81 -0
  60. metadata +217 -0
@@ -0,0 +1,142 @@
1
+ module DbSucker
2
+ class Application
3
+ class Window
4
+ class Dialog
5
+ attr_accessor :border_color
6
+
7
+ BOX = {
8
+ tl: "┌",
9
+ t: "─",
10
+ tr: "┐",
11
+ l: "│",
12
+ r: "│",
13
+ bl: "└",
14
+ b: "─",
15
+ br: "┘",
16
+ hrl: "├",
17
+ hr: "─",
18
+ hrr: "┤",
19
+ }
20
+
21
+ def initialize window, &block
22
+ @window = window
23
+ @width = false
24
+ @border_color = :yellow
25
+ @border_padding = 1
26
+ @separator_padding = 1
27
+ @lines = []
28
+ block.call(self)
29
+ end
30
+
31
+ def line str = nil, color = :yellow, &block
32
+ if block
33
+ @lines << [].tap{|a| block.call(a) }
34
+ else
35
+ @lines << [[str, color]]
36
+ end
37
+ end
38
+
39
+ def br
40
+ @lines << []
41
+ end
42
+
43
+ def hr
44
+ @lines << :hr
45
+ end
46
+
47
+ def button *a
48
+ @lines.concat(build_button(*a))
49
+ end
50
+
51
+ def build_button str, color = :yellow
52
+ tl = BOX[:tl]
53
+ t = BOX[:t]
54
+ tr = BOX[:tr]
55
+ l = BOX[:l]
56
+ r = BOX[:r]
57
+ bl = BOX[:bl]
58
+ b = BOX[:b]
59
+ br = BOX[:br]
60
+ width = str.length + 2
61
+ [].tap do |a|
62
+ a << [["#{tl}" << "".ljust(width, t) << "#{tr}", color]]
63
+ a << [["#{l}" << " #{str} ".ljust(width, t) << "#{r}", color]]
64
+ a << [["#{bl}" << "".ljust(width, b) << "#{br}", color]]
65
+ end
66
+ end
67
+
68
+ def button_group buttons = [], spaces = 2, &block
69
+ spaces = buttons if block
70
+ btns = block ? [].tap{|a| block.call(a) } : buttons.dup
71
+ spaces = "".ljust(spaces, " ")
72
+ first = btns.shift
73
+
74
+ btns.each_with_index do |lines, bi|
75
+ lines.each_with_index do |l, i|
76
+ first[i] << [spaces, :yellow]
77
+ first[i].concat(l)
78
+ end
79
+ end
80
+
81
+ @lines.concat first
82
+ end
83
+
84
+ # -----------------------------------------------------------
85
+
86
+ def _nl
87
+ @window.next_line
88
+ end
89
+
90
+ def _hr
91
+ l = BOX[:hrl]
92
+ r = BOX[:hrr]
93
+ m = "".ljust(@width - l.length - r.length, BOX[:hr])
94
+ @separator_padding.times { _line }
95
+ @window.send(@border_color, "#{l}#{m}#{r}")
96
+ _nl
97
+ end
98
+
99
+ def _line parts = []
100
+ return _hr if parts == :hr
101
+ l = BOX[:l]
102
+ r = BOX[:r]
103
+ p = "".ljust(@border_padding, " ")
104
+
105
+ @window.send(@border_color, "#{l}#{p}")
106
+ parts.each do |str, color|
107
+ @window.send(color, str)
108
+ end
109
+ @window.send(@border_color, "".ljust(@width - l.length - r.length - (p.length * 2) - parts.map{|s, _| s.length }.sum, " "))
110
+ @window.send(@border_color, "#{p}#{r}")
111
+ _nl
112
+ end
113
+
114
+ def _tborder
115
+ tl = BOX[:tl]
116
+ tr = BOX[:tr]
117
+ m = "".ljust(@width - tl.length - tr.length, BOX[:t])
118
+ @window.send(@border_color, "#{tl}#{m}#{tr}")
119
+ _nl
120
+ end
121
+
122
+ def _bborder
123
+ bl = BOX[:bl]
124
+ br = BOX[:br]
125
+ m = "".ljust(@width - bl.length - br.length, BOX[:b])
126
+ @window.send(@border_color, "#{bl}#{m}#{br}")
127
+ end
128
+
129
+ def render!
130
+ @width = @lines.map{|c| c.is_a?(Array) ? c.map{|s, _| s.length }.sum : 0 }.compact.max + (@border_padding * 2) + BOX[:l].length + BOX[:r].length
131
+
132
+ _tborder
133
+ @border_padding.times { _line }
134
+ @lines.each {|parts| _line(parts) }
135
+ @border_padding.times { _line }
136
+ _bborder
137
+ end
138
+ end
139
+ end
140
+ end
141
+ end
142
+
@@ -0,0 +1,85 @@
1
+ module DbSucker
2
+ class Application
3
+ class Window
4
+ class Keypad
5
+ module Core
6
+ def sync &block
7
+ @monitor.synchronize(&block)
8
+ end
9
+
10
+ def sklaventreiber
11
+ @window.sklaventreiber
12
+ end
13
+
14
+ def app
15
+ @window.app
16
+ end
17
+
18
+ def enabled?
19
+ app.opts[:window_keypad]
20
+ end
21
+
22
+ def start_loop
23
+ return false unless enabled?
24
+ @keyloop = app.spawn_thread(:window_keypad_loop) do |thr|
25
+ loop {
26
+ begin
27
+ handle_input(@window.send(:getch))
28
+ rescue StandardError => ex
29
+ app.notify_exception("DbSucker::Window::Keypad encountered an input handle error on tick ##{@window.tick}", ex)
30
+ end
31
+ }
32
+ end
33
+ end
34
+
35
+ def stop_loop
36
+ return unless @keyloop
37
+ sync { @keyloop.kill }
38
+ @keyloop.join
39
+ end
40
+
41
+ def prompt! *a, &b
42
+ return false unless enabled?
43
+ @prompt.set!(*a, &b)
44
+ end
45
+
46
+ def _detect_worker arg, &block
47
+ sklaventreiber.sync do
48
+ if arg == "--all"
49
+ sklaventreiber.workers.each{|w| block.call(w) }
50
+ else
51
+ wrk = sklaventreiber.workers.detect do |w|
52
+ if arg.start_with?("^")
53
+ w.table.match(/#{arg}/i)
54
+ elsif arg.start_with?("/")
55
+ w.table.match(/#{arg[1..-1]}/i)
56
+ else
57
+ w.table == arg
58
+ end
59
+ end
60
+ if wrk
61
+ block.call(wrk)
62
+ else
63
+ prompt!("Could not find any worker by the pattern `#{arg}'", color: :red)
64
+ end
65
+ end
66
+ end
67
+ end
68
+
69
+ def _eval evil
70
+ return if evil.blank?
71
+ app.dump_file "eval-result", true do |f|
72
+ begin
73
+ f.puts("#{evil}\n\n")
74
+ f.puts(app.sync{ app.instance_eval(evil) })
75
+ rescue StandardError => ex
76
+ f.puts("#{ex.class}: #{ex.message}")
77
+ ex.backtrace.each {|l| f.puts(" #{l}") }
78
+ end
79
+ end
80
+ end
81
+ end
82
+ end
83
+ end
84
+ end
85
+ end
@@ -0,0 +1,174 @@
1
+ module DbSucker
2
+ class Application
3
+ class Window
4
+ class Keypad
5
+ include Core
6
+ attr_reader :prompt, :keyloop, :window
7
+
8
+ def initialize window
9
+ @window = window
10
+ @prompt = Prompt.new(@window, self)
11
+ @monitor = Monitor.new
12
+ end
13
+
14
+ HELP_INFO = {
15
+ key_bindings: [
16
+ ["?", "shows this help"],
17
+ ["^", "eval prompt (app context, synchronized)"],
18
+ ["L", "show latest spooled log entries (no scrolling)"],
19
+ ["P", "kill SSH polling (if it stucks)"],
20
+ ["T", "create core dump and open in editor"],
21
+ ["q", "quit prompt"],
22
+ ["Q", "same as ctrl-c"],
23
+ [":", "main prompt"],
24
+ ],
25
+ main_commands: [
26
+ [["?", %w[h elp]], [], "shows this help"],
27
+ [[%w[q uit]], [], "quit prompt"],
28
+ [["q!", "quit!"], [], "same as ctrl-c"],
29
+ [["kill"], [], "(dirty) interrupts all workers"],
30
+ [["kill!"], [], "(dirty) essentially SIGKILL (no cleanup)"],
31
+ [["dump"], [], "create and open coredump"],
32
+ [["eval"], [[:optional, "code"]], "executes code or opens eval prompt (app context, synchronized)"],
33
+ [[%w[c ancel]], [[:mandatory, %w[table_name --all]]], "cancels given or all workers"],
34
+ [[%w[p ause]], [[:mandatory, %w[table_name --all]]], "pauses given or all workers"],
35
+ [[%w[r esume]], [[:mandatory, %w[table_name --all]]], "resumes given or all workers"],
36
+ ],
37
+ }
38
+
39
+ def handle_input ch
40
+ sync do
41
+ if @prompt.interactive?
42
+ @prompt.handle_input(ch)
43
+ else
44
+ case ch
45
+ when ":" then main_prompt
46
+ when "^" then eval_prompt # (@development)
47
+ when "L" then show_log # (@development)
48
+ when "T" then dump_core # (@development)
49
+ when "P" then kill_ssh_poll
50
+ when "?" then show_help
51
+ when "q" then quit_dialog
52
+ when "Q" then $core_runtime_exiting = 1
53
+ end
54
+ end
55
+ end
56
+ end
57
+
58
+ def main_prompt
59
+ prompt!(":") do |raw|
60
+ break if raw.blank?
61
+ args = raw.split(" ")
62
+ cmd = args.shift
63
+ case cmd
64
+ when "?", "h", "help" then show_help
65
+ when "c", "cancel" then cancel_workers(args)
66
+ when "q", "quit" then quit_dialog
67
+ when "q!", "quit!" then $core_runtime_exiting = 1
68
+ when "kill" then kill_workers
69
+ when "kill!" then kill_app
70
+ when "dump" then dump_core
71
+ when "eval" then args.any? ? _eval(args.join(" ")) : eval_prompt
72
+ when "p", "pause" then pause_workers(args)
73
+ when "r", "resume" then resume_workers(args)
74
+ end
75
+ end
76
+ end
77
+
78
+ def eval_prompt
79
+ prompt!("eval> ") {|evil| _eval(evil) }
80
+ end
81
+
82
+ def quit_dialog
83
+ q = "Do you want to abort all operations and quit?"
84
+ p = Proc.new do
85
+ blue q
86
+ gray " [y/q/t/1 n/f/0] "
87
+ end
88
+ prompt!(q,
89
+ prompt: p,
90
+ return_on_buffer: true,
91
+ capture_enter: false,
92
+ has_cursor: false
93
+ ) do |response|
94
+ if response == "q" || @window.strbool(response) == true
95
+ $core_runtime_exiting = 1
96
+ end
97
+ end
98
+ end
99
+
100
+ def show_help
101
+ view_was = window.change_view(:help)
102
+ prompt!("[press any key to return]",
103
+ return_on_buffer: true,
104
+ capture_enter: false,
105
+ has_cursor: false,
106
+ capture_escape: false,
107
+ cursor_visible: false
108
+ ) do |response|
109
+ window.change_view(view_was)
110
+ end
111
+ end
112
+
113
+ def show_log
114
+ view_was = window.change_view(:log)
115
+ prompt!("[press any key to return]",
116
+ return_on_buffer: true,
117
+ capture_enter: false,
118
+ has_cursor: false,
119
+ capture_escape: false,
120
+ cursor_visible: false
121
+ ) do |response|
122
+ window.change_view(view_was)
123
+ end
124
+ end
125
+
126
+ def pause_workers args
127
+ if args[0].is_a?(String)
128
+ _detect_worker(args.join(" "), &:pause)
129
+ else
130
+ prompt!("Usage: :p(ause) <table_name|--all>", color: :yellow)
131
+ end
132
+ end
133
+
134
+ def resume_workers args
135
+ if args[0].is_a?(String)
136
+ _detect_worker(args.join(" "), &:unpause)
137
+ else
138
+ prompt!("Usage: :r(esume) <table_name|--all>", color: :yellow)
139
+ end
140
+ end
141
+
142
+ def cancel_workers args
143
+ if args[0].is_a?(String)
144
+ _detect_worker(args.join(" ")) do |wrk|
145
+ wrk.cancel! "canceled by user"
146
+ end
147
+ else
148
+ prompt!("Usage: :c(cancel) <table_name|--all>", color: :yellow)
149
+ end
150
+ end
151
+
152
+ def kill_ssh_poll
153
+ return if sklaventreiber.workers.select{|w| !w.done? || w.sshing }.any?
154
+ sklaventreiber.poll.try(:kill)
155
+ end
156
+
157
+ def kill_workers
158
+ Thread.list.each do |thr|
159
+ thr.raise(Interrupt) if thr[:managed_worker]
160
+ end
161
+ end
162
+
163
+ def kill_app
164
+ exit!
165
+ end
166
+
167
+ def dump_core
168
+ app.dump_core
169
+ end
170
+ end
171
+ end
172
+ end
173
+ end
174
+
@@ -0,0 +1,124 @@
1
+ module DbSucker
2
+ class Application
3
+ class Window
4
+ class Prompt
5
+ attr_reader :cpos, :label, :buffer, :opts
6
+
7
+ def initialize window, keypad
8
+ @window = window
9
+ @keypad = keypad
10
+ reset_state(true)
11
+ end
12
+
13
+ def active?
14
+ @active
15
+ end
16
+
17
+ def interactive?
18
+ active? && @callback
19
+ end
20
+
21
+ def reset_state initializing = false
22
+ unless initializing
23
+ @window.set_cursor(0)
24
+ @keypad.app.fire(:prompt_stop, @label)
25
+ @window.force_cursor(nil)
26
+ end
27
+ @active = false
28
+ @buffer = ""
29
+ @label = nil
30
+ @callback = nil
31
+ @cpos = 0
32
+ @opts = {
33
+ color: :blue,
34
+ return_on_buffer: false,
35
+ capture_enter: true,
36
+ capture_escape: true,
37
+ has_cursor: true,
38
+ prompt: nil, # proc
39
+ cursor_visible: true,
40
+ }
41
+ end
42
+
43
+ def set! label, opts = {}, &callback
44
+ @label = label
45
+ @callback = callback
46
+ @active = true
47
+ @opts = @opts.merge(opts)
48
+ @keypad.app.fire(:prompt_start, @label, @opts)
49
+ @window.set_cursor(@opts[:cursor_visible] ? 2 : 0)
50
+ end
51
+
52
+ def render target, line
53
+ _this = self
54
+ target.instance_eval do
55
+ return unless _this.active?
56
+ setpos(line, 0)
57
+ clrtoeol
58
+ setpos(line, 0)
59
+ if _this.opts[:prompt]
60
+ instance_eval(&_this.opts[:prompt])
61
+ else
62
+ send(_this.opts[:color], _this.label)
63
+ end
64
+ white(_this.buffer)
65
+ force_cursor(line, stdscr.curx + _this.cpos)
66
+ end
67
+ end
68
+
69
+ def handle_input ch
70
+ case ch
71
+ when 27 then (@opts[:capture_escape] ? _escape : sbuf(ch))
72
+ when 127 then (@opts[:has_cursor] ? _backspace : sbuf(ch))
73
+ when 330 then (@opts[:has_cursor] ? _delete : sbuf(ch))
74
+ when 260 then (@opts[:has_cursor] ? _left_arrow : sbuf(ch))
75
+ when 261 then (@opts[:has_cursor] ? _right_arrow : sbuf(ch))
76
+ when 13 then (@opts[:capture_enter] ? _enter : sbuf(ch))
77
+ else sbuf(ch)
78
+ end
79
+ end
80
+
81
+ def sbuf ch
82
+ #@buffer.concat(ch.bytes.to_s)
83
+ @cpos.zero? ? @buffer.concat(ch) : @buffer.insert(@cpos - 1, ch)
84
+ if @opts[:return_on_buffer].is_a?(Regexp)
85
+ _enter if @buffer.match(@opts[:return_on_buffer])
86
+ elsif @opts[:return_on_buffer].is_a?(Proc)
87
+ _enter if @opts[:return_on_buffer].call(@buffer, ch)
88
+ elsif @opts[:return_on_buffer]
89
+ _enter
90
+ end
91
+ end
92
+
93
+ def _escape
94
+ reset_state
95
+ end
96
+
97
+ def _backspace
98
+ @buffer.slice!(@cpos - 1)
99
+ end
100
+
101
+ def _delete
102
+ return if @cpos.zero?
103
+ @buffer.slice!(@cpos)
104
+ @cpos += 1
105
+ end
106
+
107
+ def _left_arrow
108
+ @cpos = [@cpos - 1, -@buffer.length].max
109
+ end
110
+
111
+ def _right_arrow
112
+ @cpos = [@cpos + 1, 0].min
113
+ end
114
+
115
+ def _enter
116
+ _b = @buffer
117
+ _c = @callback
118
+ reset_state
119
+ _c.call(_b)
120
+ end
121
+ end
122
+ end
123
+ end
124
+ end