db_sucker 3.0.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (60) hide show
  1. checksums.yaml +7 -0
  2. data/.gitignore +16 -0
  3. data/CHANGELOG.md +45 -0
  4. data/Gemfile +4 -0
  5. data/LICENSE.txt +22 -0
  6. data/README.md +193 -0
  7. data/Rakefile +1 -0
  8. data/VERSION +1 -0
  9. data/bin/db_sucker +12 -0
  10. data/bin/db_sucker.sh +14 -0
  11. data/db_sucker.gemspec +29 -0
  12. data/doc/config_example.rb +53 -0
  13. data/doc/container_example.yml +150 -0
  14. data/lib/db_sucker/adapters/mysql2.rb +103 -0
  15. data/lib/db_sucker/application/colorize.rb +28 -0
  16. data/lib/db_sucker/application/container/accessors.rb +60 -0
  17. data/lib/db_sucker/application/container/ssh.rb +225 -0
  18. data/lib/db_sucker/application/container/validations.rb +53 -0
  19. data/lib/db_sucker/application/container/variation/accessors.rb +45 -0
  20. data/lib/db_sucker/application/container/variation/helpers.rb +21 -0
  21. data/lib/db_sucker/application/container/variation/worker_api.rb +65 -0
  22. data/lib/db_sucker/application/container/variation.rb +60 -0
  23. data/lib/db_sucker/application/container.rb +70 -0
  24. data/lib/db_sucker/application/container_collection.rb +47 -0
  25. data/lib/db_sucker/application/core.rb +222 -0
  26. data/lib/db_sucker/application/dispatch.rb +364 -0
  27. data/lib/db_sucker/application/evented_resultset.rb +149 -0
  28. data/lib/db_sucker/application/fake_channel.rb +22 -0
  29. data/lib/db_sucker/application/output_helper.rb +197 -0
  30. data/lib/db_sucker/application/sklaven_treiber/log_spool.rb +57 -0
  31. data/lib/db_sucker/application/sklaven_treiber/worker/accessors.rb +105 -0
  32. data/lib/db_sucker/application/sklaven_treiber/worker/core.rb +168 -0
  33. data/lib/db_sucker/application/sklaven_treiber/worker/helpers.rb +144 -0
  34. data/lib/db_sucker/application/sklaven_treiber/worker/io/base.rb +240 -0
  35. data/lib/db_sucker/application/sklaven_treiber/worker/io/file_copy.rb +81 -0
  36. data/lib/db_sucker/application/sklaven_treiber/worker/io/file_gunzip.rb +58 -0
  37. data/lib/db_sucker/application/sklaven_treiber/worker/io/file_import_sql.rb +80 -0
  38. data/lib/db_sucker/application/sklaven_treiber/worker/io/file_shasum.rb +49 -0
  39. data/lib/db_sucker/application/sklaven_treiber/worker/io/pv_wrapper.rb +73 -0
  40. data/lib/db_sucker/application/sklaven_treiber/worker/io/sftp_download.rb +57 -0
  41. data/lib/db_sucker/application/sklaven_treiber/worker/io/throughput.rb +219 -0
  42. data/lib/db_sucker/application/sklaven_treiber/worker/routines.rb +313 -0
  43. data/lib/db_sucker/application/sklaven_treiber/worker.rb +48 -0
  44. data/lib/db_sucker/application/sklaven_treiber.rb +281 -0
  45. data/lib/db_sucker/application/slot_pool.rb +137 -0
  46. data/lib/db_sucker/application/tie.rb +25 -0
  47. data/lib/db_sucker/application/window/core.rb +185 -0
  48. data/lib/db_sucker/application/window/dialog.rb +142 -0
  49. data/lib/db_sucker/application/window/keypad/core.rb +85 -0
  50. data/lib/db_sucker/application/window/keypad.rb +174 -0
  51. data/lib/db_sucker/application/window/prompt.rb +124 -0
  52. data/lib/db_sucker/application/window.rb +329 -0
  53. data/lib/db_sucker/application.rb +168 -0
  54. data/lib/db_sucker/patches/beta-warning.rb +374 -0
  55. data/lib/db_sucker/patches/developer.rb +29 -0
  56. data/lib/db_sucker/patches/net-sftp.rb +20 -0
  57. data/lib/db_sucker/patches/thread-count.rb +30 -0
  58. data/lib/db_sucker/version.rb +4 -0
  59. data/lib/db_sucker.rb +81 -0
  60. metadata +217 -0
@@ -0,0 +1,142 @@
1
+ module DbSucker
2
+ class Application
3
+ class Window
4
+ class Dialog
5
+ attr_accessor :border_color
6
+
7
+ BOX = {
8
+ tl: "┌",
9
+ t: "─",
10
+ tr: "┐",
11
+ l: "│",
12
+ r: "│",
13
+ bl: "└",
14
+ b: "─",
15
+ br: "┘",
16
+ hrl: "├",
17
+ hr: "─",
18
+ hrr: "┤",
19
+ }
20
+
21
+ def initialize window, &block
22
+ @window = window
23
+ @width = false
24
+ @border_color = :yellow
25
+ @border_padding = 1
26
+ @separator_padding = 1
27
+ @lines = []
28
+ block.call(self)
29
+ end
30
+
31
+ def line str = nil, color = :yellow, &block
32
+ if block
33
+ @lines << [].tap{|a| block.call(a) }
34
+ else
35
+ @lines << [[str, color]]
36
+ end
37
+ end
38
+
39
+ def br
40
+ @lines << []
41
+ end
42
+
43
+ def hr
44
+ @lines << :hr
45
+ end
46
+
47
+ def button *a
48
+ @lines.concat(build_button(*a))
49
+ end
50
+
51
+ def build_button str, color = :yellow
52
+ tl = BOX[:tl]
53
+ t = BOX[:t]
54
+ tr = BOX[:tr]
55
+ l = BOX[:l]
56
+ r = BOX[:r]
57
+ bl = BOX[:bl]
58
+ b = BOX[:b]
59
+ br = BOX[:br]
60
+ width = str.length + 2
61
+ [].tap do |a|
62
+ a << [["#{tl}" << "".ljust(width, t) << "#{tr}", color]]
63
+ a << [["#{l}" << " #{str} ".ljust(width, t) << "#{r}", color]]
64
+ a << [["#{bl}" << "".ljust(width, b) << "#{br}", color]]
65
+ end
66
+ end
67
+
68
+ def button_group buttons = [], spaces = 2, &block
69
+ spaces = buttons if block
70
+ btns = block ? [].tap{|a| block.call(a) } : buttons.dup
71
+ spaces = "".ljust(spaces, " ")
72
+ first = btns.shift
73
+
74
+ btns.each_with_index do |lines, bi|
75
+ lines.each_with_index do |l, i|
76
+ first[i] << [spaces, :yellow]
77
+ first[i].concat(l)
78
+ end
79
+ end
80
+
81
+ @lines.concat first
82
+ end
83
+
84
+ # -----------------------------------------------------------
85
+
86
+ def _nl
87
+ @window.next_line
88
+ end
89
+
90
+ def _hr
91
+ l = BOX[:hrl]
92
+ r = BOX[:hrr]
93
+ m = "".ljust(@width - l.length - r.length, BOX[:hr])
94
+ @separator_padding.times { _line }
95
+ @window.send(@border_color, "#{l}#{m}#{r}")
96
+ _nl
97
+ end
98
+
99
+ def _line parts = []
100
+ return _hr if parts == :hr
101
+ l = BOX[:l]
102
+ r = BOX[:r]
103
+ p = "".ljust(@border_padding, " ")
104
+
105
+ @window.send(@border_color, "#{l}#{p}")
106
+ parts.each do |str, color|
107
+ @window.send(color, str)
108
+ end
109
+ @window.send(@border_color, "".ljust(@width - l.length - r.length - (p.length * 2) - parts.map{|s, _| s.length }.sum, " "))
110
+ @window.send(@border_color, "#{p}#{r}")
111
+ _nl
112
+ end
113
+
114
+ def _tborder
115
+ tl = BOX[:tl]
116
+ tr = BOX[:tr]
117
+ m = "".ljust(@width - tl.length - tr.length, BOX[:t])
118
+ @window.send(@border_color, "#{tl}#{m}#{tr}")
119
+ _nl
120
+ end
121
+
122
+ def _bborder
123
+ bl = BOX[:bl]
124
+ br = BOX[:br]
125
+ m = "".ljust(@width - bl.length - br.length, BOX[:b])
126
+ @window.send(@border_color, "#{bl}#{m}#{br}")
127
+ end
128
+
129
+ def render!
130
+ @width = @lines.map{|c| c.is_a?(Array) ? c.map{|s, _| s.length }.sum : 0 }.compact.max + (@border_padding * 2) + BOX[:l].length + BOX[:r].length
131
+
132
+ _tborder
133
+ @border_padding.times { _line }
134
+ @lines.each {|parts| _line(parts) }
135
+ @border_padding.times { _line }
136
+ _bborder
137
+ end
138
+ end
139
+ end
140
+ end
141
+ end
142
+
@@ -0,0 +1,85 @@
1
+ module DbSucker
2
+ class Application
3
+ class Window
4
+ class Keypad
5
+ module Core
6
+ def sync &block
7
+ @monitor.synchronize(&block)
8
+ end
9
+
10
+ def sklaventreiber
11
+ @window.sklaventreiber
12
+ end
13
+
14
+ def app
15
+ @window.app
16
+ end
17
+
18
+ def enabled?
19
+ app.opts[:window_keypad]
20
+ end
21
+
22
+ def start_loop
23
+ return false unless enabled?
24
+ @keyloop = app.spawn_thread(:window_keypad_loop) do |thr|
25
+ loop {
26
+ begin
27
+ handle_input(@window.send(:getch))
28
+ rescue StandardError => ex
29
+ app.notify_exception("DbSucker::Window::Keypad encountered an input handle error on tick ##{@window.tick}", ex)
30
+ end
31
+ }
32
+ end
33
+ end
34
+
35
+ def stop_loop
36
+ return unless @keyloop
37
+ sync { @keyloop.kill }
38
+ @keyloop.join
39
+ end
40
+
41
+ def prompt! *a, &b
42
+ return false unless enabled?
43
+ @prompt.set!(*a, &b)
44
+ end
45
+
46
+ def _detect_worker arg, &block
47
+ sklaventreiber.sync do
48
+ if arg == "--all"
49
+ sklaventreiber.workers.each{|w| block.call(w) }
50
+ else
51
+ wrk = sklaventreiber.workers.detect do |w|
52
+ if arg.start_with?("^")
53
+ w.table.match(/#{arg}/i)
54
+ elsif arg.start_with?("/")
55
+ w.table.match(/#{arg[1..-1]}/i)
56
+ else
57
+ w.table == arg
58
+ end
59
+ end
60
+ if wrk
61
+ block.call(wrk)
62
+ else
63
+ prompt!("Could not find any worker by the pattern `#{arg}'", color: :red)
64
+ end
65
+ end
66
+ end
67
+ end
68
+
69
+ def _eval evil
70
+ return if evil.blank?
71
+ app.dump_file "eval-result", true do |f|
72
+ begin
73
+ f.puts("#{evil}\n\n")
74
+ f.puts(app.sync{ app.instance_eval(evil) })
75
+ rescue StandardError => ex
76
+ f.puts("#{ex.class}: #{ex.message}")
77
+ ex.backtrace.each {|l| f.puts(" #{l}") }
78
+ end
79
+ end
80
+ end
81
+ end
82
+ end
83
+ end
84
+ end
85
+ end
@@ -0,0 +1,174 @@
1
+ module DbSucker
2
+ class Application
3
+ class Window
4
+ class Keypad
5
+ include Core
6
+ attr_reader :prompt, :keyloop, :window
7
+
8
+ def initialize window
9
+ @window = window
10
+ @prompt = Prompt.new(@window, self)
11
+ @monitor = Monitor.new
12
+ end
13
+
14
+ HELP_INFO = {
15
+ key_bindings: [
16
+ ["?", "shows this help"],
17
+ ["^", "eval prompt (app context, synchronized)"],
18
+ ["L", "show latest spooled log entries (no scrolling)"],
19
+ ["P", "kill SSH polling (if it stucks)"],
20
+ ["T", "create core dump and open in editor"],
21
+ ["q", "quit prompt"],
22
+ ["Q", "same as ctrl-c"],
23
+ [":", "main prompt"],
24
+ ],
25
+ main_commands: [
26
+ [["?", %w[h elp]], [], "shows this help"],
27
+ [[%w[q uit]], [], "quit prompt"],
28
+ [["q!", "quit!"], [], "same as ctrl-c"],
29
+ [["kill"], [], "(dirty) interrupts all workers"],
30
+ [["kill!"], [], "(dirty) essentially SIGKILL (no cleanup)"],
31
+ [["dump"], [], "create and open coredump"],
32
+ [["eval"], [[:optional, "code"]], "executes code or opens eval prompt (app context, synchronized)"],
33
+ [[%w[c ancel]], [[:mandatory, %w[table_name --all]]], "cancels given or all workers"],
34
+ [[%w[p ause]], [[:mandatory, %w[table_name --all]]], "pauses given or all workers"],
35
+ [[%w[r esume]], [[:mandatory, %w[table_name --all]]], "resumes given or all workers"],
36
+ ],
37
+ }
38
+
39
+ def handle_input ch
40
+ sync do
41
+ if @prompt.interactive?
42
+ @prompt.handle_input(ch)
43
+ else
44
+ case ch
45
+ when ":" then main_prompt
46
+ when "^" then eval_prompt # (@development)
47
+ when "L" then show_log # (@development)
48
+ when "T" then dump_core # (@development)
49
+ when "P" then kill_ssh_poll
50
+ when "?" then show_help
51
+ when "q" then quit_dialog
52
+ when "Q" then $core_runtime_exiting = 1
53
+ end
54
+ end
55
+ end
56
+ end
57
+
58
+ def main_prompt
59
+ prompt!(":") do |raw|
60
+ break if raw.blank?
61
+ args = raw.split(" ")
62
+ cmd = args.shift
63
+ case cmd
64
+ when "?", "h", "help" then show_help
65
+ when "c", "cancel" then cancel_workers(args)
66
+ when "q", "quit" then quit_dialog
67
+ when "q!", "quit!" then $core_runtime_exiting = 1
68
+ when "kill" then kill_workers
69
+ when "kill!" then kill_app
70
+ when "dump" then dump_core
71
+ when "eval" then args.any? ? _eval(args.join(" ")) : eval_prompt
72
+ when "p", "pause" then pause_workers(args)
73
+ when "r", "resume" then resume_workers(args)
74
+ end
75
+ end
76
+ end
77
+
78
+ def eval_prompt
79
+ prompt!("eval> ") {|evil| _eval(evil) }
80
+ end
81
+
82
+ def quit_dialog
83
+ q = "Do you want to abort all operations and quit?"
84
+ p = Proc.new do
85
+ blue q
86
+ gray " [y/q/t/1 n/f/0] "
87
+ end
88
+ prompt!(q,
89
+ prompt: p,
90
+ return_on_buffer: true,
91
+ capture_enter: false,
92
+ has_cursor: false
93
+ ) do |response|
94
+ if response == "q" || @window.strbool(response) == true
95
+ $core_runtime_exiting = 1
96
+ end
97
+ end
98
+ end
99
+
100
+ def show_help
101
+ view_was = window.change_view(:help)
102
+ prompt!("[press any key to return]",
103
+ return_on_buffer: true,
104
+ capture_enter: false,
105
+ has_cursor: false,
106
+ capture_escape: false,
107
+ cursor_visible: false
108
+ ) do |response|
109
+ window.change_view(view_was)
110
+ end
111
+ end
112
+
113
+ def show_log
114
+ view_was = window.change_view(:log)
115
+ prompt!("[press any key to return]",
116
+ return_on_buffer: true,
117
+ capture_enter: false,
118
+ has_cursor: false,
119
+ capture_escape: false,
120
+ cursor_visible: false
121
+ ) do |response|
122
+ window.change_view(view_was)
123
+ end
124
+ end
125
+
126
+ def pause_workers args
127
+ if args[0].is_a?(String)
128
+ _detect_worker(args.join(" "), &:pause)
129
+ else
130
+ prompt!("Usage: :p(ause) <table_name|--all>", color: :yellow)
131
+ end
132
+ end
133
+
134
+ def resume_workers args
135
+ if args[0].is_a?(String)
136
+ _detect_worker(args.join(" "), &:unpause)
137
+ else
138
+ prompt!("Usage: :r(esume) <table_name|--all>", color: :yellow)
139
+ end
140
+ end
141
+
142
+ def cancel_workers args
143
+ if args[0].is_a?(String)
144
+ _detect_worker(args.join(" ")) do |wrk|
145
+ wrk.cancel! "canceled by user"
146
+ end
147
+ else
148
+ prompt!("Usage: :c(cancel) <table_name|--all>", color: :yellow)
149
+ end
150
+ end
151
+
152
+ def kill_ssh_poll
153
+ return if sklaventreiber.workers.select{|w| !w.done? || w.sshing }.any?
154
+ sklaventreiber.poll.try(:kill)
155
+ end
156
+
157
+ def kill_workers
158
+ Thread.list.each do |thr|
159
+ thr.raise(Interrupt) if thr[:managed_worker]
160
+ end
161
+ end
162
+
163
+ def kill_app
164
+ exit!
165
+ end
166
+
167
+ def dump_core
168
+ app.dump_core
169
+ end
170
+ end
171
+ end
172
+ end
173
+ end
174
+
@@ -0,0 +1,124 @@
1
+ module DbSucker
2
+ class Application
3
+ class Window
4
+ class Prompt
5
+ attr_reader :cpos, :label, :buffer, :opts
6
+
7
+ def initialize window, keypad
8
+ @window = window
9
+ @keypad = keypad
10
+ reset_state(true)
11
+ end
12
+
13
+ def active?
14
+ @active
15
+ end
16
+
17
+ def interactive?
18
+ active? && @callback
19
+ end
20
+
21
+ def reset_state initializing = false
22
+ unless initializing
23
+ @window.set_cursor(0)
24
+ @keypad.app.fire(:prompt_stop, @label)
25
+ @window.force_cursor(nil)
26
+ end
27
+ @active = false
28
+ @buffer = ""
29
+ @label = nil
30
+ @callback = nil
31
+ @cpos = 0
32
+ @opts = {
33
+ color: :blue,
34
+ return_on_buffer: false,
35
+ capture_enter: true,
36
+ capture_escape: true,
37
+ has_cursor: true,
38
+ prompt: nil, # proc
39
+ cursor_visible: true,
40
+ }
41
+ end
42
+
43
+ def set! label, opts = {}, &callback
44
+ @label = label
45
+ @callback = callback
46
+ @active = true
47
+ @opts = @opts.merge(opts)
48
+ @keypad.app.fire(:prompt_start, @label, @opts)
49
+ @window.set_cursor(@opts[:cursor_visible] ? 2 : 0)
50
+ end
51
+
52
+ def render target, line
53
+ _this = self
54
+ target.instance_eval do
55
+ return unless _this.active?
56
+ setpos(line, 0)
57
+ clrtoeol
58
+ setpos(line, 0)
59
+ if _this.opts[:prompt]
60
+ instance_eval(&_this.opts[:prompt])
61
+ else
62
+ send(_this.opts[:color], _this.label)
63
+ end
64
+ white(_this.buffer)
65
+ force_cursor(line, stdscr.curx + _this.cpos)
66
+ end
67
+ end
68
+
69
+ def handle_input ch
70
+ case ch
71
+ when 27 then (@opts[:capture_escape] ? _escape : sbuf(ch))
72
+ when 127 then (@opts[:has_cursor] ? _backspace : sbuf(ch))
73
+ when 330 then (@opts[:has_cursor] ? _delete : sbuf(ch))
74
+ when 260 then (@opts[:has_cursor] ? _left_arrow : sbuf(ch))
75
+ when 261 then (@opts[:has_cursor] ? _right_arrow : sbuf(ch))
76
+ when 13 then (@opts[:capture_enter] ? _enter : sbuf(ch))
77
+ else sbuf(ch)
78
+ end
79
+ end
80
+
81
+ def sbuf ch
82
+ #@buffer.concat(ch.bytes.to_s)
83
+ @cpos.zero? ? @buffer.concat(ch) : @buffer.insert(@cpos - 1, ch)
84
+ if @opts[:return_on_buffer].is_a?(Regexp)
85
+ _enter if @buffer.match(@opts[:return_on_buffer])
86
+ elsif @opts[:return_on_buffer].is_a?(Proc)
87
+ _enter if @opts[:return_on_buffer].call(@buffer, ch)
88
+ elsif @opts[:return_on_buffer]
89
+ _enter
90
+ end
91
+ end
92
+
93
+ def _escape
94
+ reset_state
95
+ end
96
+
97
+ def _backspace
98
+ @buffer.slice!(@cpos - 1)
99
+ end
100
+
101
+ def _delete
102
+ return if @cpos.zero?
103
+ @buffer.slice!(@cpos)
104
+ @cpos += 1
105
+ end
106
+
107
+ def _left_arrow
108
+ @cpos = [@cpos - 1, -@buffer.length].max
109
+ end
110
+
111
+ def _right_arrow
112
+ @cpos = [@cpos + 1, 0].min
113
+ end
114
+
115
+ def _enter
116
+ _b = @buffer
117
+ _c = @callback
118
+ reset_state
119
+ _c.call(_b)
120
+ end
121
+ end
122
+ end
123
+ end
124
+ end