db_sucker 3.0.3 → 3.1.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +31 -0
- data/README.md +2 -2
- data/VERSION +1 -1
- data/lib/db_sucker/application/core.rb +1 -1
- data/lib/db_sucker/application/dispatch.rb +21 -9
- data/lib/db_sucker/application/sklaven_treiber/worker/io/file_gunzip.rb +0 -1
- data/lib/db_sucker/application/sklaven_treiber/worker/io/throughput.rb +2 -2
- data/lib/db_sucker/application/sklaven_treiber/worker/routines.rb +2 -1
- data/lib/db_sucker/application/sklaven_treiber.rb +16 -0
- data/lib/db_sucker/application/window.rb +2 -5
- data/lib/db_sucker/version.rb +1 -1
- data/lib/db_sucker.rb +0 -1
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 1bad32bfbe4d3e2a24def83464a9bdce5c1c7f3158f558ed405d15967adc1b4e
|
4
|
+
data.tar.gz: b5518954de8c6633ac22979b53f225eb7ad1e609fc191931290287479da69547
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: aa025dc2bf4dac913e9e4a7295980d8030cb795cedb6edcd458d26783f8349499879bbb6520a2d09e9f1dc879e531656c7e639119fcee80e0b36df613748723c
|
7
|
+
data.tar.gz: 55f8415d7f31e16d6b6d8aec346c37ce8cd30112d4f5aaa516200dd2a75c92099ed31994d2691cba1e360f36807f93b638ae83a1c4fa367e02bfa8922b01d97f
|
data/CHANGELOG.md
CHANGED
@@ -1,3 +1,34 @@
|
|
1
|
+
## 3.1.0
|
2
|
+
|
3
|
+
### Fixes
|
4
|
+
|
5
|
+
* Prevent app to run out of consumers when tasks are waiting for defer ready
|
6
|
+
* Prevent IO errors on Ruby < 2.3 in uncompress
|
7
|
+
* Prevent racing conditions in SSH diagnose task
|
8
|
+
* Minor fixes
|
9
|
+
|
10
|
+
-------------------
|
11
|
+
|
12
|
+
## 3.0.3
|
13
|
+
|
14
|
+
* no changes, fixed my tag screwup
|
15
|
+
|
16
|
+
-------------------
|
17
|
+
|
18
|
+
## 3.0.2
|
19
|
+
|
20
|
+
* no changes, github tags can't be altered, I screwed up
|
21
|
+
|
22
|
+
-------------------
|
23
|
+
|
24
|
+
## 3.0.1
|
25
|
+
|
26
|
+
### Fixes
|
27
|
+
|
28
|
+
* hotfix for 3.0.0 release
|
29
|
+
|
30
|
+
-------------------
|
31
|
+
|
1
32
|
## 3.0.0
|
2
33
|
|
3
34
|
### Updates
|
data/README.md
CHANGED
@@ -7,7 +7,8 @@ You configure your hosts via an YAML configuration in which you can define multi
|
|
7
7
|
|
8
8
|
This tool is meant for pulling live data into your development environment. **It is not designed for backups!** but you might get away with it.
|
9
9
|
|
10
|
-
![screenshot](https://i.imgur.com/EAjWrEd.png)
|
10
|
+
[![screenshot](https://i.imgur.com/EAjWrEd.png)](https://www.youtube.com/watch?v=jdhyQzJOIkE)
|
11
|
+
[▶ see it in action](https://www.youtube.com/watch?v=jdhyQzJOIkE)
|
11
12
|
|
12
13
|
---
|
13
14
|
## Alpha product (v3 is a rewrite), use at your own risk, always have a backup!
|
@@ -159,7 +160,6 @@ Currently there is only the "binary" importer which will use the mysql client bi
|
|
159
160
|
### General
|
160
161
|
|
161
162
|
* Ruby 2.3.0 has a bug that might segfault your ruby if some exceptions occur, this is fixed since 2.3.1 and later
|
162
|
-
* Consumers that are waiting (e.g. deferred or slot pool) won't release their tasks, if you have to few consumers you might softlock
|
163
163
|
|
164
164
|
### SSH errors / MaxSessions
|
165
165
|
|
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
3.0
|
1
|
+
3.1.0
|
@@ -107,42 +107,54 @@ module DbSucker
|
|
107
107
|
|
108
108
|
# via -a/--action sshdiag
|
109
109
|
def dispatch_sshdiag
|
110
|
-
log c("\nPlease wait while we run some tests...\n", :blue)
|
111
110
|
_identifier, _ctn = false, false, false
|
112
111
|
idstr = ARGV.shift
|
113
112
|
varstr = ARGV.shift
|
114
113
|
|
115
114
|
configful_dispatch(idstr, varstr) do |identifier, ctn, variation, var|
|
115
|
+
log c("\nPlease wait while we run some tests...\n", :blue)
|
116
116
|
_identifier = identifier
|
117
117
|
_ctn = ctn
|
118
118
|
channels = []
|
119
|
+
monitor = Monitor.new
|
119
120
|
stop = false
|
120
121
|
maxsessions = :unknown
|
121
122
|
begin
|
122
123
|
t = Thread.new {
|
123
124
|
begin
|
124
125
|
loop do
|
125
|
-
ctn.loop_ssh(0.1)
|
126
|
+
ctn.loop_ssh(0.1) { monitor.synchronize { channels.any? } }
|
126
127
|
end
|
127
128
|
rescue DbSucker::Application::Container::SSH::ChannelOpenFailedError
|
128
|
-
|
129
|
-
|
129
|
+
monitor.synchronize do
|
130
|
+
maxsessions = channels.length - channels.select{|c| c[:open_failed] }.length
|
131
|
+
stop = true
|
132
|
+
print "!"
|
133
|
+
end
|
130
134
|
retry
|
131
135
|
end
|
132
136
|
}
|
133
137
|
250.times do
|
134
|
-
break if stop
|
138
|
+
break if monitor.synchronize { stop }
|
135
139
|
c, r = ctn.blocking_channel_result("sleep 60", blocking: false, channel: true, use_sh: true)
|
136
|
-
|
140
|
+
monitor.synchronize do
|
141
|
+
channels << c
|
142
|
+
print "+"
|
143
|
+
end
|
137
144
|
sleep 0.1
|
138
145
|
end
|
139
146
|
ensure
|
140
147
|
debug "Stopping sessions (#{channels.length})..."
|
141
|
-
|
142
|
-
|
148
|
+
i = 1
|
149
|
+
loop do
|
150
|
+
break if monitor.synchronize { channels.empty? }
|
151
|
+
c = monitor.synchronize { channels.shift }
|
152
|
+
debug "Channel ##{i} #{c[:pid] ? "with PID #{c[:pid]}" : "has no PID"}"
|
143
153
|
ctn.kill_remote_process(c[:pid]) if c[:pid]
|
154
|
+
print "-"
|
155
|
+
i += 1
|
144
156
|
end
|
145
|
-
log c("\nSSH MaxSessions: #{c maxsessions, :magenta}", :cyan)
|
157
|
+
log c("\n\nSSH MaxSessions: #{c maxsessions, :magenta}", :cyan)
|
146
158
|
log "This value determines how many sessions we can multiplex over a single TCP connection."
|
147
159
|
log "Currently, DbSucker can only utilize one connection, thus this value defines the maxmium concurrency."
|
148
160
|
log "If you get errors you can either"
|
@@ -66,12 +66,12 @@ module DbSucker
|
|
66
66
|
|
67
67
|
def register target
|
68
68
|
sync do
|
69
|
-
if @instances[target]
|
69
|
+
if @instances[target.object_id]
|
70
70
|
raise InstanceAlreadyRegisteredError, "throughput manager cannot register more than once on the same target: `#{target}'"
|
71
71
|
else
|
72
72
|
raise NotImplementedError, "throughput manager requires the target to respond_to?(:filesize)" unless target.respond_to?(:filesize)
|
73
73
|
raise NotImplementedError, "throughput manager requires the target to respond_to?(:offset)" unless target.respond_to?(:offset)
|
74
|
-
@instances[target] = Instance.new(self, target)
|
74
|
+
@instances[target.object_id] = Instance.new(self, target)
|
75
75
|
end
|
76
76
|
end
|
77
77
|
end
|
@@ -247,6 +247,7 @@ module DbSucker
|
|
247
247
|
|
248
248
|
def _l_wait_for_workers
|
249
249
|
@perform << "l_import_file_deferred"
|
250
|
+
Thread.main.sync { Thread.main[:summon_workers] += 1 }
|
250
251
|
wait_defer_ready
|
251
252
|
end
|
252
253
|
|
@@ -302,7 +303,7 @@ module DbSucker
|
|
302
303
|
end
|
303
304
|
}
|
304
305
|
else
|
305
|
-
raise ImporterNotFoundError, "variation `#{cfg.name}/#{name}' defines unknown importer `#{imp}' (in `#{cfg.src}')"
|
306
|
+
raise Container::Variation::ImporterNotFoundError, "variation `#{@var.cfg.name}/#{@var.name}' defines unknown importer `#{imp}' (in `#{@var.cfg.src}')"
|
306
307
|
end
|
307
308
|
t.join
|
308
309
|
end
|
@@ -242,8 +242,24 @@ module DbSucker
|
|
242
242
|
@threads.each{|t| t[:start] = true; t.signal }
|
243
243
|
|
244
244
|
# master thread (control)
|
245
|
+
additionals = 0
|
246
|
+
Thread.current[:summon_workers] = 0
|
245
247
|
while @threads.any?(&:alive?)
|
246
248
|
_control_thread
|
249
|
+
Thread.current.sync do
|
250
|
+
Thread.current[:summon_workers].times do
|
251
|
+
app.debug "Spawned additional worker due to deferred import to prevent softlocks"
|
252
|
+
@threads << app.spawn_thread(:sklaventreiber_worker) {|thr|
|
253
|
+
begin
|
254
|
+
additionals += 1
|
255
|
+
thr[:managed_worker] = cnum + additionals
|
256
|
+
_queueoff
|
257
|
+
rescue Interrupt
|
258
|
+
end
|
259
|
+
}
|
260
|
+
end
|
261
|
+
Thread.current[:summon_workers] = 0
|
262
|
+
end
|
247
263
|
Thread.current.wait(0.1)
|
248
264
|
end
|
249
265
|
@threads.each(&:join)
|
@@ -4,9 +4,6 @@ module DbSucker
|
|
4
4
|
include Core
|
5
5
|
include Curses
|
6
6
|
COLOR_GRAY = 8
|
7
|
-
COL1 = 20
|
8
|
-
COL2 = 25
|
9
|
-
COL3 = 20
|
10
7
|
OutputHelper.hook(self)
|
11
8
|
|
12
9
|
attr_reader :app, :sklaventreiber, :keypad, :tick, :spinner_frames
|
@@ -193,7 +190,7 @@ module DbSucker
|
|
193
190
|
if opts[:threads]
|
194
191
|
next_line
|
195
192
|
yellow " Threads: "
|
196
|
-
blue "#{Thread.list.length}
|
193
|
+
blue "#{Thread.list.length}".ljust(3, " ")
|
197
194
|
end
|
198
195
|
|
199
196
|
if opts[:started]
|
@@ -292,7 +289,7 @@ module DbSucker
|
|
292
289
|
|
293
290
|
puts
|
294
291
|
puts c(" Status: ") << c(sklaventreiber.status[0], sklaventreiber.status[1].presence || "red")
|
295
|
-
puts c(" Threads: ") << c("#{Thread.list.length} "
|
292
|
+
puts c(" Threads: ") << c("#{Thread.list.length} ", :blue)
|
296
293
|
puts c(" Started: ") << c("#{@app.boot}", :blue) << c(" (") << c(human_seconds(Time.current - app.boot), :blue) << c(")")
|
297
294
|
puts c("Transaction ID: ") << c("#{sklaventreiber.trxid}", :cyan)
|
298
295
|
puts c(" Database: ") << c(t_db || "?", :magenta) << c(" (transferred ") << c(t_total || "?", :blue) << c(" of ") << c(t_done || "?", :blue) << c(" tables)")
|
data/lib/db_sucker/version.rb
CHANGED
data/lib/db_sucker.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: db_sucker
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 3.0
|
4
|
+
version: 3.1.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Sven Pachnit
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2018-
|
11
|
+
date: 2018-04-10 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: curses
|