db_sucker 3.0.3 → 3.1.0

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 6bcba24bcde9938e7e815cee1a58a93d69079546dd35abc89a626ed2477c6ae9
4
- data.tar.gz: 1ea0a67982883c2d30109585782ea6681baf06924ce2e5fd0b58a0cf9e2ee5cd
3
+ metadata.gz: 1bad32bfbe4d3e2a24def83464a9bdce5c1c7f3158f558ed405d15967adc1b4e
4
+ data.tar.gz: b5518954de8c6633ac22979b53f225eb7ad1e609fc191931290287479da69547
5
5
  SHA512:
6
- metadata.gz: cb49a8700e37f0d00c9544b37fe8c97a7f04fd895ec9bc7bd7368f8bf79e266075f2dfd3038ace781cde60dcce2cf9347e9700fefc23e03acb0acba4ae10700e
7
- data.tar.gz: a669ab21301869f1bf9d78eff4e5071faf9111fd801d4c70660e2c533054ab76f7a2585aa63691045d77a956aef005d95aafc25995bb96d03fbe9f992aeb0610
6
+ metadata.gz: aa025dc2bf4dac913e9e4a7295980d8030cb795cedb6edcd458d26783f8349499879bbb6520a2d09e9f1dc879e531656c7e639119fcee80e0b36df613748723c
7
+ data.tar.gz: 55f8415d7f31e16d6b6d8aec346c37ce8cd30112d4f5aaa516200dd2a75c92099ed31994d2691cba1e360f36807f93b638ae83a1c4fa367e02bfa8922b01d97f
data/CHANGELOG.md CHANGED
@@ -1,3 +1,34 @@
1
+ ## 3.1.0
2
+
3
+ ### Fixes
4
+
5
+ * Prevent app to run out of consumers when tasks are waiting for defer ready
6
+ * Prevent IO errors on Ruby < 2.3 in uncompress
7
+ * Prevent racing conditions in SSH diagnose task
8
+ * Minor fixes
9
+
10
+ -------------------
11
+
12
+ ## 3.0.3
13
+
14
+ * no changes, fixed my tag screwup
15
+
16
+ -------------------
17
+
18
+ ## 3.0.2
19
+
20
+ * no changes, github tags can't be altered, I screwed up
21
+
22
+ -------------------
23
+
24
+ ## 3.0.1
25
+
26
+ ### Fixes
27
+
28
+ * hotfix for 3.0.0 release
29
+
30
+ -------------------
31
+
1
32
  ## 3.0.0
2
33
 
3
34
  ### Updates
data/README.md CHANGED
@@ -7,7 +7,8 @@ You configure your hosts via an YAML configuration in which you can define multi
7
7
 
8
8
  This tool is meant for pulling live data into your development environment. **It is not designed for backups!** but you might get away with it.
9
9
 
10
- ![screenshot](https://i.imgur.com/EAjWrEd.png)
10
+ [![screenshot](https://i.imgur.com/EAjWrEd.png)](https://www.youtube.com/watch?v=jdhyQzJOIkE)
11
+ [▶ see it in action](https://www.youtube.com/watch?v=jdhyQzJOIkE)
11
12
 
12
13
  ---
13
14
  ## Alpha product (v3 is a rewrite), use at your own risk, always have a backup!
@@ -159,7 +160,6 @@ Currently there is only the "binary" importer which will use the mysql client bi
159
160
  ### General
160
161
 
161
162
  * Ruby 2.3.0 has a bug that might segfault your ruby if some exceptions occur, this is fixed since 2.3.1 and later
162
- * Consumers that are waiting (e.g. deferred or slot pool) won't release their tasks, if you have to few consumers you might softlock
163
163
 
164
164
  ### SSH errors / MaxSessions
165
165
 
data/VERSION CHANGED
@@ -1 +1 @@
1
- 3.0.3
1
+ 3.1.0
@@ -94,7 +94,7 @@ module DbSucker
94
94
  end.tap do |thr|
95
95
  # set type and priority
96
96
  thr[:itype] = type
97
- thr.priority = @opts[:"tp_#{type}"]
97
+ thr.priority = @opts[:"tp_#{type}"] || 0
98
98
  thr.abort_on_exception = true
99
99
 
100
100
  # add signal methods
@@ -107,42 +107,54 @@ module DbSucker
107
107
 
108
108
  # via -a/--action sshdiag
109
109
  def dispatch_sshdiag
110
- log c("\nPlease wait while we run some tests...\n", :blue)
111
110
  _identifier, _ctn = false, false, false
112
111
  idstr = ARGV.shift
113
112
  varstr = ARGV.shift
114
113
 
115
114
  configful_dispatch(idstr, varstr) do |identifier, ctn, variation, var|
115
+ log c("\nPlease wait while we run some tests...\n", :blue)
116
116
  _identifier = identifier
117
117
  _ctn = ctn
118
118
  channels = []
119
+ monitor = Monitor.new
119
120
  stop = false
120
121
  maxsessions = :unknown
121
122
  begin
122
123
  t = Thread.new {
123
124
  begin
124
125
  loop do
125
- ctn.loop_ssh(0.1)
126
+ ctn.loop_ssh(0.1) { monitor.synchronize { channels.any? } }
126
127
  end
127
128
  rescue DbSucker::Application::Container::SSH::ChannelOpenFailedError
128
- maxsessions = channels.length - channels.select{|c| c[:open_failed] }.length
129
- stop = true
129
+ monitor.synchronize do
130
+ maxsessions = channels.length - channels.select{|c| c[:open_failed] }.length
131
+ stop = true
132
+ print "!"
133
+ end
130
134
  retry
131
135
  end
132
136
  }
133
137
  250.times do
134
- break if stop
138
+ break if monitor.synchronize { stop }
135
139
  c, r = ctn.blocking_channel_result("sleep 60", blocking: false, channel: true, use_sh: true)
136
- channels << c
140
+ monitor.synchronize do
141
+ channels << c
142
+ print "+"
143
+ end
137
144
  sleep 0.1
138
145
  end
139
146
  ensure
140
147
  debug "Stopping sessions (#{channels.length})..."
141
- channels.each_with_index do |c, i|
142
- debug "Channel ##{i+1} #{c[:pid] ? "with PID #{c[:pid]}" : "has no PID"}"
148
+ i = 1
149
+ loop do
150
+ break if monitor.synchronize { channels.empty? }
151
+ c = monitor.synchronize { channels.shift }
152
+ debug "Channel ##{i} #{c[:pid] ? "with PID #{c[:pid]}" : "has no PID"}"
143
153
  ctn.kill_remote_process(c[:pid]) if c[:pid]
154
+ print "-"
155
+ i += 1
144
156
  end
145
- log c("\nSSH MaxSessions: #{c maxsessions, :magenta}", :cyan)
157
+ log c("\n\nSSH MaxSessions: #{c maxsessions, :magenta}", :cyan)
146
158
  log "This value determines how many sessions we can multiplex over a single TCP connection."
147
159
  log "Currently, DbSucker can only utilize one connection, thus this value defines the maxmium concurrency."
148
160
  log "If you get errors you can either"
@@ -42,7 +42,6 @@ module DbSucker
42
42
  ensure
43
43
  @state = :finishing
44
44
  gz.close
45
- @in_file.close
46
45
  @out_file.close
47
46
  end
48
47
 
@@ -66,12 +66,12 @@ module DbSucker
66
66
 
67
67
  def register target
68
68
  sync do
69
- if @instances[target]
69
+ if @instances[target.object_id]
70
70
  raise InstanceAlreadyRegisteredError, "throughput manager cannot register more than once on the same target: `#{target}'"
71
71
  else
72
72
  raise NotImplementedError, "throughput manager requires the target to respond_to?(:filesize)" unless target.respond_to?(:filesize)
73
73
  raise NotImplementedError, "throughput manager requires the target to respond_to?(:offset)" unless target.respond_to?(:offset)
74
- @instances[target] = Instance.new(self, target)
74
+ @instances[target.object_id] = Instance.new(self, target)
75
75
  end
76
76
  end
77
77
  end
@@ -247,6 +247,7 @@ module DbSucker
247
247
 
248
248
  def _l_wait_for_workers
249
249
  @perform << "l_import_file_deferred"
250
+ Thread.main.sync { Thread.main[:summon_workers] += 1 }
250
251
  wait_defer_ready
251
252
  end
252
253
 
@@ -302,7 +303,7 @@ module DbSucker
302
303
  end
303
304
  }
304
305
  else
305
- raise ImporterNotFoundError, "variation `#{cfg.name}/#{name}' defines unknown importer `#{imp}' (in `#{cfg.src}')"
306
+ raise Container::Variation::ImporterNotFoundError, "variation `#{@var.cfg.name}/#{@var.name}' defines unknown importer `#{imp}' (in `#{@var.cfg.src}')"
306
307
  end
307
308
  t.join
308
309
  end
@@ -242,8 +242,24 @@ module DbSucker
242
242
  @threads.each{|t| t[:start] = true; t.signal }
243
243
 
244
244
  # master thread (control)
245
+ additionals = 0
246
+ Thread.current[:summon_workers] = 0
245
247
  while @threads.any?(&:alive?)
246
248
  _control_thread
249
+ Thread.current.sync do
250
+ Thread.current[:summon_workers].times do
251
+ app.debug "Spawned additional worker due to deferred import to prevent softlocks"
252
+ @threads << app.spawn_thread(:sklaventreiber_worker) {|thr|
253
+ begin
254
+ additionals += 1
255
+ thr[:managed_worker] = cnum + additionals
256
+ _queueoff
257
+ rescue Interrupt
258
+ end
259
+ }
260
+ end
261
+ Thread.current[:summon_workers] = 0
262
+ end
247
263
  Thread.current.wait(0.1)
248
264
  end
249
265
  @threads.each(&:join)
@@ -4,9 +4,6 @@ module DbSucker
4
4
  include Core
5
5
  include Curses
6
6
  COLOR_GRAY = 8
7
- COL1 = 20
8
- COL2 = 25
9
- COL3 = 20
10
7
  OutputHelper.hook(self)
11
8
 
12
9
  attr_reader :app, :sklaventreiber, :keypad, :tick, :spinner_frames
@@ -193,7 +190,7 @@ module DbSucker
193
190
  if opts[:threads]
194
191
  next_line
195
192
  yellow " Threads: "
196
- blue "#{Thread.list.length} ".ljust(COL1, " ")
193
+ blue "#{Thread.list.length}".ljust(3, " ")
197
194
  end
198
195
 
199
196
  if opts[:started]
@@ -292,7 +289,7 @@ module DbSucker
292
289
 
293
290
  puts
294
291
  puts c(" Status: ") << c(sklaventreiber.status[0], sklaventreiber.status[1].presence || "red")
295
- puts c(" Threads: ") << c("#{Thread.list.length} ".ljust(COL1, " "), :blue)
292
+ puts c(" Threads: ") << c("#{Thread.list.length} ", :blue)
296
293
  puts c(" Started: ") << c("#{@app.boot}", :blue) << c(" (") << c(human_seconds(Time.current - app.boot), :blue) << c(")")
297
294
  puts c("Transaction ID: ") << c("#{sklaventreiber.trxid}", :cyan)
298
295
  puts c(" Database: ") << c(t_db || "?", :magenta) << c(" (transferred ") << c(t_total || "?", :blue) << c(" of ") << c(t_done || "?", :blue) << c(" tables)")
@@ -1,4 +1,4 @@
1
1
  module DbSucker
2
- VERSION = "3.0.3"
2
+ VERSION = "3.1.0"
3
3
  UPDATE_URL = "https://raw.githubusercontent.com/2called-chaos/db_sucker/master/VERSION"
4
4
  end
data/lib/db_sucker.rb CHANGED
@@ -4,7 +4,6 @@ end
4
4
 
5
5
  # stdlib
6
6
  require "benchmark"
7
- require "optparse"
8
7
  require "fileutils"
9
8
  require "thread"
10
9
  require "monitor"
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: db_sucker
3
3
  version: !ruby/object:Gem::Version
4
- version: 3.0.3
4
+ version: 3.1.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Sven Pachnit
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2018-02-05 00:00:00.000000000 Z
11
+ date: 2018-04-10 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: curses