db_sucker 3.0.3 → 3.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6bcba24bcde9938e7e815cee1a58a93d69079546dd35abc89a626ed2477c6ae9
-  data.tar.gz: 1ea0a67982883c2d30109585782ea6681baf06924ce2e5fd0b58a0cf9e2ee5cd
+  metadata.gz: 1bad32bfbe4d3e2a24def83464a9bdce5c1c7f3158f558ed405d15967adc1b4e
+  data.tar.gz: b5518954de8c6633ac22979b53f225eb7ad1e609fc191931290287479da69547
 SHA512:
-  metadata.gz: cb49a8700e37f0d00c9544b37fe8c97a7f04fd895ec9bc7bd7368f8bf79e266075f2dfd3038ace781cde60dcce2cf9347e9700fefc23e03acb0acba4ae10700e
-  data.tar.gz: a669ab21301869f1bf9d78eff4e5071faf9111fd801d4c70660e2c533054ab76f7a2585aa63691045d77a956aef005d95aafc25995bb96d03fbe9f992aeb0610
+  metadata.gz: aa025dc2bf4dac913e9e4a7295980d8030cb795cedb6edcd458d26783f8349499879bbb6520a2d09e9f1dc879e531656c7e639119fcee80e0b36df613748723c
+  data.tar.gz: 55f8415d7f31e16d6b6d8aec346c37ce8cd30112d4f5aaa516200dd2a75c92099ed31994d2691cba1e360f36807f93b638ae83a1c4fa367e02bfa8922b01d97f

data/CHANGELOG.md

@@ -1,3 +1,34 @@
+## 3.1.0
+
+### Fixes
+
+* Prevent app to run out of consumers when tasks are waiting for defer ready
+* Prevent IO errors on Ruby < 2.3 in uncompress
+* Prevent racing conditions in SSH diagnose task
+* Minor fixes
+
+-------------------
+
+## 3.0.3
+
+* no changes, fixed my tag screwup
+
+-------------------
+
+## 3.0.2
+
+* no changes, github tags can't be altered, I screwed up
+
+-------------------
+
+## 3.0.1
+
+### Fixes
+
+* hotfix for 3.0.0 release
+
+-------------------
+
 ## 3.0.0
 
 ### Updates

data/README.md

@@ -7,7 +7,8 @@ You configure your hosts via an YAML configuration in which you can define multi
 
 This tool is meant for pulling live data into your development environment. **It is not designed for backups!** but you might get away with it.
 
-![screenshot](https://i.imgur.com/EAjWrEd.png)
+[![screenshot](https://i.imgur.com/EAjWrEd.png)](https://www.youtube.com/watch?v=jdhyQzJOIkE)
+[▶ see it in action](https://www.youtube.com/watch?v=jdhyQzJOIkE)
 
 ---
 ## Alpha product (v3 is a rewrite), use at your own risk, always have a backup!
@@ -159,7 +160,6 @@ Currently there is only the "binary" importer which will use the mysql client bi
 ### General
 
 * Ruby 2.3.0 has a bug that might segfault your ruby if some exceptions occur, this is fixed since 2.3.1 and later
-* Consumers that are waiting (e.g. deferred or slot pool) won't release their tasks, if you have to few consumers you might softlock
 
 ### SSH errors / MaxSessions
 

data/VERSION

@@ -1 +1 @@
-3.0.3
+3.1.0

data/lib/db_sucker/application/core.rb

@@ -94,7 +94,7 @@ module DbSucker
           end.tap do |thr|
             # set type and priority
             thr[:itype] = type
-            thr.priority = @opts[:"tp_#{type}"]
+            thr.priority = @opts[:"tp_#{type}"] || 0
             thr.abort_on_exception = true
 
             # add signal methods

data/lib/db_sucker/application/dispatch.rb

@@ -107,42 +107,54 @@ module DbSucker
 
       # via -a/--action sshdiag
       def dispatch_sshdiag
-        log c("\nPlease wait while we run some tests...\n", :blue)
         _identifier, _ctn = false, false, false
         idstr = ARGV.shift
         varstr = ARGV.shift
 
         configful_dispatch(idstr, varstr) do |identifier, ctn, variation, var|
+          log c("\nPlease wait while we run some tests...\n", :blue)
           _identifier = identifier
           _ctn = ctn
           channels = []
+          monitor = Monitor.new
           stop = false
           maxsessions = :unknown
           begin
             t = Thread.new {
               begin
                 loop do
-                  ctn.loop_ssh(0.1)
+                  ctn.loop_ssh(0.1) { monitor.synchronize { channels.any? } }
                 end
               rescue DbSucker::Application::Container::SSH::ChannelOpenFailedError
-                maxsessions = channels.length - channels.select{|c| c[:open_failed] }.length
-                stop = true
+                monitor.synchronize do
+                  maxsessions = channels.length - channels.select{|c| c[:open_failed] }.length
+                  stop = true
+                  print "!"
+                end
                 retry
               end
             }
             250.times do
-              break if stop
+              break if monitor.synchronize { stop }
               c, r = ctn.blocking_channel_result("sleep 60", blocking: false, channel: true, use_sh: true)
-              channels << c
+              monitor.synchronize do
+                channels << c
+                print "+"
+              end
               sleep 0.1
             end
           ensure
             debug "Stopping sessions (#{channels.length})..."
-            channels.each_with_index do |c, i|
-              debug "Channel ##{i+1} #{c[:pid] ? "with PID #{c[:pid]}" : "has no PID"}"
+            i = 1
+            loop do
+              break if monitor.synchronize { channels.empty? }
+              c = monitor.synchronize { channels.shift }
+              debug "Channel ##{i} #{c[:pid] ? "with PID #{c[:pid]}" : "has no PID"}"
               ctn.kill_remote_process(c[:pid]) if c[:pid]
+              print "-"
+              i += 1
             end
-            log c("\nSSH MaxSessions: #{c maxsessions, :magenta}", :cyan)
+            log c("\n\nSSH MaxSessions: #{c maxsessions, :magenta}", :cyan)
             log "This value determines how many sessions we can multiplex over a single TCP connection."
             log "Currently, DbSucker can only utilize one connection, thus this value defines the maxmium concurrency."
             log "If you get errors you can either"

data/lib/db_sucker/application/sklaven_treiber/worker/io/file_gunzip.rb

@@ -42,7 +42,6 @@ module DbSucker
                 ensure
                   @state = :finishing
                   gz.close
-                  @in_file.close
                   @out_file.close
                 end
 

data/lib/db_sucker/application/sklaven_treiber/worker/io/throughput.rb

@@ -66,12 +66,12 @@ module DbSucker
 
             def register target
               sync do
-                if @instances[target]
+                if @instances[target.object_id]
                   raise InstanceAlreadyRegisteredError, "throughput manager cannot register more than once on the same target: `#{target}'"
                 else
                   raise NotImplementedError, "throughput manager requires the target to respond_to?(:filesize)" unless target.respond_to?(:filesize)
                   raise NotImplementedError, "throughput manager requires the target to respond_to?(:offset)" unless target.respond_to?(:offset)
-                  @instances[target] = Instance.new(self, target)
+                  @instances[target.object_id] = Instance.new(self, target)
                 end
               end
             end

data/lib/db_sucker/application/sklaven_treiber/worker/routines.rb

@@ -247,6 +247,7 @@ module DbSucker
 
           def _l_wait_for_workers
             @perform << "l_import_file_deferred"
+            Thread.main.sync { Thread.main[:summon_workers] += 1 }
             wait_defer_ready
           end
 
@@ -302,7 +303,7 @@ module DbSucker
                 end
               }
             else
-              raise ImporterNotFoundError, "variation `#{cfg.name}/#{name}' defines unknown importer `#{imp}' (in `#{cfg.src}')"
+              raise Container::Variation::ImporterNotFoundError, "variation `#{@var.cfg.name}/#{@var.name}' defines unknown importer `#{imp}' (in `#{@var.cfg.src}')"
             end
             t.join
           end

data/lib/db_sucker/application/sklaven_treiber.rb

@@ -242,8 +242,24 @@ module DbSucker
         @threads.each{|t| t[:start] = true; t.signal }
 
         # master thread (control)
+        additionals = 0
+        Thread.current[:summon_workers] = 0
         while @threads.any?(&:alive?)
           _control_thread
+          Thread.current.sync do
+            Thread.current[:summon_workers].times do
+              app.debug "Spawned additional worker due to deferred import to prevent softlocks"
+              @threads << app.spawn_thread(:sklaventreiber_worker) {|thr|
+                begin
+                  additionals += 1
+                  thr[:managed_worker] = cnum + additionals
+                  _queueoff
+                rescue Interrupt
+                end
+              }
+            end
+            Thread.current[:summon_workers] = 0
+          end
           Thread.current.wait(0.1)
         end
         @threads.each(&:join)

data/lib/db_sucker/application/window.rb

@@ -4,9 +4,6 @@ module DbSucker
       include Core
       include Curses
       COLOR_GRAY = 8
-      COL1 = 20
-      COL2 = 25
-      COL3 = 20
       OutputHelper.hook(self)
 
       attr_reader :app, :sklaventreiber, :keypad, :tick, :spinner_frames
@@ -193,7 +190,7 @@ module DbSucker
         if opts[:threads]
           next_line
           yellow "       Threads: "
-          blue "#{Thread.list.length} ".ljust(COL1, " ")
+          blue "#{Thread.list.length}".ljust(3, " ")
         end
 
         if opts[:started]
@@ -292,7 +289,7 @@ module DbSucker
 
         puts
         puts c("        Status: ") << c(sklaventreiber.status[0], sklaventreiber.status[1].presence || "red")
-        puts c("       Threads: ") << c("#{Thread.list.length} ".ljust(COL1, " "), :blue)
+        puts c("       Threads: ") << c("#{Thread.list.length} ", :blue)
         puts c("       Started: ") << c("#{@app.boot}", :blue) << c(" (") << c(human_seconds(Time.current - app.boot), :blue) << c(")")
         puts c("Transaction ID: ") << c("#{sklaventreiber.trxid}", :cyan)
         puts c("      Database: ") << c(t_db || "?", :magenta) << c(" (transferred ") << c(t_total || "?", :blue) << c(" of ") << c(t_done || "?", :blue) << c(" tables)")

data/lib/db_sucker/version.rb

@@ -1,4 +1,4 @@
 module DbSucker
-  VERSION = "3.0.3"
+  VERSION = "3.1.0"
   UPDATE_URL = "https://raw.githubusercontent.com/2called-chaos/db_sucker/master/VERSION"
 end

data/lib/db_sucker.rb

@@ -4,7 +4,6 @@ end
 
 # stdlib
 require "benchmark"
-require "optparse"
 require "fileutils"
 require "thread"
 require "monitor"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: db_sucker
 version: !ruby/object:Gem::Version
-  version: 3.0.3
+  version: 3.1.0
 platform: ruby
 authors:
 - Sven Pachnit
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-02-05 00:00:00.000000000 Z
+date: 2018-04-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: curses