db_sucker 3.1.1 → 3.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a5629ff79f3a0b48bf58f3a19c3f3f342082d76ae896ffca04d781e19de8959d
-  data.tar.gz: 0601cd98271331cd62ffbda5f7ae0c6ecd27333bad3598229b224d5bceef586b
+  metadata.gz: b8074c1cae68f9c791d50bb5494ff86eb51bb840470bd2bf94d72cebbc2822a5
+  data.tar.gz: 8fbfadaf58d757f5bc626e23288c0056ffe4eaf1db470718784ad25c3403db3c
 SHA512:
-  metadata.gz: 2f856a743e274a0a4caf46aa614268d14e92ea40f63cc60f248d7561ddc421c260ddec37542b5a1095e8f6b1b654e8afc4616dd8191170251a48d67ba40645cf
-  data.tar.gz: 3a633a3ca67f10b3f0914b8a4b5c7fa6355d4b471a94dab1b8423135919b6b183b4accce904b2a350394e17601715dfa24410f43873e79cc839dd0367bfe3a55
+  metadata.gz: 6f5784e1d45ed9878c412de70ba0a1d3a243a38c1c6528c790a9aba1dadfcd74b223792b8060647ecf660e0703bb21da0110421c0581c6984127264129246506
+  data.tar.gz: 4ad5f0a6e3bc1466eb6e92de0d39eab511435fd06510c6e60380f9ac6b3a8eec7cc3458b18dbb73787b290f2949726897fc4feefd2918464cc48db1f4ecb42dd

data/CHANGELOG.md

@@ -1,3 +1,16 @@
+## 3.2.0
+
+### Updates
+
+* Added support for native sftp command line utility (see application option `file_transport`) but it
+  only works with non-interactive key authentication.
+
+### Fixes
+
+* Prevent application from crashing when eval produces non-StandardErrors (e.g. SyntaxErrors)
+
+-------------------
+
 ## 3.1.1
 
 ### Fixes

data/README.md

@@ -130,7 +130,7 @@ To get a list of available interface options and shortcuts press `?` or type `:h
 
 ## Configuration (application) - Ruby format
 
-DbSucker has a lot of settings and other mechanisms which you can tweak and utilize by creating a `~/.db_sucker/config.rb` file. You can change settings, add hooks or define own actions. For more information please take a look at the [documented example config](https://github.com/2called-chaos/db_sucker/blob/master/doc/config_example.rb) and/or [complete list of all settings](https://github.com/2called-chaos/db_sucker/blob/master/lib/db_sucker/application.rb#L58-L129).
+DbSucker has a lot of settings and other mechanisms which you can tweak and utilize by creating a `~/.db_sucker/config.rb` file. You can change settings, add hooks or define own actions. For more information please take a look at the [documented example config](https://github.com/2called-chaos/db_sucker/blob/master/doc/config_example.rb) and/or [complete list of all settings](https://github.com/2called-chaos/db_sucker/blob/master/lib/db_sucker/application.rb#L60-L134).
 
 
 ## Deferred import

data/VERSION

@@ -1 +1 @@
-3.1.1
+3.2.0

data/doc/config_example.rb

@@ -14,6 +14,10 @@ opts[:deferred_threshold] = 50_000_000  # 50 MB
 opts[:status_format] = :full            # used for IO operations, can be one of: none, minimal, full
 opts[:pv_enabled] = true                # disable pv utility autodiscovery (force non-usage)
 
+# Use native SFTP command for way faster transfer rates, only use with non-interactive key authentication!
+opts[:file_transport] = :ruby
+#opts[:file_transport] = :native
+
 # used to open core dumps (should be a blocking call, e.g. `subl -w' or `mate -w')
 # MUST be windowed! vim, nano, etc. will not work!
 opts[:core_dump_editor] = "subl -w"

data/lib/db_sucker/application/sklaven_treiber/worker/helpers.rb

@@ -15,6 +15,12 @@ module DbSucker
             end
           end
 
+          def sftp_native_download *args, &block
+            IO::SftpNativeDownload.new(self, *args).tap do |op|
+              block.try(:call, op)
+            end
+          end
+
           def file_copy *args, &block
             IO::FileCopy.new(self, *args).tap do |op|
               block.try(:call, op)

data/lib/db_sucker/application/sklaven_treiber/worker/io/sftp_native_download.rb

@@ -0,0 +1,79 @@
+module DbSucker
+  class Application
+    class SklavenTreiber
+      class Worker
+        module IO
+          class SftpNativeDownload < Base
+            UnknownEventError = Class.new(::RuntimeError)
+            attr_reader :downloader
+
+            def init
+              @label = "downloading"
+              @entity = "download"
+              @throughput.categories << :inet << :inet_down
+            end
+
+            def reset_state
+              super
+              @downloader = nil
+            end
+
+            def build_sftp_command src, dst
+              [].tap{|cmd|
+                cmd << %{sftp}
+                cmd << %{-P #{@ctn.source["ssh"]["port"]}} if @ctn.source["ssh"]["port"]
+                @ctn.ssh_key_files.each {|f| cmd << %{-i "#{f}"} }
+                cmd << %{"#{@ctn.source["ssh"]["username"]}@#{@ctn.source["ssh"]["hostname"]}:#{src}"}
+                cmd << %{"#{dst}"}
+              }.join(" ").strip
+            end
+
+            def download! opts = {}
+              opts = opts.reverse_merge(tries: 3, read_size: @read_size, force_new_connection: true)
+              cmd = build_sftp_command(@remote, @local)
+              prepare_local_destination
+
+              execute(opts.slice(:tries).merge(sleep_error: 3)) do
+                begin
+                  @state = :init
+                  @ctn.sftp_start(opts[:force_new_connection]) do |sftp|
+                    @filesize = sftp.lstat!(@remote).size
+                  end
+
+                  # status thread
+                  status_thread = @worker.app.spawn_thread(:sklaventreiber_worker_ctrl) do |thr|
+                    loop do
+                      @offset = File.size(@local) if File.exist?(@local)
+                      break if thr[:stop]
+                      thr.wait(0.25)
+                    end
+                  end
+
+                  @state = :downloading
+                  debug "Opening process `#{cmd}'"
+                  Open3.popen2e(cmd, pgroup: true) do |_stdin, _stdouterr, _thread|
+                    # close & exit status
+                    _stdin.close_write
+                    exit_status = _thread.value
+                    if exit_status == 0
+                      debug "Process exited (#{exit_status}) `#{cmd}'"
+                    else
+                      warning "Process exited (#{exit_status}) `#{cmd}'"
+                    end
+                  end
+
+                  status_thread[:stop] = true
+                  status_thread.signal
+                  status_thread.join
+                ensure
+                  @state = :finishing
+                end
+              end
+              @state = :done
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/db_sucker/application/sklaven_treiber/worker/routines.rb

@@ -142,11 +142,23 @@ module DbSucker
             @local_file_compressed = local_tmp_file(File.basename(@remote_file_compressed))
             @local_files_to_remove << @local_file_compressed
 
-            sftp_download(@ctn, @remote_file_compressed => @local_file_compressed) do |dl|
-              dl.status_format = app.opts[:status_format]
-              @status = [dl, "yellow"]
-              dl.abort_if { @should_cancel }
-              dl.download!
+            case app.opts[:file_transport]
+            when :ruby
+              sftp_download(@ctn, @remote_file_compressed => @local_file_compressed) do |dl|
+                dl.status_format = app.opts[:status_format]
+                @status = [dl, "yellow"]
+                dl.abort_if { @should_cancel }
+                dl.download!
+              end
+            when :native
+              sftp_native_download(@ctn, @remote_file_compressed => @local_file_compressed) do |dl|
+                dl.status_format = app.opts[:status_format]
+                @status = [dl, "yellow"]
+                dl.abort_if { @should_cancel }
+                dl.download!
+              end
+            else
+              raise UnknownFileTransportError, "Unknown file transport `#{app.opts[:file_transport]}' configured, valid are `ruby' and `native'!"
             end
           end
 

data/lib/db_sucker/application/sklaven_treiber/worker.rb

@@ -4,6 +4,7 @@ module DbSucker
       class Worker
         SlotPoolNotInitializedError = Class.new(::RuntimeError)
         ChannelFailRetryError = Class.new(::RuntimeError)
+        UnknownFileTransportError = Class.new(::RuntimeError)
 
         include Core
         include Accessors

data/lib/db_sucker/application/window/keypad/core.rb

@@ -72,7 +72,7 @@ module DbSucker
               begin
                 f.puts("#{evil}\n\n")
                 f.puts(app.sync{ app.instance_eval(evil) })
-              rescue StandardError => ex
+              rescue Exception => ex
                 f.puts("#{ex.class}: #{ex.message}")
                 ex.backtrace.each {|l| f.puts("  #{l}") }
               end

data/lib/db_sucker/application.rb

@@ -77,6 +77,12 @@ module DbSucker
         status_format: :full,            # used for IO operations, can be one of: none, minimal, full
         pv_enabled: true,                # disable pv utility autodiscovery (force non-usage)
 
+        # file transport: how to copy files from the remote
+        #   ruby       Use ruby sftp-library (why slow?)
+        #   native     Shell out to native sftp command (we try our best to build the command according to your SSH settings)
+        #              NOTE: only use with non-interactive key authentication
+        file_transport: :ruby,
+
         # sklaven treiber
         window_enabled: true,            # if disabled effectively disables any status progress or window drawing
         window_draw: true,               # wether to refresh screen or not

data/lib/db_sucker/version.rb

@@ -1,4 +1,4 @@
 module DbSucker
-  VERSION = "3.1.1"
+  VERSION = "3.2.0"
   UPDATE_URL = "https://raw.githubusercontent.com/2called-chaos/db_sucker/master/VERSION"
 end

data/lib/db_sucker.rb

@@ -32,6 +32,7 @@ require "db_sucker/application/sklaven_treiber/log_spool"
 require "db_sucker/application/sklaven_treiber/worker/io/base"
 require "db_sucker/application/sklaven_treiber/worker/io/throughput"
 require "db_sucker/application/sklaven_treiber/worker/io/sftp_download"
+require "db_sucker/application/sklaven_treiber/worker/io/sftp_native_download"
 require "db_sucker/application/sklaven_treiber/worker/io/file_copy"
 require "db_sucker/application/sklaven_treiber/worker/io/file_gunzip"
 require "db_sucker/application/sklaven_treiber/worker/io/file_shasum"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: db_sucker
 version: !ruby/object:Gem::Version
-  version: 3.1.1
+  version: 3.2.0
 platform: ruby
 authors:
 - Sven Pachnit
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-06-17 00:00:00.000000000 Z
+date: 2018-07-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: curses
@@ -175,6 +175,7 @@ files:
 - lib/db_sucker/application/sklaven_treiber/worker/io/file_shasum.rb
 - lib/db_sucker/application/sklaven_treiber/worker/io/pv_wrapper.rb
 - lib/db_sucker/application/sklaven_treiber/worker/io/sftp_download.rb
+- lib/db_sucker/application/sklaven_treiber/worker/io/sftp_native_download.rb
 - lib/db_sucker/application/sklaven_treiber/worker/io/throughput.rb
 - lib/db_sucker/application/sklaven_treiber/worker/routines.rb
 - lib/db_sucker/application/slot_pool.rb