webtranslateit-safe 0.4.7 → 0.4.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 620a90f70005e6406e8a1d355a90e287ffb38893c1e1ab39ffca0e82707d4122
-  data.tar.gz: 52eb2a49cf8aa91cc03c0ee471282e097628487d99ee9f91203b3ed72518f22d
+  metadata.gz: 12f0e6be06240bcba9f44a010c0885c9d29459ae7299e91de82c62f13f2d37ec
+  data.tar.gz: aa10fc13934f1ccfa3680a786998973152c317d411227efe9107b7dea0ec8312
 SHA512:
-  metadata.gz: 2121d992be6eccad69dcede5b04a7c83a78f156d129d118f49a713d433e0736ead3d8e0185ea5ca9f5650f843e9e7981a619a89cbe1a1f726a84fc1ea0755ab4
-  data.tar.gz: 6df61f68e23d994300b155880425f8316744545bba20d13eac1bd1268e97dd0be2a81e7119b935db9a647daafb177398835d081241d10edcb80832ec8ce54a32
+  metadata.gz: 4c18458f0b32bc78fe6b98cc2fcbae409419da81d1dde870b8aae32a17256523a0f86d667ea5a714b5834cfbbd8653b71576e6b96d6976ae17197c17245609ad
+  data.tar.gz: 36bd512b2581664fadbb55712936ee4b106f59d4daa93873306be81cbe925744c83f780eec0e224da283987cec779837620869158834407dd247fb7f27e08347

data/CHANGELOG

@@ -1,3 +1,12 @@
+## 0.4.9 - 2023-07-06
+
+* Use pigz instead of gzip when available on system to speed up compression time. 
+## 0.4.8 - 2023-07-05
+
+* Remove svndump feature.
+* Remove FTP transfer feature.
+* Merge SCP with SFTP, add `use_scp` option to SFTP to force SCP transfers.
+
 ## 0.4.7 - 2023-06-30
 
 * Hotfix.

data/README.markdown

@@ -9,6 +9,8 @@ We’ve added:
 - Support for ruby 3.2
 - Standardized code with rubocop
 - Added support for SCP transfers — On some conditions with servers with higher latency, transfering large files (> 18GB) with SFTP can take a very long time.
+- Removed svndump feature
+- Removed FTP transfer feature
 
 ## Installation
 
@@ -132,13 +134,7 @@ safe do
     user "astrails"
     # port 8023
     password "ssh password for sftp"
-  end
-
-  sftp do
-    host "sftp.astrails.com"
-    user "astrails"
-    # port 8023
-    password "ssh password for scp"
+    use_scp: true # use SCP if possible
   end
 
   gpg do
@@ -156,7 +152,6 @@ safe do
     s3 100
     cloudfiles 100
     sftp 100
-    scp 100
   end
 
   mysqldump do
@@ -176,12 +171,6 @@ safe do
 
   end
 
-  svndump do
-    repo :my_repo do
-      repo_path "/home/svn/my_repo"
-    end
-  end
-
   pgdump do
     options "-i -x -O"   # -i => ignore version, -x => do not dump privileges (grant/revoke), -O => skip restoration of object ownership in plain text format
 

data/lib/web_translate_it/safe/config/builder.rb

@@ -74,10 +74,10 @@ module WebTranslateIt
 
         simple_value :verbose, :dry_run, :local_only, :path, :command,
                      :options, :user, :host, :port, :password, :key, :secret, :bucket,
-                     :api_key, :container, :socket, :service_net, :repo_path
+                     :api_key, :container, :socket, :service_net, :repo_path, :use_scp
         multi_value :skip_tables, :exclude, :files
-        mixed_value :s3, :local, :cloudfiles, :sftp, :mysqldump, :tar, :gpg, :keep, :pgdump, :tar, :svndump,
-                    :ftp, :mongodump, :scp
+        mixed_value :s3, :local, :cloudfiles, :sftp, :mysqldump, :tar, :gpg, :keep, :pgdump, :tar,
+                    :mongodump
         collection :database, :archive, :repo
 
         private

data/lib/web_translate_it/safe/pigz.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module WebTranslateIt
+
+  module Safe
+
+    class Pigz < Pipe
+
+      protected
+
+      def post_process
+        @backup.compressed = true
+      end
+
+      def pipe
+        '|pigz'
+      end
+
+      def extension
+        '.gz'
+      end
+
+      def active?
+        !@backup.compressed && !(find_executable 'pigz').nil?
+      end
+
+    end
+
+  end
+
+end

data/lib/web_translate_it/safe/sftp.rb

@@ -19,7 +19,7 @@ module WebTranslateIt
       def save # rubocop:todo Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity, Metrics/MethodLength
         raise 'pipe-streaming not supported for SFTP.' unless @backup.path
 
-        puts "Uploading #{host}:#{full_path} via SFTP" if verbose? || dry_run?
+        puts "Uploading #{host}:#{full_path} via #{use_scp? ? 'SCP' : 'SFTP'}" if verbose? || dry_run?
 
         return if dry_run? || local_only?
 
@@ -27,10 +27,11 @@ module WebTranslateIt
         opts = {}
         opts[:password] = password if password
         opts[:port] = port if port
-        Net::SFTP.start(host, user, opts) do |sftp|
+        protocol = use_scp? ? Net::SCP : Net::SFTP
+        protocol.start(host, user, opts) do |connexion| # rubocop:todo Metrics/BlockLength
           puts "Sending #{@backup.path} to #{full_path}" if verbose?
           begin
-            sftp.upload! @backup.path, full_path
+            connexion.upload! @backup.path, full_path
           rescue IO::TimeoutError
             puts 'Upload timed out, retrying'
             retries += 1
@@ -39,17 +40,19 @@ module WebTranslateIt
             else
               retry unless retries >= MAX_RETRIES
             end
-          rescue Net::SFTP::StatusException
-            puts "Ensuring remote path (#{path}) exists" if verbose?
-            # mkdir -p
-            folders = path.split('/')
-            folders.each_index do |i|
-              folder = folders[0..i].join('/')
-              puts "Creating #{folder} on remote" if verbose?
-              begin
-                sftp.mkdir!(folder)
-              rescue StandardError
-                Net::SFTP::StatusException
+          rescue Net::SFTP::StatusException, Net::SCP::Error
+            Net::SFTP.start(host, user, opts) do |sftp|
+              puts "Ensuring remote path (#{path}) exists" if verbose?
+              # mkdir -p
+              folders = path.split('/')
+              folders.each_index do |i|
+                folder = folders[0..i].join('/')
+                puts "Creating #{folder} on remote" if verbose?
+                begin
+                  sftp.mkdir!(folder)
+                rescue StandardError
+                  Net::SFTP::StatusException
+                end
               end
             end
             retry
@@ -100,6 +103,10 @@ module WebTranslateIt
         config[:sftp, :port]
       end
 
+      def use_scp?
+        config[:sftp, :use_scp] == true
+      end
+
     end
 
   end

data/lib/web_translate_it/safe.rb

@@ -5,8 +5,8 @@ require 'net/scp'
 require 'tmpdir'
 require 'fileutils'
 require 'benchmark'
-
 require 'tempfile'
+require 'mkmf'
 
 require 'web_translate_it/safe/tmp_file'
 
@@ -21,20 +21,23 @@ require 'web_translate_it/safe/source'
 require 'web_translate_it/safe/mysqldump'
 require 'web_translate_it/safe/pgdump'
 require 'web_translate_it/safe/archive'
-require 'web_translate_it/safe/svndump'
 require 'web_translate_it/safe/mongodump'
 
 require 'web_translate_it/safe/pipe'
 require 'web_translate_it/safe/gpg'
 require 'web_translate_it/safe/gzip'
+require 'web_translate_it/safe/pigz'
 
 require 'web_translate_it/safe/sink'
 require 'web_translate_it/safe/local'
 require 'web_translate_it/safe/s3'
 require 'web_translate_it/safe/cloudfiles'
-require 'web_translate_it/safe/scp'
 require 'web_translate_it/safe/sftp'
-require 'web_translate_it/safe/ftp'
+
+# Keeps mkmf from littering STDOUT
+MakeMakefile::Logging.quiet = true
+# Keeps mkmf from creating a mkmf.log file on current path
+MakeMakefile::Logging.logfile(File::NULL)
 
 module WebTranslateIt
 
@@ -50,12 +53,11 @@ module WebTranslateIt
       [[Mysqldump, %i[mysqldump databases]],
        [Pgdump,    %i[pgdump databases]],
        [Mongodump, %i[mongodump databases]],
-       [Archive,   %i[tar archives]],
-       [Svndump,   %i[svndump repos]]].each do |klass, path|
+       [Archive,   %i[tar archives]]].each do |klass, path|
         next unless collection = config[*path]
 
         collection.each do |name, c|
-          klass.new(name, c).backup.run(c, :gpg, :gzip, :local, :s3, :cloudfiles, :scp, :sftp, :ftp)
+          klass.new(name, c).backup.run(c, :gpg, :pigz, :gzip, :local, :s3, :cloudfiles, :sftp)
         end
       end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: webtranslateit-safe
 version: !ruby/object:Gem::Version
-  version: 0.4.7
+  version: 0.4.9
 platform: ruby
 authors:
 - Edouard Briere
@@ -9,7 +9,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-06-30 00:00:00.000000000 Z
+date: 2023-07-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-s3
@@ -68,9 +68,9 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 description: |
-  WebTranslateIt-Safe is a simple tool to backup databases (MySQL and PostgreSQL), Subversion repositories (with svndump) and just files.
-  Backups can be stored locally or remotely and can be enctypted.
-  Remote storage is supported on Amazon S3, Rackspace Cloud Files, or just plain FTP/SFTP.
+  WebTranslateIt-Safe is a simple tool to backup databases (MySQL and PostgreSQL) and files.
+  Backups can be stored locally or remotely and can be encrypted.
+  Remote storage is supported on Amazon S3, Rackspace Cloud Files, or just plain SFTP/SCP.
 email:
 - support@webtranslateit.com
 executables:
@@ -87,21 +87,19 @@ files:
 - lib/web_translate_it/safe/cloudfiles.rb
 - lib/web_translate_it/safe/config/builder.rb
 - lib/web_translate_it/safe/config/node.rb
-- lib/web_translate_it/safe/ftp.rb
 - lib/web_translate_it/safe/gpg.rb
 - lib/web_translate_it/safe/gzip.rb
 - lib/web_translate_it/safe/local.rb
 - lib/web_translate_it/safe/mongodump.rb
 - lib/web_translate_it/safe/mysqldump.rb
 - lib/web_translate_it/safe/pgdump.rb
+- lib/web_translate_it/safe/pigz.rb
 - lib/web_translate_it/safe/pipe.rb
 - lib/web_translate_it/safe/s3.rb
-- lib/web_translate_it/safe/scp.rb
 - lib/web_translate_it/safe/sftp.rb
 - lib/web_translate_it/safe/sink.rb
 - lib/web_translate_it/safe/source.rb
 - lib/web_translate_it/safe/stream.rb
-- lib/web_translate_it/safe/svndump.rb
 - lib/web_translate_it/safe/tmp_file.rb
 homepage: http://github.com/webtranslateit/safe
 licenses:

data/lib/web_translate_it/safe/ftp.rb

@@ -1,87 +0,0 @@
-module WebTranslateIt
-
-  module Safe
-
-    class Ftp < Sink
-
-      protected
-
-      def active?
-        host && user
-      end
-
-      def path
-        @path ||= expand(config[:ftp, :path] || config[:local, :path] || ':kind/:id')
-      end
-
-      def save # rubocop:todo Metrics/CyclomaticComplexity, Metrics/AbcSize, Metrics/MethodLength, Metrics/PerceivedComplexity
-        raise 'pipe-streaming not supported for FTP.' unless @backup.path
-
-        puts "Uploading #{host}:#{full_path} via FTP" if verbose? || dry_run?
-
-        return if dry_run? || local_only?
-
-        port ||= 21
-        Net::FTP.open(host) do |ftp|
-          ftp.connect(host, port)
-          ftp.login(user, password)
-          puts "Sending #{@backup.path} to #{full_path}" if verbose?
-          begin
-            ftp.put(@backup.path, full_path)
-          rescue Net::FTPPermError
-            puts "Ensuring remote path (#{path}) exists" if verbose?
-          end
-        end
-        puts '...done' if verbose?
-      end
-
-      # rubocop:todo Metrics/PerceivedComplexity
-      def cleanup # rubocop:todo Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity, Metrics/AbcSize, Metrics/MethodLength
-        return if local_only? || dry_run?
-
-        return unless keep = config[:keep, :ftp]
-
-        puts "listing files: #{host}:#{base}*" if verbose?
-        port ||= 21
-        Net::FTP.open(host) do |ftp|
-          ftp.connect(host, port)
-          ftp.login(user, password)
-          files = ftp.nlst(path)
-          pattern = File.basename(base.to_s)
-          files = files.select { |x| x.start_with?(pattern) }
-          puts(files.collect { |x| x }) if verbose?
-
-          files = files
-                  .collect { |x| x }
-                  .sort
-
-          cleanup_with_limit(files, keep) do |f|
-            file = File.join(path, f)
-            puts "removing ftp file #{host}:#{file}" if dry_run? || verbose?
-            ftp.delete(file) unless dry_run? || local_only?
-          end
-        end
-      end
-      # rubocop:enable Metrics/PerceivedComplexity
-
-      def host
-        config[:ftp, :host]
-      end
-
-      def user
-        config[:ftp, :user]
-      end
-
-      def password
-        config[:ftp, :password]
-      end
-
-      def port
-        config[:ftp, :port]
-      end
-
-    end
-
-  end
-
-end

data/lib/web_translate_it/safe/scp.rb

@@ -1,109 +0,0 @@
-# frozen_string_literal: true
-
-module WebTranslateIt
-
-  module Safe
-
-    class Scp < Sink
-
-      MAX_RETRIES = 5
-
-      protected
-
-      def active?
-        host && user
-      end
-
-      def path
-        @path ||= expand(config[:scp, :path] || config[:local, :path] || ':kind/:id')
-      end
-
-      def save # rubocop:todo Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity, Metrics/MethodLength
-        raise 'pipe-streaming not supported for SCP.' unless @backup.path
-
-        puts "Uploading #{host}:#{full_path} via SCP" if verbose? || dry_run?
-
-        return if dry_run? || local_only?
-
-        retries = 0
-        opts = {}
-        opts[:password] = password if password
-        opts[:port] = port if port
-        Net::SCP.start(host, user, opts) do |scp|
-          puts "Sending #{@backup.path} to #{full_path}" if verbose?
-          begin
-            scp.upload! @backup.path, full_path
-          rescue IO::TimeoutError
-            puts 'Upload timed out, retrying'
-            retries += 1
-            if retries >= MAX_RETRIES
-              puts "Tried #{retries} times. Giving up."
-            else
-              retry unless retries >= MAX_RETRIES
-            end
-          rescue Net::SCP::Error
-            puts "Ensuring remote path (#{path}) exists" if verbose?
-            # mkdir -p
-            folders = path.split('/')
-            folders.each_index do |i|
-              folder = folders[0..i].join('/')
-              puts "Creating #{folder} on remote" if verbose?
-              begin
-                scp.mkdir!(folder)
-              rescue StandardError
-                Net::SCP::Error
-              end
-            end
-            retry
-          end
-        end
-        puts '...done' if verbose?
-      end
-
-      # rubocop:todo Metrics/PerceivedComplexity
-      def cleanup # rubocop:todo Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity, Metrics/AbcSize, Metrics/MethodLength
-        return if local_only? || dry_run?
-
-        return unless (keep = config[:keep, :scp])
-
-        puts "listing files: #{host}:#{base}*" if verbose?
-        opts = {}
-        opts[:password] = password if password
-        opts[:port] = port if port
-        Net::SFTP.start(host, user, opts) do |sftp|
-          files = sftp.dir.glob(path, File.basename("#{base}*"))
-
-          puts files.collect(&:name) if verbose?
-
-          files = files.collect(&:name).sort
-
-          cleanup_with_limit(files, keep) do |f|
-            file = File.join(path, f)
-            puts "removing sftp file #{host}:#{file}" if dry_run? || verbose?
-            sftp.remove!(file) unless dry_run? || local_only?
-          end
-        end
-      end
-      # rubocop:enable Metrics/PerceivedComplexity
-
-      def host
-        config[:scp, :host]
-      end
-
-      def user
-        config[:scp, :user]
-      end
-
-      def password
-        config[:scp, :password]
-      end
-
-      def port
-        config[:scp, :port]
-      end
-
-    end
-
-  end
-
-end

data/lib/web_translate_it/safe/svndump.rb

@@ -1,17 +0,0 @@
-module WebTranslateIt
-
-  module Safe
-
-    class Svndump < Source
-
-      def command
-        "svnadmin dump #{config[:options]} #{config[:repo_path]}"
-      end
-
-      def extension = '.svn'
-
-    end
-
-  end
-
-end