webtranslateit-safe 0.4.6 → 0.4.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b87215e582012cb8e81e826fa3447c65c0b30a36940840d0e49114832c705fff
-  data.tar.gz: e93d445482019f3e864599b8df34506463edaf2b2bb864dfee4d3d0b44cb559c
+  metadata.gz: 622f6f352864fc8dc4b3938346a25a46a47874e3b439938ed3d9b177218f2564
+  data.tar.gz: fa08df6cdc398f48a2bb5f8c7fd2289bf50b9cf6c3f6aaa39b542f8143363357
 SHA512:
-  metadata.gz: '08678656aeb78fdd500dd26209ac86859517e2a828773b2ec0c9e2d9b3262b0e322c5a75618029cd49566d60a11f69861336c63101a9b42230bece81a3ffce50'
-  data.tar.gz: dcb82c4237e64d83393bc22de543dd6ba599abae7dbb0a57202dce73ef112da0e59f8a2fac7b6f1016dfcb91430fc7ada1da6cac160c7917f33992c85be0d623
+  metadata.gz: 458c754bea2197d46e29e9405705af3961a03b1fec4f2f72e994c022c5549412ff4f1baa00cbd868d36df07a0d94a2cda7007b6fb447b739ed36df0efefd63e5
+  data.tar.gz: e70216d1c71e82b4d980a264e50f411b1c392a9d0f26f60605e71a8dc633199a4e195bed14786e858ff84aa415e90a69204f45869c8ea1a7850583b781591349

data/CHANGELOG

@@ -1,3 +1,12 @@
+## 0.4.8 - 2023-07-05
+
+* Remove svndump feature.
+* Remove FTP transfer feature.
+* Merge SCP with SFTP, add `use_scp` option to SFTP to force SCP transfers.
+
+## 0.4.7 - 2023-06-30
+
+* Hotfix.
 ## 0.4.6 - 2023-06-30
 
 * Code refactoring.

data/README.markdown

@@ -1,39 +1,16 @@
 # webtranslateit-safe
 
-Simple database and filesystem backups with S3 and Rackspace Cloud Files support (with optional encryption)
+Simple database and filesystem backups with S3 and Rackspace Cloud Files support (with optional encryption).
 
-* Code: [http://github.com/webtranslateit/safe](http://github.com/webtranslateit/safe)
+This is a fork of now abandonned [astrails-safe](https://github.com/astrails/safe) that we’ve been using since 2010. It is now unmaintained and not compatible with ruby 3.2, so we forked it.
 
-## Motivation
+We’ve added:
 
-We needed a backup solution that will satisfy the following requirements:
-
-* opensource
-* simple to install and configure
-* support for simple ‘tar’ backups of directories (with includes/excludes)
-* support for simple mysqldump of mysql databases
-* support for symmetric or public key encryption
-* support for local filesystem, Amazon S3, and Rackspace Cloud Files for storage
-* support for backup rotation. we don’t want backups filling all the diskspace or cost a fortune on S3 or Cloud Files
-
-And since we didn't find any, we wrote our own :)
-
-## Contributions
-
-The following functionality was contributed by webtranslateit-safe users:
-
-* PostgreSQL dump using `pg_dump` (by Mark Mansour <mark@stateofflux.com>)
-* Subversion dump using svndump (by Richard Luther <richard.luther@gmail.com>)
-* SFTP remote storage (by Adam <adam@mediadrive.ca>)
-* benchmarking output (By Neer)
-* README fixes (by Bobby Wilson)
-* improved config file parsing (by Fedor Kocherga <fkocherga@gmail.com>)
-* mysql password file quoting (by Jonathan Sutherland <jonathan.sutherland@gmail.com>)
-* Rackspace Cloud Files support (by H. Wade Minter <minter@lunenburg.org>)
-* Plan FTP support (by seroy <seroy@bk.ru>)
-* mongodump support (by Matt Berther <matt@mattberther.com>)
-
-Thanks to all :)
+- Support for ruby 3.2
+- Standardized code with rubocop
+- Added support for SCP transfers — On some conditions with servers with higher latency, transfering large files (> 18GB) with SFTP can take a very long time.
+- Removed svndump feature
+- Removed FTP transfer feature
 
 ## Installation
 
@@ -157,13 +134,7 @@ safe do
     user "astrails"
     # port 8023
     password "ssh password for sftp"
-  end
-
-  sftp do
-    host "sftp.astrails.com"
-    user "astrails"
-    # port 8023
-    password "ssh password for scp"
+    use_scp: true # use SCP if possible
   end
 
   gpg do
@@ -201,12 +172,6 @@ safe do
 
   end
 
-  svndump do
-    repo :my_repo do
-      repo_path "/home/svn/my_repo"
-    end
-  end
-
   pgdump do
     options "-i -x -O"   # -i => ignore version, -x => do not dump privileges (grant/revoke), -O => skip restoration of object ownership in plain text format
 
@@ -237,14 +202,6 @@ safe do
 end
 ```
 
-## Contributing
-
-1. Fork it
-2. Create your feature branch (`git checkout -b my-new-feature`)
-3. Commit your changes (`git commit -am 'Add some feature'`)
-4. Push to the branch (`git push origin my-new-feature`)
-5. Create new Pull Request
-
 ## Copyright
 
 Copyright (c) 2010-2023 WebTranslateIt Software SL. See LICENSE.txt for details.

data/bin/webtranslateit-safe

@@ -1,6 +1,6 @@
 #!/usr/bin/env ruby
 
-require 'webtranslateit/safe'
+require 'web_translate_it/safe'
 
 include WebTranslateIt::Safe
 

data/lib/web_translate_it/safe/config/builder.rb

@@ -74,10 +74,10 @@ module WebTranslateIt
 
         simple_value :verbose, :dry_run, :local_only, :path, :command,
                      :options, :user, :host, :port, :password, :key, :secret, :bucket,
-                     :api_key, :container, :socket, :service_net, :repo_path
+                     :api_key, :container, :socket, :service_net, :repo_path, :use_scp
         multi_value :skip_tables, :exclude, :files
-        mixed_value :s3, :local, :cloudfiles, :sftp, :mysqldump, :tar, :gpg, :keep, :pgdump, :tar, :svndump,
-                    :ftp, :mongodump, :scp
+        mixed_value :s3, :local, :cloudfiles, :sftp, :mysqldump, :tar, :gpg, :keep, :pgdump, :tar,
+                    :mongodump
         collection :database, :archive, :repo
 
         private

data/lib/web_translate_it/safe/sftp.rb

@@ -19,7 +19,7 @@ module WebTranslateIt
       def save # rubocop:todo Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity, Metrics/MethodLength
         raise 'pipe-streaming not supported for SFTP.' unless @backup.path
 
-        puts "Uploading #{host}:#{full_path} via SFTP" if verbose? || dry_run?
+        puts "Uploading #{host}:#{full_path} via #{use_scp? ? 'SCP' : 'SFTP'}" if verbose? || dry_run?
 
         return if dry_run? || local_only?
 
@@ -27,10 +27,11 @@ module WebTranslateIt
         opts = {}
         opts[:password] = password if password
         opts[:port] = port if port
-        Net::SFTP.start(host, user, opts) do |sftp|
+        protocol = use_scp? ? Net::SCP : Net::SFTP
+        protocol.start(host, user, opts) do |connexion| # rubocop:todo Metrics/BlockLength
           puts "Sending #{@backup.path} to #{full_path}" if verbose?
           begin
-            sftp.upload! @backup.path, full_path
+            connexion.upload! @backup.path, full_path
           rescue IO::TimeoutError
             puts 'Upload timed out, retrying'
             retries += 1
@@ -39,17 +40,19 @@ module WebTranslateIt
             else
               retry unless retries >= MAX_RETRIES
             end
-          rescue Net::SFTP::StatusException
-            puts "Ensuring remote path (#{path}) exists" if verbose?
-            # mkdir -p
-            folders = path.split('/')
-            folders.each_index do |i|
-              folder = folders[0..i].join('/')
-              puts "Creating #{folder} on remote" if verbose?
-              begin
-                sftp.mkdir!(folder)
-              rescue StandardError
-                Net::SFTP::StatusException
+          rescue Net::SFTP::StatusException, Net::SCP::Error
+            Net::SFTP.start(host, user, opts) do |sftp|
+              puts "Ensuring remote path (#{path}) exists" if verbose?
+              # mkdir -p
+              folders = path.split('/')
+              folders.each_index do |i|
+                folder = folders[0..i].join('/')
+                puts "Creating #{folder} on remote" if verbose?
+                begin
+                  sftp.mkdir!(folder)
+                rescue StandardError
+                  Net::SFTP::StatusException
+                end
               end
             end
             retry
@@ -100,6 +103,10 @@ module WebTranslateIt
         config[:sftp, :port]
       end
 
+      def use_scp?
+        config[:sftp, :use_scp] == true
+      end
+
     end
 
   end

data/lib/web_translate_it/safe.rb

@@ -21,7 +21,6 @@ require 'web_translate_it/safe/source'
 require 'web_translate_it/safe/mysqldump'
 require 'web_translate_it/safe/pgdump'
 require 'web_translate_it/safe/archive'
-require 'web_translate_it/safe/svndump'
 require 'web_translate_it/safe/mongodump'
 
 require 'web_translate_it/safe/pipe'
@@ -32,9 +31,7 @@ require 'web_translate_it/safe/sink'
 require 'web_translate_it/safe/local'
 require 'web_translate_it/safe/s3'
 require 'web_translate_it/safe/cloudfiles'
-require 'web_translate_it/safe/scp'
 require 'web_translate_it/safe/sftp'
-require 'web_translate_it/safe/ftp'
 
 module WebTranslateIt
 
@@ -50,12 +47,11 @@ module WebTranslateIt
       [[Mysqldump, %i[mysqldump databases]],
        [Pgdump,    %i[pgdump databases]],
        [Mongodump, %i[mongodump databases]],
-       [Archive,   %i[tar archives]],
-       [Svndump,   %i[svndump repos]]].each do |klass, path|
+       [Archive,   %i[tar archives]]].each do |klass, path|
         next unless collection = config[*path]
 
         collection.each do |name, c|
-          klass.new(name, c).backup.run(c, :gpg, :gzip, :local, :s3, :cloudfiles, :scp, :sftp, :ftp)
+          klass.new(name, c).backup.run(c, :gpg, :gzip, :local, :s3, :cloudfiles, :sftp)
         end
       end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: webtranslateit-safe
 version: !ruby/object:Gem::Version
-  version: 0.4.6
+  version: 0.4.8
 platform: ruby
 authors:
 - Edouard Briere
@@ -9,7 +9,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-06-30 00:00:00.000000000 Z
+date: 2023-07-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-s3
@@ -68,9 +68,9 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 description: |
-  WebTranslateIt-Safe is a simple tool to backup databases (MySQL and PostgreSQL), Subversion repositories (with svndump) and just files.
+  WebTranslateIt-Safe is a simple tool to backup databases (MySQL and PostgreSQL) and just files.
   Backups can be stored locally or remotely and can be enctypted.
-  Remote storage is supported on Amazon S3, Rackspace Cloud Files, or just plain FTP/SFTP.
+  Remote storage is supported on Amazon S3, Rackspace Cloud Files, or just plain SFTP.
 email:
 - support@webtranslateit.com
 executables:
@@ -87,7 +87,6 @@ files:
 - lib/web_translate_it/safe/cloudfiles.rb
 - lib/web_translate_it/safe/config/builder.rb
 - lib/web_translate_it/safe/config/node.rb
-- lib/web_translate_it/safe/ftp.rb
 - lib/web_translate_it/safe/gpg.rb
 - lib/web_translate_it/safe/gzip.rb
 - lib/web_translate_it/safe/local.rb
@@ -96,12 +95,10 @@ files:
 - lib/web_translate_it/safe/pgdump.rb
 - lib/web_translate_it/safe/pipe.rb
 - lib/web_translate_it/safe/s3.rb
-- lib/web_translate_it/safe/scp.rb
 - lib/web_translate_it/safe/sftp.rb
 - lib/web_translate_it/safe/sink.rb
 - lib/web_translate_it/safe/source.rb
 - lib/web_translate_it/safe/stream.rb
-- lib/web_translate_it/safe/svndump.rb
 - lib/web_translate_it/safe/tmp_file.rb
 homepage: http://github.com/webtranslateit/safe
 licenses:

data/lib/web_translate_it/safe/ftp.rb

@@ -1,87 +0,0 @@
-module WebTranslateIt
-
-  module Safe
-
-    class Ftp < Sink
-
-      protected
-
-      def active?
-        host && user
-      end
-
-      def path
-        @path ||= expand(config[:ftp, :path] || config[:local, :path] || ':kind/:id')
-      end
-
-      def save # rubocop:todo Metrics/CyclomaticComplexity, Metrics/AbcSize, Metrics/MethodLength, Metrics/PerceivedComplexity
-        raise 'pipe-streaming not supported for FTP.' unless @backup.path
-
-        puts "Uploading #{host}:#{full_path} via FTP" if verbose? || dry_run?
-
-        return if dry_run? || local_only?
-
-        port ||= 21
-        Net::FTP.open(host) do |ftp|
-          ftp.connect(host, port)
-          ftp.login(user, password)
-          puts "Sending #{@backup.path} to #{full_path}" if verbose?
-          begin
-            ftp.put(@backup.path, full_path)
-          rescue Net::FTPPermError
-            puts "Ensuring remote path (#{path}) exists" if verbose?
-          end
-        end
-        puts '...done' if verbose?
-      end
-
-      # rubocop:todo Metrics/PerceivedComplexity
-      def cleanup # rubocop:todo Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity, Metrics/AbcSize, Metrics/MethodLength
-        return if local_only? || dry_run?
-
-        return unless keep = config[:keep, :ftp]
-
-        puts "listing files: #{host}:#{base}*" if verbose?
-        port ||= 21
-        Net::FTP.open(host) do |ftp|
-          ftp.connect(host, port)
-          ftp.login(user, password)
-          files = ftp.nlst(path)
-          pattern = File.basename(base.to_s)
-          files = files.select { |x| x.start_with?(pattern) }
-          puts(files.collect { |x| x }) if verbose?
-
-          files = files
-                  .collect { |x| x }
-                  .sort
-
-          cleanup_with_limit(files, keep) do |f|
-            file = File.join(path, f)
-            puts "removing ftp file #{host}:#{file}" if dry_run? || verbose?
-            ftp.delete(file) unless dry_run? || local_only?
-          end
-        end
-      end
-      # rubocop:enable Metrics/PerceivedComplexity
-
-      def host
-        config[:ftp, :host]
-      end
-
-      def user
-        config[:ftp, :user]
-      end
-
-      def password
-        config[:ftp, :password]
-      end
-
-      def port
-        config[:ftp, :port]
-      end
-
-    end
-
-  end
-
-end

data/lib/web_translate_it/safe/scp.rb

@@ -1,109 +0,0 @@
-# frozen_string_literal: true
-
-module WebTranslateIt
-
-  module Safe
-
-    class Scp < Sink
-
-      MAX_RETRIES = 5
-
-      protected
-
-      def active?
-        host && user
-      end
-
-      def path
-        @path ||= expand(config[:scp, :path] || config[:local, :path] || ':kind/:id')
-      end
-
-      def save # rubocop:todo Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity, Metrics/MethodLength
-        raise 'pipe-streaming not supported for SCP.' unless @backup.path
-
-        puts "Uploading #{host}:#{full_path} via SCP" if verbose? || dry_run?
-
-        return if dry_run? || local_only?
-
-        retries = 0
-        opts = {}
-        opts[:password] = password if password
-        opts[:port] = port if port
-        Net::SCP.start(host, user, opts) do |scp|
-          puts "Sending #{@backup.path} to #{full_path}" if verbose?
-          begin
-            scp.upload! @backup.path, full_path
-          rescue IO::TimeoutError
-            puts 'Upload timed out, retrying'
-            retries += 1
-            if retries >= MAX_RETRIES
-              puts "Tried #{retries} times. Giving up."
-            else
-              retry unless retries >= MAX_RETRIES
-            end
-          rescue Net::SCP::Error
-            puts "Ensuring remote path (#{path}) exists" if verbose?
-            # mkdir -p
-            folders = path.split('/')
-            folders.each_index do |i|
-              folder = folders[0..i].join('/')
-              puts "Creating #{folder} on remote" if verbose?
-              begin
-                scp.mkdir!(folder)
-              rescue StandardError
-                Net::SCP::Error
-              end
-            end
-            retry
-          end
-        end
-        puts '...done' if verbose?
-      end
-
-      # rubocop:todo Metrics/PerceivedComplexity
-      def cleanup # rubocop:todo Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity, Metrics/AbcSize, Metrics/MethodLength
-        return if local_only? || dry_run?
-
-        return unless (keep = config[:keep, :scp])
-
-        puts "listing files: #{host}:#{base}*" if verbose?
-        opts = {}
-        opts[:password] = password if password
-        opts[:port] = port if port
-        Net::SFTP.start(host, user, opts) do |sftp|
-          files = sftp.dir.glob(path, File.basename("#{base}*"))
-
-          puts files.collect(&:name) if verbose?
-
-          files = files.collect(&:name).sort
-
-          cleanup_with_limit(files, keep) do |f|
-            file = File.join(path, f)
-            puts "removing sftp file #{host}:#{file}" if dry_run? || verbose?
-            sftp.remove!(file) unless dry_run? || local_only?
-          end
-        end
-      end
-      # rubocop:enable Metrics/PerceivedComplexity
-
-      def host
-        config[:scp, :host]
-      end
-
-      def user
-        config[:scp, :user]
-      end
-
-      def password
-        config[:scp, :password]
-      end
-
-      def port
-        config[:scp, :port]
-      end
-
-    end
-
-  end
-
-end

data/lib/web_translate_it/safe/svndump.rb

@@ -1,17 +0,0 @@
-module WebTranslateIt
-
-  module Safe
-
-    class Svndump < Source
-
-      def command
-        "svnadmin dump #{config[:options]} #{config[:repo_path]}"
-      end
-
-      def extension = '.svn'
-
-    end
-
-  end
-
-end