webtranslateit-safe 0.4.2 → 0.4.4

data/lib/webtranslateit/safe/pgdump.rb

@@ -1,17 +1,15 @@
 module WebTranslateIt
+
   module Safe
+
     class Pgdump < Source
 
       def command
-        if config['password']
-          ENV['PGPASSWORD'] = config['password']
-        else
-          ENV['PGPASSWORD'] = nil
-        end
+        ENV['PGPASSWORD'] = (config['password'] || nil)
         "pg_dump #{postgres_options} #{postgres_username} #{postgres_host} #{postgres_port} #{@id}"
       end
 
-      def extension; '.sql'; end
+      def extension = '.sql'
 
       protected
 
@@ -20,17 +18,19 @@ module WebTranslateIt
       end
 
       def postgres_host
-        config['host'] && "--host='#{config["host"]}'"
+        config['host'] && "--host='#{config['host']}'"
       end
 
       def postgres_port
-        config['port'] && "--port='#{config["port"]}'"
+        config['port'] && "--port='#{config['port']}'"
       end
 
       def postgres_username
-        config['user'] && "--username='#{config["user"]}'"
+        config['user'] && "--username='#{config['user']}'"
       end
 
     end
+
   end
+
 end

data/lib/webtranslateit/safe/pipe.rb

@@ -1,6 +1,9 @@
 module WebTranslateIt
+
   module Safe
+
     class Pipe < Stream
+
       # process adds required commands to the current
       # shell command string
       # :active?, :pipe, :extension and :post_process are
@@ -12,6 +15,9 @@ module WebTranslateIt
         @backup.extension << extension
         post_process
       end
+
     end
+
   end
+
 end

data/lib/webtranslateit/safe/s3.rb

@@ -1,7 +1,10 @@
 module WebTranslateIt
+
   module Safe
+
     class S3 < Sink
-      MAX_S3_FILE_SIZE = 5368709120
+
+      MAX_S3_FILE_SIZE = 5_368_709_120
 
       def active?
         bucket && key && secret
@@ -15,26 +18,26 @@ module WebTranslateIt
 
       def save
         # FIXME: user friendly error here :)
-        raise RuntimeError, 'pipe-streaming not supported for S3.' unless @backup.path
+        raise 'pipe-streaming not supported for S3.' unless @backup.path
 
         # needed in cleanup even on dry run
-        AWS::S3::Base.establish_connection!(:access_key_id => key, :secret_access_key => secret, :use_ssl => true) unless local_only?
+        AWS::S3::Base.establish_connection!(access_key_id: key, secret_access_key: secret, use_ssl: true) unless local_only?
 
         puts "Uploading #{bucket}:#{full_path}" if verbose? || dry_run?
-        unless dry_run? || local_only?
-          if File.stat(@backup.path).size > MAX_S3_FILE_SIZE
-            STDERR.puts "ERROR: File size exceeds maximum allowed for upload to S3 (#{MAX_S3_FILE_SIZE}): #{@backup.path}"
-            return
-          end
-          benchmark = Benchmark.realtime do
-            AWS::S3::Bucket.create(bucket) unless bucket_exists?(bucket)
-            File.open(@backup.path) do |file|
-              AWS::S3::S3Object.store(full_path, file, bucket)
-            end
+        return if dry_run? || local_only?
+
+        if File.stat(@backup.path).size > MAX_S3_FILE_SIZE
+          warn "ERROR: File size exceeds maximum allowed for upload to S3 (#{MAX_S3_FILE_SIZE}): #{@backup.path}"
+          return
+        end
+        benchmark = Benchmark.realtime do
+          AWS::S3::Bucket.create(bucket) unless bucket_exists?(bucket)
+          File.open(@backup.path) do |file|
+            AWS::S3::S3Object.store(full_path, file, bucket)
           end
-          puts '...done' if verbose?
-          puts('Upload took ' + sprintf('%.2f', benchmark) + ' second(s).') if verbose?
         end
+        puts '...done' if verbose?
+        puts("Upload took #{format('%.2f', benchmark)} second(s).") if verbose?
       end
 
       def cleanup
@@ -43,16 +46,16 @@ module WebTranslateIt
         return unless keep = config[:keep, :s3]
 
         puts "listing files: #{bucket}:#{base}*" if verbose?
-        files = AWS::S3::Bucket.objects(bucket, :prefix => base, :max_keys => keep * 2)
-        puts files.collect {|x| x.key} if verbose?
+        files = AWS::S3::Bucket.objects(bucket, prefix: base, max_keys: keep * 2)
+        puts files.collect(&:key) if verbose?
 
-        files = files.
-          collect {|x| x.key}.
-          sort
+        files = files
+                .collect(&:key)
+                .sort
 
         cleanup_with_limit(files, keep) do |f|
           puts "removing s3 file #{bucket}:#{f}" if dry_run? || verbose?
-          AWS::S3::Bucket.objects(bucket, :prefix => f)[0].delete unless dry_run? || local_only?
+          AWS::S3::Bucket.objects(bucket, prefix: f)[0].delete unless dry_run? || local_only?
         end
       end
 
@@ -69,12 +72,15 @@ module WebTranslateIt
       end
 
       private
-      
+
       def bucket_exists?(bucket)
         true if AWS::S3::Bucket.find(bucket)
       rescue AWS::S3::NoSuchBucket
         false
       end
+
     end
+
   end
+
 end

data/lib/webtranslateit/safe/sftp.rb

@@ -1,7 +1,11 @@
 module WebTranslateIt
+
   module Safe
+
     class Sftp < Sink
 
+      MAX_RETRIES = 5
+
       protected
 
       def active?
@@ -12,33 +16,46 @@ module WebTranslateIt
         @path ||= expand(config[:sftp, :path] || config[:local, :path] || ':kind/:id')
       end
 
-      def save
-        raise RuntimeError, 'pipe-streaming not supported for SFTP.' unless @backup.path
+      def save # rubocop:todo Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
+        raise 'pipe-streaming not supported for SFTP.' unless @backup.path
 
         puts "Uploading #{host}:#{full_path} via SFTP" if verbose? || dry_run?
 
-        unless dry_run? || local_only?
-          opts = {}
-          opts[:password] = password if password
-          opts[:port] = port if port
-          Net::SFTP.start(host, user, opts) do |sftp|
-            puts "Sending #{@backup.path} to #{full_path}" if verbose?
-            begin
-              sftp.upload! @backup.path, full_path
-            rescue Net::SFTP::StatusException
-              puts "Ensuring remote path (#{path}) exists" if verbose?
-              # mkdir -p
-              folders = path.split('/')
-              folders.each_index do |i|
-                folder = folders[0..i].join('/')
-                puts "Creating #{folder} on remote" if verbose?
-                sftp.mkdir!(folder) rescue Net::SFTP::StatusException
+        return if dry_run? || local_only?
+
+        retries = 0
+        opts = {}
+        opts[:password] = password if password
+        opts[:port] = port if port
+        Net::SFTP.start(host, user, opts) do |sftp|
+          puts "Sending #{@backup.path} to #{full_path}" if verbose?
+          begin
+            sftp.upload! @backup.path, full_path
+          rescue IO::TimeoutError
+            puts 'Upload timed out, retrying'
+            retries += 1
+            if retries >= MAX_RETRIES
+              puts "Tried #{retries} times. Giving up."
+            else
+              retry unless retries >= MAX_RETRIES
+            end
+          rescue Net::SFTP::StatusException
+            puts "Ensuring remote path (#{path}) exists" if verbose?
+            # mkdir -p
+            folders = path.split('/')
+            folders.each_index do |i|
+              folder = folders[0..i].join('/')
+              puts "Creating #{folder} on remote" if verbose?
+              begin
+                sftp.mkdir!(folder)
+              rescue StandardError
+                Net::SFTP::StatusException
               end
-              retry
             end
+            retry
           end
-          puts '...done' if verbose?
         end
+        puts '...done' if verbose?
       end
 
       def cleanup
@@ -53,11 +70,9 @@ module WebTranslateIt
         Net::SFTP.start(host, user, opts) do |sftp|
           files = sftp.dir.glob(path, File.basename("#{base}*"))
 
-          puts files.collect {|x| x.name } if verbose?
+          puts files.collect(&:name) if verbose?
 
-          files = files.
-            collect {|x| x.name }.
-            sort
+          files = files.collect(&:name).sort
 
           cleanup_with_limit(files, keep) do |f|
             file = File.join(path, f)
@@ -84,5 +99,7 @@ module WebTranslateIt
       end
 
     end
+
   end
-end
+
+end

data/lib/webtranslateit/safe/sink.rb

@@ -1,5 +1,7 @@
 module WebTranslateIt
+
   module Safe
+
     class Sink < Stream
 
       def process
@@ -15,7 +17,7 @@ module WebTranslateIt
       # base is used in 'cleanup' to find all files that begin with base. the '.'
       # at the end is essential to distinguish b/w foo.* and foobar.* archives for example
       def base
-        @base ||= File.join(path, File.basename(@backup.filename).split('.').first + '.')
+        @base ||= File.join(path, "#{File.basename(@backup.filename).split('.').first}.")
       end
 
       def full_path
@@ -30,6 +32,9 @@ module WebTranslateIt
         # TODO: validate here
         to_remove.each(&block)
       end
+
     end
+
   end
-end
+
+end

data/lib/webtranslateit/safe/source.rb

@@ -1,10 +1,14 @@
 module WebTranslateIt
+
   module Safe
+
     class Source < Stream
 
       attr_accessor :id
+
       def initialize(id, config)
-        @id, @config = id.to_s, config
+        @id = id.to_s
+        @config = config
       end
 
       def timestamp
@@ -21,12 +25,13 @@ module WebTranslateIt
 
       def backup
         return @backup if @backup
+
         @backup = Backup.new(
-          :id        => @id,
-          :kind      => kind,
-          :extension => extension,
-          :command   => command,
-          :timestamp => timestamp
+          id: @id,
+          kind: kind,
+          extension: extension,
+          command: command,
+          timestamp: timestamp
         )
         # can't do this in the initializer hash above since
         # filename() calls expand() which requires @backup
@@ -35,13 +40,12 @@ module WebTranslateIt
         @backup
       end
 
-      protected
-
       def self.human_name
         name.split('::').last.downcase
       end
 
     end
+
   end
-end
 
+end

data/lib/webtranslateit/safe/stream.rb

@@ -1,17 +1,22 @@
 module WebTranslateIt
+
   module Safe
+
     class Stream
 
       attr_accessor :config, :backup
+
       def initialize(config, backup)
-        @config, @backup = config, backup
+        @config = config
+        @backup = backup
       end
+
       # FIXME: move to Backup
       def expand(path)
-        path .
-        gsub(/:kind\b/, @backup.kind.to_s) .
-        gsub(/:id\b/, @backup.id.to_s) .
-        gsub(/:timestamp\b/, @backup.timestamp)
+        path
+          .gsub(/:kind\b/, @backup.kind.to_s)
+          .gsub(/:id\b/, @backup.id.to_s)
+          .gsub(/:timestamp\b/, @backup.timestamp)
       end
 
       private
@@ -27,6 +32,9 @@ module WebTranslateIt
       def dry_run?
         config[:dry_run]
       end
+
     end
+
   end
-end
+
+end

data/lib/webtranslateit/safe/svndump.rb

@@ -1,13 +1,17 @@
 module WebTranslateIt
+
   module Safe
+
     class Svndump < Source
 
       def command
         "svnadmin dump #{config[:options]} #{config[:repo_path]}"
       end
 
-      def extension; '.svn'; end
+      def extension = '.svn'
 
     end
+
   end
+
 end

data/lib/webtranslateit/safe/tmp_file.rb

@@ -1,7 +1,10 @@
 require 'tmpdir'
 module WebTranslateIt
+
   module Safe
+
     module TmpFile
+
       @keep_files = []
 
       def self.tmproot
@@ -12,22 +15,21 @@ module WebTranslateIt
         begin
           FileUtils.remove_entry_secure tmproot
         rescue ArgumentError => e
-          if e.message =~ /parent directory is world writable/
-            puts <<-ERR
+          raise unless e.message.include?('parent directory is world writable')
+
+          puts <<~ERR
 
 
-********************************************************************************
-It looks like you have wrong permissions on your TEMP directory.  The usual
-case is when you have world writable TEMP directory withOUT the sticky bit.
+            ********************************************************************************
+            It looks like you have wrong permissions on your TEMP directory.  The usual
+            case is when you have world writable TEMP directory withOUT the sticky bit.
 
-Try "chmod +t" on it.
+            Try "chmod +t" on it.
 
-********************************************************************************
+            ********************************************************************************
 
-ERR
-          else
-            raise
-          end
+          ERR
+          raise
         end
         @tmproot = nil
       end
@@ -43,6 +45,9 @@ ERR
         @keep_files << file # so that it will not get gcollected and removed from filesystem until the end
         file.path
       end
+
     end
+
   end
+
 end

data/lib/webtranslateit/safe.rb

@@ -1,5 +1,3 @@
-require 'webtranslateit/safe/version'
-
 require 'aws/s3'
 require 'cloudfiles'
 require 'net/sftp'
@@ -8,7 +6,6 @@ require 'fileutils'
 require 'benchmark'
 
 require 'tempfile'
-require 'extensions/mktmpdir'
 
 require 'webtranslateit/safe/tmp_file'
 
@@ -38,7 +35,9 @@ require 'webtranslateit/safe/sftp'
 require 'webtranslateit/safe/ftp'
 
 module WebTranslateIt
+
   module Safe
+
     ROOT = File.join(File.dirname(__FILE__), '..', '..')
 
     def safe(&block)
@@ -46,17 +45,15 @@ module WebTranslateIt
     end
 
     def process(config)
+      [[Mysqldump, %i[mysqldump databases]],
+       [Pgdump,    %i[pgdump databases]],
+       [Mongodump, %i[mongodump databases]],
+       [Archive,   %i[tar archives]],
+       [Svndump,   %i[svndump repos]]].each do |klass, path|
+        next unless collection = config[*path]
 
-      [[Mysqldump, [:mysqldump, :databases]],
-       [Pgdump,    [:pgdump,    :databases]],
-       [Mongodump, [:mongodump, :databases]],
-       [Archive,   [:tar,       :archives]],
-       [Svndump,   [:svndump,   :repos]]
-      ].each do |klass, path|
-        if collection = config[*path]
-          collection.each do |name, c|
-            klass.new(name, c).backup.run(c, :gpg, :gzip, :local, :s3, :cloudfiles, :sftp, :ftp)
-          end
+        collection.each do |name, c|
+          klass.new(name, c).backup.run(c, :gpg, :gzip, :local, :s3, :cloudfiles, :sftp, :ftp)
         end
       end
 
@@ -64,5 +61,7 @@ module WebTranslateIt
     end
     module_function :safe
     module_function :process
+
   end
+
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: webtranslateit-safe
 version: !ruby/object:Gem::Version
-  version: 0.4.2
+  version: 0.4.4
 platform: ruby
 authors:
 - Edouard Briere
@@ -9,7 +9,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-05-22 00:00:00.000000000 Z
+date: 2023-06-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-s3
@@ -64,25 +64,9 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
-- ".autotest"
-- ".document"
-- ".github/dependabot.yml"
-- ".github/release-drafter.yml"
-- ".github/workflows/ci.yml"
-- ".github/workflows/release-drafter.yml"
-- ".gitignore"
-- ".rspec"
-- ".rubocop.yml"
-- ".rubocop_todo.yml"
 - CHANGELOG
-- Gemfile
-- Gemfile.lock
-- LICENSE.txt
 - README.markdown
-- Rakefile
-- TODO
 - bin/webtranslateit-safe
-- lib/extensions/mktmpdir.rb
 - lib/webtranslateit/safe.rb
 - lib/webtranslateit/safe/archive.rb
 - lib/webtranslateit/safe/backup.rb
@@ -104,23 +88,6 @@ files:
 - lib/webtranslateit/safe/stream.rb
 - lib/webtranslateit/safe/svndump.rb
 - lib/webtranslateit/safe/tmp_file.rb
-- lib/webtranslateit/safe/version.rb
-- spec/integration/archive_integration_spec.rb
-- spec/integration/cleanup_spec.rb
-- spec/spec_helper.rb
-- spec/webtranslateit/safe/archive_spec.rb
-- spec/webtranslateit/safe/cloudfiles_spec.rb
-- spec/webtranslateit/safe/config_spec.rb
-- spec/webtranslateit/safe/gpg_spec.rb
-- spec/webtranslateit/safe/gzip_spec.rb
-- spec/webtranslateit/safe/local_spec.rb
-- spec/webtranslateit/safe/mongodump_spec.rb
-- spec/webtranslateit/safe/mysqldump_spec.rb
-- spec/webtranslateit/safe/pgdump_spec.rb
-- spec/webtranslateit/safe/s3_spec.rb
-- spec/webtranslateit/safe/svndump_spec.rb
-- templates/script.rb
-- webtranslateit-safe.gemspec
 homepage: http://github.com/webtranslateit/safe
 licenses:
 - MIT
@@ -141,7 +108,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.4.13
+rubygems_version: 3.4.14
 signing_key:
 specification_version: 4
 summary: Backup filesystem and databases (MySQL and PostgreSQL) locally or to a remote

data/.autotest

@@ -1,3 +0,0 @@
-Autotest.add_hook(:initialize) do |at|
-  at.add_mapping(%r{lib/webtranslateit/safe/config/.*\.rb$}, true) {'spec/webtranslateit/safe/config_spec.rb'}
-end

data/.document

@@ -1,5 +0,0 @@
-README.rdoc
-lib/**/*.rb
-bin/*
-features/**/*.feature
-LICENSE

data/.github/dependabot.yml

@@ -1,26 +0,0 @@
-# Please see the documentation for all configuration options:
-# https://help.github.com/github/administering-a-repository/configuration-options-for-dependency-updates
-
-version: 2
-updates:
-  - package-ecosystem: "bundler"
-    directory: "/"
-    schedule:
-      interval: "weekly"
-      day: "monday"
-      time: "04:00"
-    reviewers:
-    - edouard
-    assignees:
-    - edouard
-
-  - package-ecosystem: github-actions
-    directory: "/"
-    schedule:
-      interval: weekly
-      day: monday
-      time: "04:00"
-    reviewers:
-    - edouard
-    assignees:
-    - edouard

data/.github/release-drafter.yml

@@ -1,36 +0,0 @@
-name-template: '$RESOLVED_VERSION 🌈'
-tag-template: '$RESOLVED_VERSION'
-categories:
-  - title: '🚀 Features'
-    labels:
-      - 'feature'
-      - 'enhancement'
-  - title: '🐛 Bug Fixes'
-    labels:
-      - 'bug'
-      - 'design'
-      - 'performance'
-  - title: '🧰 Maintenance'
-    labels:
-      - 'admin'
-      - 'refactoring'
-      - 'dependencies'
-      - 'github_actions'
-      - 'tests'
-change-template: '- $TITLE @$AUTHOR (#$NUMBER)'
-change-title-escapes: '\<*_&' # You can add # and @ to disable mentions, and add ` to disable code blocks.
-version-resolver:
-  major:
-    labels:
-      - 'major'
-  minor:
-    labels:
-      - 'minor'
-  patch:
-    labels:
-      - 'patch'
-  default: patch
-template: |
-  ## Changes
-
-  $CHANGES