webtranslateit-safe 0.4.4 → 0.4.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1d56d9c1814184138e0ac2f4c7c1c62d977db103d6aecebd441ae6c861a4f85e
-  data.tar.gz: 818c66a0f9307e18add81ef25fd29822a77ae1980cbbbdeafbcfdd5684ae65b7
+  metadata.gz: 63bd94e905a8a424ce2b408fa3d0d2898fc5d22c3be9739a95ff73668cd612ad
+  data.tar.gz: 2f33a476839a38aa403e89a3f0e35ad5c78a9c403756c0472f15c45337a0b575
 SHA512:
-  metadata.gz: 10a974ca460fdfd6ccf52f19a55466b70f74ea2f7e929084935453cf04bc67f0c9618c44f7b9d70bca2fd23f7af84ee54e77d2161a0a7219df3800c73e466e77
-  data.tar.gz: 6975cc66f99b3220b69556a20e5646c51ace16c7544380a1459f61cd9ce2781b8f5923dd942088676390cb50701daed4b964898562bf001edb4b9da590874ebd
+  metadata.gz: b50f77cc5ee6b609ad05e85fa5fc106ed21af49e014397427aa9a77de4bba9e5d6e3cff8773d78e14314e44388d6f0cf6cbf258d0e3ffd9db0ddeedc063a525d
+  data.tar.gz: e21c75b914bf576842f22893098c872659ba5d7dd27b84d4f087226e7aa80a43f32a133f668a20c09492762f2459707ae1c3e7f9e074db4a8b9fc7c933b4f885

data/CHANGELOG

@@ -1,3 +1,7 @@
+## 0.4.5 - 2023-06-30
+
+* Add SCP backup strategy.
+
 ## 0.4.4 - 2023-06-30
 
 * Remove `default_executable` line and simplify executables definition.

data/README.markdown

@@ -155,6 +155,13 @@ The procedure to create and transfer the key is as follows:
         password "ssh password for sftp"
       end
 
+      sftp do
+        host "sftp.astrails.com"
+        user "astrails"
+        # port 8023
+        password "ssh password for scp"
+      end
+
       gpg do
         command "/usr/local/bin/gpg"
         options  "--no-use-agent"
@@ -170,6 +177,7 @@ The procedure to create and transfer the key is as follows:
         s3 100
         cloudfiles 100
         sftp 100
+        scp 100
       end
 
       mysqldump do

data/lib/webtranslateit/safe/config/builder.rb

@@ -77,7 +77,7 @@ module WebTranslateIt
                      :api_key, :container, :socket, :service_net, :repo_path
         multi_value :skip_tables, :exclude, :files
         mixed_value :s3, :local, :cloudfiles, :sftp, :mysqldump, :tar, :gpg, :keep, :pgdump, :tar, :svndump,
-                    :ftp, :mongodump
+                    :ftp, :mongodump, :scp
         collection :database, :archive, :repo
 
         private

data/lib/webtranslateit/safe/scp.rb

@@ -0,0 +1,107 @@
+# frozen_string_literal: true
+
+module WebTranslateIt
+
+  module Safe
+
+    class Scp < Sink
+
+      MAX_RETRIES = 5
+
+      protected
+
+      def active?
+        host && user
+      end
+
+      def path
+        @path ||= expand(config[:scp, :path] || config[:local, :path] || ':kind/:id')
+      end
+
+      def save # rubocop:todo Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
+        raise 'pipe-streaming not supported for SCP.' unless @backup.path
+
+        puts "Uploading #{host}:#{full_path} via SCP" if verbose? || dry_run?
+
+        return if dry_run? || local_only?
+
+        retries = 0
+        opts = {}
+        opts[:password] = password if password
+        opts[:port] = port if port
+        Net::SCP.start(host, user, opts) do |scp|
+          puts "Sending #{@backup.path} to #{full_path}" if verbose?
+          begin
+            scp.upload! @backup.path, full_path
+          rescue IO::TimeoutError
+            puts 'Upload timed out, retrying'
+            retries += 1
+            if retries >= MAX_RETRIES
+              puts "Tried #{retries} times. Giving up."
+            else
+              retry unless retries >= MAX_RETRIES
+            end
+          rescue Net::SCP::Error
+            puts "Ensuring remote path (#{path}) exists" if verbose?
+            # mkdir -p
+            folders = path.split('/')
+            folders.each_index do |i|
+              folder = folders[0..i].join('/')
+              puts "Creating #{folder} on remote" if verbose?
+              begin
+                scp.mkdir!(folder)
+              rescue StandardError
+                Net::SCP::Error
+              end
+            end
+            retry
+          end
+        end
+        puts '...done' if verbose?
+      end
+
+      def cleanup
+        return if local_only? || dry_run?
+
+        return unless (keep = config[:keep, :scp])
+
+        puts "listing files: #{host}:#{base}*" if verbose?
+        opts = {}
+        opts[:password] = password if password
+        opts[:port] = port if port
+        Net::SFTP.start(host, user, opts) do |sftp|
+          files = sftp.dir.glob(path, File.basename("#{base}*"))
+
+          puts files.collect(&:name) if verbose?
+
+          files = files.collect(&:name).sort
+
+          cleanup_with_limit(files, keep) do |f|
+            file = File.join(path, f)
+            puts "removing sftp file #{host}:#{file}" if dry_run? || verbose?
+            sftp.remove!(file) unless dry_run? || local_only?
+          end
+        end
+      end
+
+      def host
+        config[:scp, :host]
+      end
+
+      def user
+        config[:scp, :user]
+      end
+
+      def password
+        config[:scp, :password]
+      end
+
+      def port
+        config[:scp, :port]
+      end
+
+    end
+
+  end
+
+end

data/lib/webtranslateit/safe.rb

@@ -1,6 +1,7 @@
 require 'aws/s3'
 require 'cloudfiles'
 require 'net/sftp'
+require 'net/scp'
 # require 'net/ftp'
 require 'fileutils'
 require 'benchmark'
@@ -31,6 +32,7 @@ require 'webtranslateit/safe/sink'
 require 'webtranslateit/safe/local'
 require 'webtranslateit/safe/s3'
 require 'webtranslateit/safe/cloudfiles'
+require 'webtranslateit/safe/scp'
 require 'webtranslateit/safe/sftp'
 require 'webtranslateit/safe/ftp'
 
@@ -53,7 +55,7 @@ module WebTranslateIt
         next unless collection = config[*path]
 
         collection.each do |name, c|
-          klass.new(name, c).backup.run(c, :gpg, :gzip, :local, :s3, :cloudfiles, :sftp, :ftp)
+          klass.new(name, c).backup.run(c, :gpg, :gzip, :local, :s3, :cloudfiles, :scp, :sftp, :ftp)
         end
       end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: webtranslateit-safe
 version: !ruby/object:Gem::Version
-  version: 0.4.4
+  version: 0.4.5
 platform: ruby
 authors:
 - Edouard Briere
@@ -39,6 +39,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: net-scp
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: net-sftp
   requirement: !ruby/object:Gem::Requirement
@@ -82,6 +96,7 @@ files:
 - lib/webtranslateit/safe/pgdump.rb
 - lib/webtranslateit/safe/pipe.rb
 - lib/webtranslateit/safe/s3.rb
+- lib/webtranslateit/safe/scp.rb
 - lib/webtranslateit/safe/sftp.rb
 - lib/webtranslateit/safe/sink.rb
 - lib/webtranslateit/safe/source.rb