pg_export 0.7.3 → 1.0.0.rc1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b37e313458127333a2dbe576aa319542438d996d10874cf545054b83a056dfc2
-  data.tar.gz: 5b64bcdc82a31242b7dd0c1676771a8942f06ce5ca884bb66bcb664001001b17
+  metadata.gz: 3eb793296ca1e2e22519a8abf1644f88fd06daa02a672d392f318a19128c50cb
+  data.tar.gz: '085a37ec1e0b7cc96f811e73a0a3271086e1c12fba3e6a2da831c6f58a4f9fde'
 SHA512:
-  metadata.gz: fcd767643cba4b7834600d7715fe76605f76f04606f9c01b4f938ff9a02e50399fa5629f931881ddc8afba117c44bb6466ec4f55aad6c900acb818347b2ab4a1
-  data.tar.gz: 4ad8e9393779a16fcf40e38acb4074df20fe134e071842d13386899ef65af58d977900dc16bd699ea3c846d229b49bf7114fab253960c7a925b4d3b8811f8dfa
+  metadata.gz: a3b89d600dc035cca7cdc909b503ac1d62e7844531d5f8a35c8213339968074ccb21d49669a1623f3d616f0aa52c2ccfb1c6c0a54f864e602ffcbe32a43c0886
+  data.tar.gz: 9a93edd9855a3a20d5745491cedbcff104cd8aee180e1c10537c63c129c8ed96933f36d5120476b85812aece2bcaba507391910e282097b5a8555e210a0e209e

data/.gitignore

@@ -7,3 +7,4 @@
 /pkg/
 /spec/reports/
 /tmp/
+.idea

data/.travis.yml

@@ -1,9 +1,12 @@
-sudo: false
 language: ruby
 rvm:
-  - 2.3.0
+  - 3.0.0
+  - 2.7.2
+  - 2.4.10
+services:
+  - postgresql
 addons:
   code_climate:
     repo_token: db03e5968c5bcd68b12ca50f5d41ae07dd74fe80d4e1421d754e31c316e7477a
-before_install: gem install bundler -v 1.16.6
+before_install: gem install bundler -v 2.2.3
 after_success: codeclimate-test-reporter

data/CHANGELOG.md

@@ -1,3 +1,20 @@
+### 1.0.0 - 2021.03.20
+- Make it compatible with Ruby 3.0
+- Change configuration envs:
+  - BACKUP_FTP_HOST -> PG_EXPORT_GATEWAY_HOST
+  - BACKUP_FTP_USER -> PG_EXPORT_GATEWAY_USER
+  - BACKUP_FTP_PASSWORD -> PG_EXPORT_GATEWAY_PASSWORD
+  - DUMP_ENCRYPTION_KEY -> PG_EXPORT_ENCRYPTION_KEY
+- Drop Ruby 2.3 support
+
+### 0.7.7 - 2020.09.07
+
+- Upgrade dry-initializer 
+
+### 0.7.6 - 2020.09.05
+
+- Upgrade dry-types, dry-struct dry-system 
+
 ### 0.7.0 - 2018.10.18
 
 - Change required ruby version from 2.2.0 to 2.3.0.

data/README.md

@@ -26,7 +26,7 @@ Features:
 
 ## Dependencies
 
-  * Ruby >= 2.3.0
+  * Ruby >= 2.4 (works with Ruby 3.0)
   * $ pg_dump
   * $ pg_restore
 
@@ -51,29 +51,32 @@ Or install it yourself as:
     $ pg_export -h
 
     Usage: pg_export [options]
+        -g, --gateway GATEWAY            [Optional] ssh or ftp (default: ftp)
         -d, --database DATABASE          [Required] Name of the database to export
         -k, --keep [KEEP]                [Optional] Number of dump files to keep on FTP (default: 10)
         -t, --timestamped                [Optional] Enables log messages with timestamps
         -m, --muted                      [Optional] Mutes log messages (overrides -t option)
         -i, --interactive                [Optional] Interactive command line mode - for restoring dumps into databases
+        -v, --version                    Show version
         -h, --help                       Show this message
-    
+
     Setting can be verified by running following commands:
         -c, --configuration              Prints the configuration
-        -f, --ftp                        Tries connecting to FTP to verify the connection
+        -w, --welcome                    Tries connecting to the gateway (FTP or SSH) to verify the connection
+
 
 ## How to start
 
 __Step 1.__ Prepare ENV variables.
 
     /* FTP storage for database dumps. */
-    BACKUP_FTP_HOST=yourftp.example.com
-    BACKUP_FTP_USER=user
-    BACKUP_FTP_PASSWORD=password
+    PG_EXPORT_GATEWAY_HOST=yourftp.example.com
+    PG_EXPORT_GATEWAY_USER=user
+    PG_EXPORT_GATEWAY_PASSWORD=password
     
     /* Encryption key shoul have exactly 16 characters. */
     /* Dumps will be SSL(AES-128-CBC) encrypted using this key. */
-    DUMP_ENCRYPTION_KEY=1234567890abcdef
+    PG_EXPORT_ENCRYPTION_KEY=1234567890abcdef
     
     /* Dumps to be kept on FTP */
     /* Optional, defaults to 10 */
@@ -84,12 +87,12 @@ Note, that variables cannot include `#` sign, [more info](http://serverfault.com
 __Step 2.__ Print the configuration to verify if env variables has been loaded properly.
 
     $ pg_export --configuration
-    => {:dump_encryption_key=>"k4***", :ftp_host=>"yourftp.example.com", :ftp_user=>"your_ftp_user", 
-       :ftp_password=>"pass***", :logger_format=>"plain", :keep_dumps=>2} 
+    => {:dump_encryption_key=>"k4***", :gateway_host=>"yourftp.example.com", :gateway_user=>"your_gateway_user",
+       :gateway_password=>"pass***", :logger_format=>"plain", :keep_dumps=>2}
        
 __Step 3.__ Try connecting to FTP to verify the connection.
 
-    $ pg_export --ftp
+    $ pg_export --gateway ftp --welcome
     => 230 User your_ftp_user logged in
     
 __Step 4.__ Perform database export.

data/bin/pg_export

@@ -4,15 +4,20 @@
 require 'optparse'
 
 require 'pg_export'
-require 'pg_export/container'
 
 ENV['KEEP_DUMPS'] = ENV['KEEP_DUMPS'] || '10'
+ENV['GATEWAY'] = 'ftp'
+
 interactive = false
 database = nil
 
 option_parser = OptionParser.new do |opts|
   opts.banner = 'Usage: pg_export [options]'
 
+  opts.on('-g', '--gateway GATEWAY', String, '[Optional] ssh or ftp (default: ftp)') do |g|
+    ENV['GATEWAY'] = g
+  end
+
   opts.on('-d', '--database DATABASE', String, '[Required] Name of the database to export') do |d|
     database = d
   end
@@ -33,6 +38,11 @@ option_parser = OptionParser.new do |opts|
     interactive = true
   end
 
+  opts.on('-v', '--version', 'Show version') do
+    puts PgExport::VERSION
+    exit
+  end
+
   opts.on('-h', '--help', 'Show this message') do
     puts opts
     exit
@@ -41,16 +51,17 @@ option_parser = OptionParser.new do |opts|
   opts.separator "\nSetting can be verified by running following commands:"
 
   opts.on('-c', '--configuration', 'Prints the configuration') do
+    require 'pg_export/container'
     PgExport::Container.start(:config)
     puts PgExport::Container['config'].to_h
     exit
   end
 
-  opts.on('-f', '--ftp', 'Tries connecting to FTP to verify the connection') do
-    PgExport::Container.start(:ftp)
-    ftp_adapter = PgExport::Container['factories.ftp_adapter_factory'].ftp_adapter
-    ftp = ftp_adapter.open_ftp
-    puts ftp.welcome
+  opts.on('-w', '--welcome', 'Tries connecting to the gateway (FTP or SSH) to verify the connection') do
+    require 'pg_export/container'
+    PgExport::Container.start(ENV['GATEWAY'].to_sym)
+    gateway = PgExport::Container['factories.gateway_factory'].gateway
+    puts gateway.welcome
     exit
   end
 end
@@ -63,6 +74,8 @@ rescue OptionParser::ParseError => e
   exit
 end
 
+require 'pg_export/container'
+
 begin
   pg_export =
     if interactive
@@ -78,5 +91,5 @@ end
 
 pg_export.call(database) do |result|
   result.success { puts 'Success' }
-  result.failure { |message:| warn message }
+  result.failure { |outcome| warn outcome[:message] }
 end

data/lib/pg_export/configuration.rb

@@ -5,21 +5,21 @@ require 'dry-struct'
 
 class PgExport
   class Configuration < Dry::Struct
-    include Dry::Types.module
+    include Dry::Types()
 
     attribute :dump_encryption_key, Strict::String.constrained(size: 16)
-    attribute :ftp_host,            Strict::String
-    attribute :ftp_user,            Strict::String
-    attribute :ftp_password,        Strict::String
+    attribute :gateway_host,        Strict::String
+    attribute :gateway_user,        Strict::String
+    attribute :gateway_password,    Strict::String.optional
     attribute :logger_format,       Coercible::String.enum('plain', 'timestamped', 'muted')
     attribute :keep_dumps,          Coercible::Integer.constrained(gteq: 0)
 
     def self.build(env)
       new(
-        dump_encryption_key: env['DUMP_ENCRYPTION_KEY'],
-        ftp_host: env['BACKUP_FTP_HOST'],
-        ftp_user: env['BACKUP_FTP_USER'],
-        ftp_password: env['BACKUP_FTP_PASSWORD'],
+        dump_encryption_key: env['PG_EXPORT_ENCRYPTION_KEY'],
+        gateway_host: env['PG_EXPORT_GATEWAY_HOST'],
+        gateway_user: env['PG_EXPORT_GATEWAY_USER'],
+        gateway_password: env['PG_EXPORT_GATEWAY_PASSWORD'] == '' ? nil : env['PG_EXPORT_GATEWAY_PASSWORD'],
         logger_format: env['LOGGER_FORMAT'] || 'plain',
         keep_dumps: env['KEEP_DUMPS'] || 10
       )
@@ -27,6 +27,10 @@ class PgExport
       raise PgExport::InitializationError, e.message.gsub('[PgExport::Configuration.new] ', '')
     end
 
+    def gateway
+      ENV['GATEWAY'].to_sym
+    end
+
     def logger_muted?
       logger_format == 'muted'
     end

data/lib/pg_export/container.rb

@@ -16,32 +16,41 @@ class PgExport
 
     boot(:ftp) do
       init do
-        require 'pg_export/lib/pg_export/factories/ftp_adapter_factory'
+        require 'pg_export/lib/pg_export/factories/ftp_gateway_factory'
       end
 
       start do
         use :config
+        register('factories.gateway_factory') { ::PgExport::Factories::FtpGatewayFactory.new }
+      end
+    end
+
+    boot(:ssh) do
+      init do
+        require 'pg_export/lib/pg_export/factories/ssh_gateway_factory'
+      end
 
-        register('factories.ftp_adapter_factory') do
-          ::PgExport::Factories::FtpAdapterFactory.new
-        end
+      start do
+        use :config
+        register('factories.gateway_factory') { ::PgExport::Factories::SshGatewayFactory.new }
       end
     end
 
-    boot(:main) do
+    boot(:main) do |system|
       init do
         require 'pg_export/lib/pg_export/operations/encrypt_dump'
         require 'pg_export/lib/pg_export/operations/decrypt_dump'
         require 'pg_export/lib/pg_export/operations/remove_old_dumps_from_ftp'
-        require 'pg_export/lib/pg_export/operations/open_ftp_connection'
+        require 'pg_export/lib/pg_export/operations/open_connection'
       end
 
       start do
-        use :ftp
+        use(system[:config].gateway)
+
         register('operations.encrypt_dump') { ::PgExport::Operations::EncryptDump.new }
         register('operations.decrypt_dump') { ::PgExport::Operations::DecryptDump.new }
         register('operations.remove_old_dumps_from_ftp') { ::PgExport::Operations::RemoveOldDumpsFromFtp.new }
-        register('operations.open_ftp_connection') { ::PgExport::Operations::OpenFtpConnection.new }
+        register('operations.open_connection') { ::PgExport::Operations::OpenConnection.new }
       end
     end
   end

data/lib/pg_export/import.rb

@@ -3,5 +3,5 @@
 require_relative 'container'
 
 class PgExport
-  Import = PgExport::Container.injector.hash
+  Import = PgExport::Container.injector
 end

data/lib/pg_export/lib/pg_export/factories/ftp_gateway_factory.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+# auto_register: false
+
+require 'pg_export/lib/pg_export/gateways/ftp'
+require 'pg_export/import'
+
+class PgExport
+  module Factories
+    class FtpGatewayFactory
+      include Import['config']
+
+      def gateway
+        ::PgExport::Gateways::Ftp.new(
+          host: config.gateway_host,
+          user: config.gateway_user,
+          password: config.gateway_password
+        )
+      end
+    end
+  end
+end

data/lib/pg_export/lib/pg_export/factories/ssh_gateway_factory.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+# auto_register: false
+
+require 'pg_export/lib/pg_export/gateways/ssh'
+require 'pg_export/import'
+
+class PgExport
+  module Factories
+    class SshGatewayFactory
+      include Import['config']
+
+      def gateway
+        ::PgExport::Gateways::Ssh.new(
+          host: config.gateway_host,
+          user: config.gateway_user,
+          password: config.gateway_password
+        )
+      end
+    end
+  end
+end

data/lib/pg_export/lib/pg_export/{adapters/ftp_adapter.rb → gateways/ftp.rb}

@@ -5,22 +5,25 @@
 require 'net/ftp'
 
 class PgExport
-  module Adapters
-    class FtpAdapter
+  module Gateways
+    class Ftp
       CHUNK_SIZE = (2**16).freeze
 
       def initialize(host:, user:, password:)
         @host, @user, @password, @logger = host, user, password
-        ObjectSpace.define_finalizer(self, proc { ftp.close if @ftp })
       end
 
-      def open_ftp
+      def open
         @ftp = Net::FTP.new(host, user, password)
         @ftp.passive = true
         @ftp
       end
 
-      def close_ftp
+      def welcome
+        open.welcome
+      end
+
+      def close
         @ftp&.close
       end
 
@@ -49,7 +52,7 @@ class PgExport
       end
 
       def ftp
-        @ftp ||= open_ftp
+        @ftp ||= open
       end
 
       private

data/lib/pg_export/lib/pg_export/gateways/ssh.rb

@@ -0,0 +1,74 @@
+# frozen_string_literal: true
+
+# auto_register: false
+
+require 'net/ssh'
+require 'net/scp'
+
+class PgExport
+  module Gateways
+    class Ssh
+      CHUNK_SIZE = (2**16).freeze
+
+      def initialize(host:, user:, password:)
+        @host, @user, @password, @logger = host, user, password
+      end
+
+      def open
+        if password.nil?
+          @ssh = Net::SSH.start(host, user)
+        else
+          @ssh = Net::SSH.start(host, user, password)
+        end
+      end
+
+      def welcome
+        open.exec!('hostname')
+      end
+
+      def close
+        @ssh&.close
+      end
+
+      def list(regex_string)
+        ssh
+          .exec!("ls -l | grep #{regex_string.gsub('*', '')}")
+          .split("\n")
+          .map { |row| extract_meaningful_attributes(row) }
+          .sort_by { |item| item[:name] }
+          .reverse
+      end
+
+      def delete(name)
+        # @TODO
+      end
+
+      def persist(file, name)
+        ssh.scp.upload(file.path, name).wait
+      end
+
+      def get(file, name)
+        ssh.scp.download(name, file.path).wait
+      end
+
+      def to_s
+        host
+      end
+
+      def ssh
+        @ssh ||= open
+      end
+
+      private
+
+      attr_reader :host, :user, :password
+
+      def extract_meaningful_attributes(item)
+        MEANINGFUL_KEYS.zip(item.split(' ').values_at(8, 4)).to_h
+      end
+
+      MEANINGFUL_KEYS = %i[name size].freeze
+      private_constant :MEANINGFUL_KEYS
+    end
+  end
+end