pg_export 0.7.4 → 1.0.0.rc2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4053a7e2cf643fd2ddac77dca85a976f67debb6d1b1af2298d1c4822f85004bc
-  data.tar.gz: 4ddb7f36e7938d4c0219e706f29ffae86a6027ea5ab0795d4f4fed5efe250a16
+  metadata.gz: 96c2a7a23f195dba8cb7e21e3de8d9d130fd5849291e68ed153892b1f9e7aef6
+  data.tar.gz: 6f3fe11961d2d6b2089b6abacf93a1f37e48deee88d697c8402a9ac73b3dbbd3
 SHA512:
-  metadata.gz: 7ca8ae75c2ef881d39d19e7ea8414be3beafb83a840a154e1677556d07658ceaeca98227e3db0693d91831384deb766b6bca8246002bd79402f8c55f50bf61f3
-  data.tar.gz: 5bdf98eec343414e3ed78cc6756df13ec161d25f636ee392aaa26406eb2baa37a533b132f953ca79937a41e75c6d5bc185d3159ec1653472efe8e3dff1924065
+  metadata.gz: 320d541ded251f9053bffa686fb429e22572b306f48227cb9b51aaa4bae76fd452ce2ab8e8f84ac63a0b1b4628bc8c70fbfbfb26d2295d5fc7bd1c1f9188330f
+  data.tar.gz: db3b2d4799b5ddc7e0628070b8d99406655d7fc3111a34a93dc8237a189ea01ef004a72d8ed2716ad03d74b901ec597d4dc796c9e893b0557e0f698639788a48

data/.gitignore

@@ -7,3 +7,4 @@
 /pkg/
 /spec/reports/
 /tmp/
+.idea

data/.travis.yml

@@ -1,9 +1,12 @@
-sudo: false
 language: ruby
 rvm:
-  - 2.3.0
+  - 3.0.0
+  - 2.7.2
+  - 2.4.10
+services:
+  - postgresql
 addons:
   code_climate:
     repo_token: db03e5968c5bcd68b12ca50f5d41ae07dd74fe80d4e1421d754e31c316e7477a
-before_install: gem install bundler -v 1.16.6
+before_install: gem install bundler -v 2.2.3
 after_success: codeclimate-test-reporter

data/CHANGELOG.md

@@ -1,3 +1,21 @@
+### 1.0.0 - 2021.03.20
+- Make it compatible with Ruby 3.0
+- Change configuration envs:
+  - BACKUP_FTP_HOST -> PG_EXPORT_GATEWAY_HOST
+  - BACKUP_FTP_USER -> PG_EXPORT_GATEWAY_USER
+  - BACKUP_FTP_PASSWORD -> PG_EXPORT_GATEWAY_PASSWORD
+  - DUMP_ENCRYPTION_KEY -> PG_EXPORT_ENCRYPTION_KEY
+- Drop Ruby 2.3 support
+- Add SSH option
+
+### 0.7.7 - 2020.09.07
+
+- Upgrade dry-initializer 
+
+### 0.7.6 - 2020.09.05
+
+- Upgrade dry-types, dry-struct dry-system 
+
 ### 0.7.0 - 2018.10.18
 
 - Change required ruby version from 2.2.0 to 2.3.0.

data/README.md

@@ -26,7 +26,7 @@ Features:
 
 ## Dependencies
 
-  * Ruby >= 2.3.0
+  * Ruby >= 2.4 (works with Ruby 3.0)
   * $ pg_dump
   * $ pg_restore
 
@@ -51,29 +51,32 @@ Or install it yourself as:
     $ pg_export -h
 
     Usage: pg_export [options]
+        -g, --gateway GATEWAY            [Optional] ssh or ftp (default: ftp)
         -d, --database DATABASE          [Required] Name of the database to export
         -k, --keep [KEEP]                [Optional] Number of dump files to keep on FTP (default: 10)
         -t, --timestamped                [Optional] Enables log messages with timestamps
         -m, --muted                      [Optional] Mutes log messages (overrides -t option)
         -i, --interactive                [Optional] Interactive command line mode - for restoring dumps into databases
+        -v, --version                    Show version
         -h, --help                       Show this message
-    
+
     Setting can be verified by running following commands:
         -c, --configuration              Prints the configuration
-        -f, --ftp                        Tries connecting to FTP to verify the connection
+        -w, --welcome                    Tries connecting to the gateway (FTP or SSH) to verify the connection
+
 
 ## How to start
 
 __Step 1.__ Prepare ENV variables.
 
     /* FTP storage for database dumps. */
-    BACKUP_FTP_HOST=yourftp.example.com
-    BACKUP_FTP_USER=user
-    BACKUP_FTP_PASSWORD=password
+    PG_EXPORT_GATEWAY_HOST=yourftp.example.com
+    PG_EXPORT_GATEWAY_USER=user
+    PG_EXPORT_GATEWAY_PASSWORD=password
     
-    /* Encryption key shoul have exactly 16 characters. */
+    /* Encryption key should have exactly 16 characters. */
     /* Dumps will be SSL(AES-128-CBC) encrypted using this key. */
-    DUMP_ENCRYPTION_KEY=1234567890abcdef
+    PG_EXPORT_ENCRYPTION_KEY=1234567890abcdef
     
     /* Dumps to be kept on FTP */
     /* Optional, defaults to 10 */
@@ -84,12 +87,12 @@ Note, that variables cannot include `#` sign, [more info](http://serverfault.com
 __Step 2.__ Print the configuration to verify if env variables has been loaded properly.
 
     $ pg_export --configuration
-    => {:dump_encryption_key=>"k4***", :ftp_host=>"yourftp.example.com", :ftp_user=>"your_ftp_user", 
-       :ftp_password=>"pass***", :logger_format=>"plain", :keep_dumps=>2} 
+    => {:dump_encryption_key=>"k4***", :gateway_host=>"yourftp.example.com", :gateway_user=>"your_gateway_user",
+       :gateway_password=>"pass***", :logger_format=>"plain", :keep_dumps=>2}
        
 __Step 3.__ Try connecting to FTP to verify the connection.
 
-    $ pg_export --ftp
+    $ pg_export --gateway ftp --welcome
     => 230 User your_ftp_user logged in
     
 __Step 4.__ Perform database export.

data/bin/pg_export

@@ -4,65 +4,121 @@
 require 'optparse'
 
 require 'pg_export'
-require 'pg_export/container'
 
 ENV['KEEP_DUMPS'] = ENV['KEEP_DUMPS'] || '10'
+ENV['GATEWAY'] = 'ftp'
+
 interactive = false
 database = nil
 
+
 option_parser = OptionParser.new do |opts|
-  opts.banner = 'Usage: pg_export [options]'
+  opts.banner = <<~TXT
+NAME
+    pg_export - CLI for exporting/importing PostgreSQL dumps via FTP/SSH
+
+SYNOPSIS (default mode)      
+    pg_export DATABASE [options]
+
+SYNOPSIS (interactive mode)      
+    pg_export --interactive [DATABASE] [options]
+
+ARGUMENTS
+    DATABASE - database name to export (when default mode)
+             - phrase to filter database dumps by (when interactive mode)
+
+OPTIONS
+  TXT
+
+  opts.on('-g', '--gateway GATEWAY', String, 'ssh or ftp (default: ftp) - credentials need to be set via ENVs') do |g|
+    ENV['GATEWAY'] = g
+  end
+
+  opts.on('-s', '--ssh', 'Same as "--gateway ssh"') do
+    ENV['GATEWAY'] = 'ssh'
+  end
+
+  opts.on('-f', '--ftp', 'Same as "--gateway ftp"') do
+    ENV['GATEWAY'] = 'ftp'
+  end
 
-  opts.on('-d', '--database DATABASE', String, '[Required] Name of the database to export') do |d|
+  opts.on('-d', '--database DATABASE', String, 'Alternative way of specifying name of the database to export or phrase to filter by') do |d|
     database = d
   end
 
-  opts.on('-k', '--keep [KEEP]', String, "[Optional] Number of dump files to keep on FTP (default: #{ENV['KEEP_DUMPS']})") do |keep|
+  opts.on('-e', '--encryption_key KEY', String, 'Dumps will be SSL(AES-128-CBC) encrypted using this key. Should have exactly 16 characters. Overwrites PG_EXPORT_ENCRYPTION_KEY env') do |key|
+    ENV['PG_EXPORT_ENCRYPTION_KEY'] = key
+  end
+
+  opts.on('-k', '--keep KEEP', String, "Number of dump files to keep on FTP (default: 10). Overwrites KEEP_DUMPS env") do |keep|
     ENV['KEEP_DUMPS'] = keep
   end
 
-  opts.on('-t', '--timestamped', '[Optional] Enables log messages with timestamps') do
+  opts.on('-t', '--timestamped', 'Enables log messages with timestamps') do
     ENV['LOGGER_FORMAT'] = 'timestamped'
   end
 
-  opts.on('-m', '--muted', '[Optional] Mutes log messages (overrides -t option)') do
+  opts.on('-m', '--muted', 'Mutes log messages (overrides -t option)') do
     ENV['LOGGER_FORMAT'] = 'muted'
   end
 
-  opts.on('-i', '--interactive', '[Optional] Interactive command line mode - for restoring dumps into databases') do
+  opts.on('-i', '--interactive', 'Interactive mode, for importing dumps') do
     interactive = true
   end
 
+  opts.on('-v', '--version', 'Show version') do
+    puts PgExport::VERSION
+    exit
+  end
+
   opts.on('-h', '--help', 'Show this message') do
     puts opts
     exit
   end
 
-  opts.separator "\nSetting can be verified by running following commands:"
+  opts.separator <<~TXT
+
+ENV
+    PG_EXPORT_GATEWAY_HOST           required
+    PG_EXPORT_GATEWAY_USER           required
+    PG_EXPORT_GATEWAY_PASSWORD       optional when eg. authorized key is added
+    PG_EXPORT_ENCRYPTION_KEY         required or set by --encryption_key)
+  TXT
+
+  opts.separator "\nTEST RUN"
 
-  opts.on('-c', '--configuration', 'Prints the configuration') do
+  opts.on('-c', '--configuration', 'Print the configuration') do
+    require 'pg_export/container'
     PgExport::Container.start(:config)
     puts PgExport::Container['config'].to_h
     exit
   end
 
-  opts.on('-f', '--ftp', 'Tries connecting to FTP to verify the connection') do
-    PgExport::Container.start(:ftp)
-    ftp_adapter = PgExport::Container['factories.ftp_adapter_factory'].ftp_adapter
-    ftp = ftp_adapter.open_ftp
-    puts ftp.welcome
+  opts.on('-w', '--welcome', 'Try connecting to the gateway (FTP or SSH) to verify the connection') do
+    require 'pg_export/container'
+    PgExport::Container.start(ENV['GATEWAY'].to_sym)
+    gateway = PgExport::Container['factories.gateway_factory'].gateway
+    puts gateway.welcome
+    exit
+  end
+
+  if ARGV.empty?
+    puts opts
     exit
   end
 end
 
 begin
   option_parser.parse!
+  database = ARGV.first unless ARGV.empty?
 rescue OptionParser::ParseError => e
   warn e.message.capitalize
-  warn 'Type "pg_export -h" for available options'
+  warn 'Type "pg_export" for available options'
   exit
 end
 
+require 'pg_export/container'
+
 begin
   pg_export =
     if interactive
@@ -78,5 +134,5 @@ end
 
 pg_export.call(database) do |result|
   result.success { puts 'Success' }
-  result.failure { |message:| warn message }
+  result.failure { |outcome| warn outcome[:message] }
 end

data/lib/pg_export/configuration.rb

@@ -5,21 +5,21 @@ require 'dry-struct'
 
 class PgExport
   class Configuration < Dry::Struct
-    include Dry::Types.module
+    include Dry::Types()
 
     attribute :dump_encryption_key, Strict::String.constrained(size: 16)
-    attribute :ftp_host,            Strict::String
-    attribute :ftp_user,            Strict::String
-    attribute :ftp_password,        Strict::String
+    attribute :gateway_host,        Strict::String
+    attribute :gateway_user,        Strict::String
+    attribute :gateway_password,    Strict::String.optional
     attribute :logger_format,       Coercible::String.enum('plain', 'timestamped', 'muted')
     attribute :keep_dumps,          Coercible::Integer.constrained(gteq: 0)
 
     def self.build(env)
       new(
-        dump_encryption_key: env['DUMP_ENCRYPTION_KEY'],
-        ftp_host: env['BACKUP_FTP_HOST'],
-        ftp_user: env['BACKUP_FTP_USER'],
-        ftp_password: env['BACKUP_FTP_PASSWORD'],
+        dump_encryption_key: env['PG_EXPORT_ENCRYPTION_KEY'],
+        gateway_host: env['PG_EXPORT_GATEWAY_HOST'],
+        gateway_user: env['PG_EXPORT_GATEWAY_USER'],
+        gateway_password: env['PG_EXPORT_GATEWAY_PASSWORD'] == '' ? nil : env['PG_EXPORT_GATEWAY_PASSWORD'],
         logger_format: env['LOGGER_FORMAT'] || 'plain',
         keep_dumps: env['KEEP_DUMPS'] || 10
       )
@@ -27,6 +27,10 @@ class PgExport
       raise PgExport::InitializationError, e.message.gsub('[PgExport::Configuration.new] ', '')
     end
 
+    def gateway
+      ENV['GATEWAY'].to_sym
+    end
+
     def logger_muted?
       logger_format == 'muted'
     end

data/lib/pg_export/container.rb

@@ -16,32 +16,41 @@ class PgExport
 
     boot(:ftp) do
       init do
-        require 'pg_export/lib/pg_export/factories/ftp_adapter_factory'
+        require 'pg_export/lib/pg_export/factories/ftp_gateway_factory'
       end
 
       start do
         use :config
+        register('factories.gateway_factory') { ::PgExport::Factories::FtpGatewayFactory.new }
+      end
+    end
+
+    boot(:ssh) do
+      init do
+        require 'pg_export/lib/pg_export/factories/ssh_gateway_factory'
+      end
 
-        register('factories.ftp_adapter_factory') do
-          ::PgExport::Factories::FtpAdapterFactory.new
-        end
+      start do
+        use :config
+        register('factories.gateway_factory') { ::PgExport::Factories::SshGatewayFactory.new }
       end
     end
 
-    boot(:main) do
+    boot(:main) do |system|
       init do
         require 'pg_export/lib/pg_export/operations/encrypt_dump'
         require 'pg_export/lib/pg_export/operations/decrypt_dump'
         require 'pg_export/lib/pg_export/operations/remove_old_dumps_from_ftp'
-        require 'pg_export/lib/pg_export/operations/open_ftp_connection'
+        require 'pg_export/lib/pg_export/operations/open_connection'
       end
 
       start do
-        use :ftp
+        use(system[:config].gateway)
+
         register('operations.encrypt_dump') { ::PgExport::Operations::EncryptDump.new }
         register('operations.decrypt_dump') { ::PgExport::Operations::DecryptDump.new }
         register('operations.remove_old_dumps_from_ftp') { ::PgExport::Operations::RemoveOldDumpsFromFtp.new }
-        register('operations.open_ftp_connection') { ::PgExport::Operations::OpenFtpConnection.new }
+        register('operations.open_connection') { ::PgExport::Operations::OpenConnection.new }
       end
     end
   end

data/lib/pg_export/import.rb

@@ -3,5 +3,5 @@
 require_relative 'container'
 
 class PgExport
-  Import = PgExport::Container.injector.hash
+  Import = PgExport::Container.injector
 end

data/lib/pg_export/lib/pg_export/factories/ftp_gateway_factory.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+# auto_register: false
+
+require 'pg_export/lib/pg_export/gateways/ftp'
+require 'pg_export/import'
+
+class PgExport
+  module Factories
+    class FtpGatewayFactory
+      include Import['config']
+
+      def gateway
+        ::PgExport::Gateways::Ftp.new(
+          host: config.gateway_host,
+          user: config.gateway_user,
+          password: config.gateway_password
+        )
+      end
+    end
+  end
+end

data/lib/pg_export/lib/pg_export/factories/ssh_gateway_factory.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+# auto_register: false
+
+require 'pg_export/lib/pg_export/gateways/ssh'
+require 'pg_export/import'
+
+class PgExport
+  module Factories
+    class SshGatewayFactory
+      include Import['config']
+
+      def gateway
+        ::PgExport::Gateways::Ssh.new(
+          host: config.gateway_host,
+          user: config.gateway_user,
+          password: config.gateway_password
+        )
+      end
+    end
+  end
+end

data/lib/pg_export/lib/pg_export/{adapters/ftp_adapter.rb → gateways/ftp.rb}

@@ -5,28 +5,31 @@
 require 'net/ftp'
 
 class PgExport
-  module Adapters
-    class FtpAdapter
+  module Gateways
+    class Ftp
       CHUNK_SIZE = (2**16).freeze
 
       def initialize(host:, user:, password:)
         @host, @user, @password, @logger = host, user, password
-        ObjectSpace.define_finalizer(self, proc { ftp.close if @ftp })
       end
 
-      def open_ftp
+      def open
         @ftp = Net::FTP.new(host, user, password)
         @ftp.passive = true
         @ftp
       end
 
-      def close_ftp
+      def welcome
+        open.welcome
+      end
+
+      def close
         @ftp&.close
       end
 
-      def list(regex_string)
+      def list(name)
         ftp
-          .list(regex_string)
+          .list([name, '*'].join('_'))
           .map { |row| extracted_meaningful_attributes(row) }
           .sort_by { |item| item[:name] }
           .reverse
@@ -49,7 +52,7 @@ class PgExport
       end
 
       def ftp
-        @ftp ||= open_ftp
+        @ftp ||= open
       end
 
       private