pg_export 0.6.1 → 0.7.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: c9d176bb30786ee0c9e6995d2d1875607463188a
-  data.tar.gz: 0866aa29b2ebf8945568aab3c3aa102fd4a6e1e2
+SHA256:
+  metadata.gz: e6e920a09c0e04dd2b2e2eb612db083a390cbb74fae09bc1b5cfa510bc74751a
+  data.tar.gz: d23c28f594f37ac01bd586668bb62b8641ce56b9b1c5f4ff05a528dc1e2a69b2
 SHA512:
-  metadata.gz: 742ff5d0198a1fcca3a9b4f351dbcc9c975f0e07307bebdff30ea657825668dc04fee09e16e9e516d1cbd4f5a6505a4d3078ce1eca0d4ef00556ceb6d22f6c31
-  data.tar.gz: 53c17e1c8ca81d967ab1d3b4b09f13463afd4657151554dc517db2e9ff501bb580b3e34fad497886fd3e17dbaf35a45c8d6ec0044f14450fa314e36c01bf142a
+  metadata.gz: 92a31d0fa7d86a759060cf0f404fcc35e6d14a6f8043b66c5782e3b69646c2a1a5708276a88697d7f8e40fc85ae5a464ad98129e89e5b9f6bc61f6dfb72039c4
+  data.tar.gz: 4b6d1c3a9889a6b7b5a494605e300684680fa21d40ed1f79f9ae64356555f4fea6e5b4245b835e3e705d1c9f53459117ba672ebe70db4306e01f61e80cae9605

data/.rubocop.yml

@@ -1,26 +1,22 @@
 AllCops:
-  TargetRubyVersion: 2.1
-  Exclude:
-    - 'spec/spec_helper.rb'
+  TargetRubyVersion: 2.3
 
-Style/BlockLength:
+Metrics/BlockLength:
   Exclude:
     - 'bin/pg_export'
-    - 'spec/**/*.rb'
+  ExcludedMethods: ['describe', 'context']
 
 Metrics/LineLength:
   Max: 200
 
-Lint/AmbiguousOperator:
-  Exclude:
-    - 'lib/pg_export/configuration.rb'
-
-Style/AlignArray:
-  Exclude:
-    - 'spec/configuration_spec.rb'
-
 Style/ParallelAssignment:
   Enabled: false
 
+Naming/UncommunicativeMethodParamName:
+  Enabled: false
+
 Documentation:
   Enabled: false
+
+Lint/UnusedMethodArgument:
+  Enabled: false

data/.travis.yml

@@ -1,9 +1,9 @@
 sudo: false
 language: ruby
 rvm:
-  - 2.2.0
+  - 2.3.0
 addons:
   code_climate:
     repo_token: db03e5968c5bcd68b12ca50f5d41ae07dd74fe80d4e1421d754e31c316e7477a
-before_install: gem install bundler -v 1.13.3
+before_install: gem install bundler -v 1.16.6
 after_success: codeclimate-test-reporter

data/CHANGELOG.md

@@ -1,3 +1,12 @@
+### 0.7.0 - 2018.10.18
+
+- Change required ruby version from 2.2.0 to 2.3.0.
+- Refactor architecture to be more functional using dry-system
+- Use tty for interactive CLI (instead of cli_spinnable)
+- Improve configuration parsing
+- Add -m option for muting log messages
+- Make log messages more verbose
+
 ### 0.6.1 - 2017.08.20
 
 - Change required ruby version from 2.1.0 to 2.2.0.

data/Gemfile

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 source 'https://rubygems.org'
 
 # Specify your gem's dependencies in pg_export.gemspec

data/README.md

@@ -26,7 +26,7 @@ Features:
 
 ## Dependencies
 
-  * Ruby >= 2.2.0
+  * Ruby >= 2.3.0
   * $ pg_dump
   * $ pg_restore
 
@@ -54,7 +54,8 @@ Or install it yourself as:
         -d, --database DATABASE          [Required] Name of the database to export
         -k, --keep [KEEP]                [Optional] Number of dump files to keep on FTP (default: 10)
         -t, --timestamped                [Optional] Enables log messages with timestamps
-        -i, --interactive                Interactive, command line mode, for restoring dumps into databases
+        -m, --muted                      [Optional] Mutes log messages (overrides -t option)
+        -i, --interactive                [Optional] Interactive command line mode - for restoring dumps into databases
         -h, --help                       Show this message
     
     Setting can be verified by running following commands:
@@ -63,50 +64,48 @@ Or install it yourself as:
 
 ## How to start
 
-__Step 1.__ Prepare FTP account and put configuration into env variables. Dumps will be exported into that location.
+__Step 1.__ Prepare ENV variables.
 
-    # /etc/environment
-    BACKUP_FTP_HOST="yourftp.example.com"
-    BACKUP_FTP_USER="user"
-    BACKUP_FTP_PASSWORD="password"
+    /* FTP storage for database dumps. */
+    BACKUP_FTP_HOST=yourftp.example.com
+    BACKUP_FTP_USER=user
+    BACKUP_FTP_PASSWORD=password
     
-__Step 2.__ Put dump encryption key into env variable (exactly 16 characters). Dumps will be SSL(AES-128-CBC) encrypted using that key.
- 
-    # /etc/environment
-    DUMP_ENCRYPTION_KEY="1234567890abcdef"
+    /* Encryption key shoul have exactly 16 characters. */
+    /* Dumps will be SSL(AES-128-CBC) encrypted using this key. */
+    DUMP_ENCRYPTION_KEY=1234567890abcdef
     
-Variables cannot include `#` sign, [more info](http://serverfault.com/questions/539730/environment-variable-in-etc-environment-with-pound-hash-sign-in-the-value).
-
-__Step 3.__ Configure how many dumps should be kept in FTP (optional).
-
-    # /etc/environment
-    KEEP_DUMPS=5   # default: 10
+    /* Dumps to be kept on FTP */
+    /* Optional, defaults to 10 */
+    KEEP_DUMPS=5
+    
+Note, that variables cannot include `#` sign, [more info](http://serverfault.com/questions/539730/environment-variable-in-etc-environment-with-pound-hash-sign-in-the-value). 
 
-__Step 4.__ Print the configuration to verify whether env variables has been loaded.
+__Step 2.__ Print the configuration to verify if env variables has been loaded properly.
 
     $ pg_export --configuration
-    => {:database=>"undefined", :keep_dumps=>10, :dump_encryption_key=>"k4***", :ftp_host=>"yourftp.example.com", 
-      :ftp_user=>"user", :ftp_password=>"pass***", :logger_format=>:plain} 
+    => {:dump_encryption_key=>"k4***", :ftp_host=>"yourftp.example.com", :ftp_user=>"your_ftp_user", 
+       :ftp_password=>"pass***", :logger_format=>"plain", :keep_dumps=>2} 
        
-__Step 5.__ Try connecting to FTP to verify the connection.
+__Step 3.__ Try connecting to FTP to verify the connection.
 
     $ pg_export --ftp
-    => Connect to yourftp.example.com
+    => 230 User your_ftp_user logged in
     
-__Step 6.__ Perform database export.
+__Step 4.__ Perform database export.
 
-    $ pg_export -d your_database
-    => Create Dump Tempfile (1.36MB)
-       Create Encrypted Dump Tempfile (1.34MB)
+    $ pg_export -d your_database [-k 5]
+    => Dump database your_database to your_database_20181016_121314 (1.36MB)
+       Encrypt your_database_20181016_121314 (1.34MB)
        Connect to yourftp.example.com
-       Export Encrypted Dump Tempfile (1.34MB) your_database_20161020_125747 to yourftp.example.com
+       Upload your_database_20181016_121314 (1.34MB) to yourftp.example.com
        Close FTP
        
 ## How to restore a dump?
 
-Go to interactive mode and follow the instructions:
+Run interactive mode and follow the instructions:
 
-    pg_export -i
+    pg_export [-d your_database] -i
 
 ## Development
 

data/Rakefile

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'bundler/gem_tasks'
 require 'rspec/core/rake_task'
 

data/bin/console

@@ -1,14 +1,8 @@
 #!/usr/bin/env ruby
+# frozen_string_literal: true
 
 require 'bundler/setup'
 require 'pg_export'
-
-# You can add fixtures and/or initialization code here to make experimenting
-# with your gem easier. You can also use a different console, if you like.
-
-# (If you use this, don't forget to add pry to your Gemfile!)
-# require "pry"
-# Pry.start
-
 require 'irb'
+
 IRB.start

data/bin/pg_export

@@ -1,40 +1,36 @@
 #!/usr/bin/env ruby
+# frozen_string_literal: true
 
 require 'optparse'
-require 'ostruct'
 
 require 'pg_export'
-require 'pg_export/roles/colourable_string'
-
-config = {
-  database: 'undefined',
-  keep_dumps: ENV['KEEP_DUMPS'],
-  dump_encryption_key: ENV['DUMP_ENCRYPTION_KEY'],
-  ftp_host: ENV['BACKUP_FTP_HOST'],
-  ftp_user: ENV['BACKUP_FTP_USER'],
-  ftp_password: ENV['BACKUP_FTP_PASSWORD'],
-  logger_format: :plain,
-  interactive: false
-}
+require 'pg_export/container'
+
+ENV['KEEP_DUMPS'] = ENV['KEEP_DUMPS'] || '10'
+interactive = false
+database = nil
 
 option_parser = OptionParser.new do |opts|
   opts.banner = 'Usage: pg_export [options]'
 
-  opts.on('-d', '--database DATABASE', String, '[Required] Name of the database to export') do |database|
-    config[:database] = database
+  opts.on('-d', '--database DATABASE', String, '[Required] Name of the database to export') do |d|
+    database = d
   end
 
-  opts.on('-k', '--keep [KEEP]', Integer, "[Optional] Number of dump files to keep on FTP (default: #{config[:keep_dumps]})") do |keep|
-    config[:keep_dumps] = keep
+  opts.on('-k', '--keep [KEEP]', String, "[Optional] Number of dump files to keep on FTP (default: #{ENV['KEEP_DUMPS']})") do |keep|
+    ENV['KEEP_DUMPS'] = keep
   end
 
   opts.on('-t', '--timestamped', '[Optional] Enables log messages with timestamps') do
-    config[:logger_format] = :timestamped
+    ENV['LOGGER_FORMAT'] = 'timestamped'
+  end
+
+  opts.on('-m', '--muted', '[Optional] Mutes log messages (overrides -t option)') do
+    ENV['LOGGER_FORMAT'] = 'muted'
   end
 
-  opts.on('-i', '--interactive', 'Interactive, command line mode, for restoring dumps into databases') do
-    config[:logger_format] = :muted
-    config[:interactive] = true
+  opts.on('-i', '--interactive', '[Optional] Interactive command line mode - for restoring dumps into databases') do
+    interactive = true
   end
 
   opts.on('-h', '--help', 'Show this message') do
@@ -45,39 +41,42 @@ option_parser = OptionParser.new do |opts|
   opts.separator "\nSetting can be verified by running following commands:"
 
   opts.on('-c', '--configuration', 'Prints the configuration') do
-    puts config
+    PgExport::Container.start(:config)
+    puts PgExport::Container['config'].to_h
     exit
   end
 
   opts.on('-f', '--ftp', 'Tries connecting to FTP to verify the connection') do
-    PgExport::Ftp::Connection.new(
-      host: config[:ftp_host],
-      user: config[:ftp_user],
-      password: config[:ftp_password],
-      logger: PgExport::BuildLogger.call(stream: $stdout, format: :plain)
-    ).open
+    PgExport::Container.start(:ftp)
+    ftp_adapter = PgExport::Container['factories.ftp_adapter_factory'].ftp_adapter
+    ftp = ftp_adapter.open_ftp
+    puts ftp.welcome
     exit
   end
 end
 
 begin
   option_parser.parse!
+rescue OptionParser::ParseError => e
+  warn e.message.capitalize
+  warn 'Type "pg_export -h" for available options'
+  exit
+end
+
+begin
+  pg_export =
+    if interactive
+      PgExport.interactive
+    else
+      PgExport.plain
+    end
+rescue PgExport::InitializationError => e
+  warn 'Unable to initialize PgExport due to invalid configuration. Check you ENVs.'
+  warn "Detailed message: #{e.message}"
+  exit
+end
 
-  PgExport.new(
-    dump_encryption_key: config[:dump_encryption_key],
-    ftp_host: config[:ftp_host],
-    ftp_user: config[:ftp_user],
-    ftp_password: config[:ftp_password],
-    logger_format: config[:logger_format],
-    interactive: config[:interactive]
-  ).call(
-    config[:database],
-    config[:keep_dumps]
-  )
-rescue OptionParser::MissingArgument, PgExport::PgExportError, ArgumentError => e
-  using PgExport::Roles::ColourableString
-  puts "Error: #{e}".red
-  puts option_parser
-rescue PgExport::PgDumpError => e
-  puts e
+pg_export.call(database) do |result|
+  result.success { puts 'Success' }
+  result.failure { |message:| warn message }
 end

data/lib/pg_export/build_logger.rb

@@ -1,10 +1,12 @@
+# frozen_string_literal: true
+
 require 'logger'
 
 class PgExport
   class BuildLogger
     FORMATS = {
       plain: ->(_, _, _, message) { "#{message}\n" },
-      muted: ->(_, _, _, _) {},
+      muted: ->(*) { raise 'Do not initialize logger when it is muted' },
       timestamped: lambda do |severity, datetime, progname, message|
         "#{datetime} #{Process.pid} TID-#{Thread.current.object_id.to_s(36)}#{progname} #{severity}: #{message}\n"
       end
@@ -12,7 +14,7 @@ class PgExport
 
     def self.call(stream:, format:)
       Logger.new(stream).tap do |logger|
-        logger.formatter = FORMATS.fetch(format)
+        logger.formatter = FORMATS.fetch(format.to_sym)
       end
     end
   end

data/lib/pg_export/configuration.rb

@@ -1,3 +1,6 @@
+# frozen_string_literal: true
+
+require 'dry-types'
 require 'dry-struct'
 
 class PgExport
@@ -8,7 +11,24 @@ class PgExport
     attribute :ftp_host,            Strict::String
     attribute :ftp_user,            Strict::String
     attribute :ftp_password,        Strict::String
-    attribute :logger_format,       Strict::Symbol.enum(:plain, :timestamped, :muted)
-    attribute :interactive,         Strict::Bool
+    attribute :logger_format,       Coercible::String.enum('plain', 'timestamped', 'muted')
+    attribute :keep_dumps,          Coercible::Integer.constrained(gteq: 0)
+
+    def self.build(env)
+      new(
+        dump_encryption_key: env['DUMP_ENCRYPTION_KEY'],
+        ftp_host: env['BACKUP_FTP_HOST'],
+        ftp_user: env['BACKUP_FTP_USER'],
+        ftp_password: env['BACKUP_FTP_PASSWORD'],
+        logger_format: env['LOGGER_FORMAT'] || 'plain',
+        keep_dumps: env['KEEP_DUMPS'] || 10
+      )
+    rescue Dry::Struct::Error => e
+      raise PgExport::InitializationError, e.message.gsub('[PgExport::Configuration.new] ', '')
+    end
+
+    def logger_muted?
+      logger_format == 'muted'
+    end
   end
 end

data/lib/pg_export/container.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+require 'dry/system/container'
+require 'pg_export/lib/pg_export/types'
+
+class PgExport
+  class Container < Dry::System::Container
+    configure do
+      config.root = Pathname(__FILE__).realpath.dirname
+      config.name = :pg_export
+      config.default_namespace = 'pg_export'
+      config.auto_register = %w[lib]
+    end
+
+    load_paths!('lib')
+
+    boot(:ftp) do
+      init do
+        require 'pg_export/lib/pg_export/factories/ftp_adapter_factory'
+      end
+
+      start do
+        use :config
+
+        register('factories.ftp_adapter_factory') do
+          ::PgExport::Factories::FtpAdapterFactory.new
+        end
+      end
+    end
+
+    boot(:main) do
+      init do
+        require 'pg_export/lib/pg_export/operations/encrypt_dump'
+        require 'pg_export/lib/pg_export/operations/decrypt_dump'
+        require 'pg_export/lib/pg_export/operations/remove_old_dumps_from_ftp'
+        require 'pg_export/lib/pg_export/operations/open_ftp_connection'
+      end
+
+      start do
+        use :ftp
+        register('operations.encrypt_dump') { ::PgExport::Operations::EncryptDump.new }
+        register('operations.decrypt_dump') { ::PgExport::Operations::DecryptDump.new }
+        register('operations.remove_old_dumps_from_ftp') { ::PgExport::Operations::RemoveOldDumpsFromFtp.new }
+        register('operations.open_ftp_connection') { ::PgExport::Operations::OpenFtpConnection.new }
+      end
+    end
+  end
+end

data/lib/pg_export/import.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+require_relative 'container'
+
+class PgExport
+  Import = PgExport::Container.injector.hash
+end

data/lib/pg_export/lib/pg_export/adapters/bash_adapter.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+require 'open3'
+
+class PgExport
+  module Adapters
+    class BashAdapter
+      class PgRestoreError < StandardError; end
+      class PgDumpError < StandardError; end
+
+      def pg_dump(file, db_name)
+        popen("pg_dump -Fc --file #{file.path} #{db_name}") do |errors|
+          raise PgDumpError, errors.chomp unless errors.empty?
+        end
+
+        file
+      end
+
+      def pg_restore(file, db_name)
+        popen("pg_restore -c -d #{db_name} #{file.path}") do |errors|
+          raise PgRestoreError, errors.chomp if /FATAL/ =~ errors
+        end
+      end
+
+      private
+
+      def popen(command)
+        Open3.popen3(command) do |_, _, err|
+          errors = err.read
+          yield errors
+        end
+
+        self
+      end
+    end
+  end
+end

data/lib/pg_export/lib/pg_export/adapters/ftp_adapter.rb

@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+# auto_register: false
+
+require 'net/ftp'
+
+class PgExport
+  module Adapters
+    class FtpAdapter
+      CHUNK_SIZE = (2**16).freeze
+
+      def initialize(host:, user:, password:)
+        @host, @user, @password, @logger = host, user, password
+        ObjectSpace.define_finalizer(self, proc { ftp.close if @ftp })
+      end
+
+      def open_ftp
+        @ftp = Net::FTP.new(host, user, password)
+        @ftp.passive = true
+        @ftp
+      end
+
+      def close_ftp
+        @ftp&.close
+      end
+
+      def list(regex_string)
+        ftp
+          .list(regex_string)
+          .map { |row| extracted_meaningful_attributes(row) }
+          .sort_by { |item| item[:name] }
+          .reverse
+      end
+
+      def delete(filename)
+        ftp.delete(filename)
+      end
+
+      def persist(file, name)
+        ftp.putbinaryfile(file.path, name, CHUNK_SIZE)
+      end
+
+      def get(file, name)
+        ftp.getbinaryfile(name, file.path, CHUNK_SIZE)
+      end
+
+      def to_s
+        host
+      end
+
+      def ftp
+        @ftp ||= open_ftp
+      end
+
+      private
+
+      attr_reader :host, :user, :password
+
+      def extracted_meaningful_attributes(item)
+        MEANINGFUL_KEYS.zip(item.split(' ').values_at(8, 4)).to_h
+      end
+
+      MEANINGFUL_KEYS = %i[name size].freeze
+      private_constant :MEANINGFUL_KEYS
+    end
+  end
+end

data/lib/pg_export/lib/pg_export/entities/dump.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+require 'dry-initializer'
+require 'pg_export/lib/pg_export/types'
+
+class PgExport
+  module Entities
+    class Dump
+      extend Dry::Initializer[undefined: false]
+
+      option :name,     Types::DumpName
+      option :type,     Types::DumpType
+      option :database, Types::Strict::String.constrained(filled: true)
+      option :file,     Types::DumpFile, default: proc { PgExport::ValueObjects::DumpFile.new }
+
+      def encrypt(cipher_factory:)
+        self.file = file.copy(cipher: cipher_factory.encryptor)
+        self.type = :encrypted
+
+        self
+      end
+
+      def decrypt(cipher_factory:)
+        self.file = file.copy(cipher: cipher_factory.decryptor)
+        self.type = :plain
+
+        self
+      end
+
+      def to_s
+        "#{name} (#{file.size_human})"
+      end
+
+      def file=(f)
+        @file = Types::DumpFile[f]
+      end
+
+      protected
+
+      def type=(t)
+        @type = Types::DumpType[t]
+      end
+    end
+  end
+end

data/lib/pg_export/lib/pg_export/factories/cipher_factory.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+require 'openssl'
+require 'pg_export/import'
+
+class PgExport
+  module Factories
+    class CipherFactory
+      include Import['config']
+
+      def encryptor
+        cipher(:encrypt)
+      end
+
+      def decryptor
+        cipher(:decrypt)
+      end
+
+      private
+
+      ALGORITHM = 'AES-128-CBC'
+      private_constant :ALGORITHM
+
+      def cipher(type)
+        OpenSSL::Cipher.new(ALGORITHM).tap do |cipher|
+          cipher.public_send(type)
+          cipher.key = config.dump_encryption_key
+        end
+      end
+    end
+  end
+end

data/lib/pg_export/lib/pg_export/factories/dump_factory.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+require 'open3'
+require 'tempfile'
+
+require 'pg_export/lib/pg_export/entities/dump'
+require 'pg_export/import'
+
+class PgExport
+  module Factories
+    class DumpFactory
+      def plain(database:, file:)
+        Entities::Dump.new(
+          name: [database, timestamp].join('_'),
+          database: database,
+          file: file,
+          type: :plain
+        )
+      end
+
+      private
+
+      TIMESTAMP_FORMAT = '%Y%m%d_%H%M%S'
+      private_constant :TIMESTAMP_FORMAT
+
+      def timestamp
+        Time.now.strftime(TIMESTAMP_FORMAT)
+      end
+    end
+  end
+end