pg_export 0.5.1 → 0.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 763552e9063089d2137c2eff38b159caee96f457
-  data.tar.gz: 445e0c662a841abf1c9ee1f3ba4fa05dc845e6e3
+  metadata.gz: 4e6bcca69be07806c87aa5e971292269c6e24ea3
+  data.tar.gz: 421af6b9f3430c9a5e3431da6f71d78b91463157
 SHA512:
-  metadata.gz: 9f8e27408b4aa14d29783f3672c456d09521cce85195fee112fbd32b8fd3b987f039a82dc28e652553fc40f63be0d80509cf05eb1299494c9beaac57ff8d7f2f
-  data.tar.gz: ab1c4938fbbb933735c35f806d6eca4745d240a7d749641180724ccebaba06e976b49a73762bc36eb4a7a577fdb6f2fe8e20ca41f4028cba29f8c235125c9204
+  metadata.gz: a2d7b152087be40ae99ae42a5e3acfb90b1aacef6780bfc6ec255883db9230c77455ae17065226e0aad91971ab74e89743badd74c4dd3c7f3571638fb8837ffe
+  data.tar.gz: 633e2d33af88ee049ee3ba5490ce228391ee445569533b8752b87ec5c29435bd1bdc0d518801b6c889e4a2898d08127d99a513d6239b1b54de3de1adea28b8f5

data/CHANGELOG.md

@@ -1,3 +1,7 @@
+### 0.6.0 - 2017.08.18
+
+- Improve internal architecture
+
 ### 0.5.1 - 2017.03.31
 
 - Simplify Dump entity design

data/README.md

@@ -3,7 +3,6 @@
 [![Gem Version](https://badge.fury.io/rb/pg_export.svg)](https://badge.fury.io/rb/pg_export)
 [![Build Status](https://travis-ci.org/maicher/pg_export.svg?branch=master)](https://travis-ci.org/maicher/pg_export)
 [![Code Climate](https://codeclimate.com/github/maicher/pg_export/badges/gpa.svg)](https://codeclimate.com/github/maicher/pg_export)
-[![Test Coverage](https://codeclimate.com/github/maicher/cli_spinnable/badges/coverage.svg)](https://codeclimate.com/github/maicher/pg_export/coverage)
 
 CLI for creating and exporting PostgreSQL dumps to FTP.
 
@@ -86,18 +85,13 @@ __Step 3.__ Configure how many dumps should be kept in FTP (optional).
 __Step 4.__ Print the configuration to verify whether env variables has been loaded.
 
     $ pg_export --configuration
-    => database: 
-       keep_dumps: 5
-       dump_password: k40***
-       ftp_host: yourftp.example.com
-       ftp_user: user
-       ftp_password: pass***
+    => {:database=>"undefined", :keep_dumps=>10, :dump_encryption_key=>"k4***", :ftp_host=>"yourftp.example.com", 
+      :ftp_user=>"user", :ftp_password=>"pass***", :logger_format=>:plain} 
        
 __Step 5.__ Try connecting to FTP to verify the connection.
 
     $ pg_export --ftp
     => Connect to yourftp.example.com
-       Close FTP
     
 __Step 6.__ Perform database export.
 

data/bin/pg_export

@@ -3,34 +3,38 @@
 require 'optparse'
 require 'ostruct'
 
-require 'cli_spinnable'
-
 require 'pg_export'
-require 'pg_export/includable_modules/colourable_string'
-require 'pg_export/includable_modules/interactive'
+require 'pg_export/roles/colourable_string'
 
-config = PgExport::ServicesContainer.config
-pg_export = PgExport.new
+config = {
+  database: 'undefined',
+  keep_dumps: ENV['KEEP_DUMPS'],
+  dump_encryption_key: ENV['DUMP_ENCRYPTION_KEY'],
+  ftp_host: ENV['BACKUP_FTP_HOST'],
+  ftp_user: ENV['BACKUP_FTP_USER'],
+  ftp_password: ENV['BACKUP_FTP_PASSWORD'],
+  logger_format: :plain,
+  interactive: false
+}
 
 option_parser = OptionParser.new do |opts|
   opts.banner = 'Usage: pg_export [options]'
 
-  opts.on('-d', '--database DATABASE', '[Required] Name of the database to export') do |database|
-    config.database = database
+  opts.on('-d', '--database DATABASE', String, '[Required] Name of the database to export') do |database|
+    config[:database] = database
   end
 
-  opts.on('-k', '--keep [KEEP]', Integer, "[Optional] Number of dump files to keep on FTP (default: #{config.keep_dumps})") do |keep|
-    config.keep_dumps = keep
+  opts.on('-k', '--keep [KEEP]', Integer, "[Optional] Number of dump files to keep on FTP (default: #{config[:keep_dumps]})") do |keep|
+    config[:keep_dumps] = keep
   end
 
   opts.on('-t', '--timestamped', '[Optional] Enables log messages with timestamps') do
-    PgExport::Logging.format_timestamped
+    config[:logger_format] = :timestamped
   end
 
   opts.on('-i', '--interactive', 'Interactive, command line mode, for restoring dumps into databases') do
-    PgExport::Logging.mute
-    config.database ||= 'undefined'
-    pg_export.extend(PgExport::Interactive)
+    config[:logger_format] = :muted
+    config[:interactive] = true
   end
 
   opts.on('-h', '--help', 'Show this message') do
@@ -41,21 +45,37 @@ option_parser = OptionParser.new do |opts|
   opts.separator "\nSetting can be verified by running following commands:"
 
   opts.on('-c', '--configuration', 'Prints the configuration') do
-    puts config.to_s
+    puts config
     exit
   end
 
   opts.on('-f', '--ftp', 'Tries connecting to FTP to verify the connection') do
-    PgExport::ServicesContainer.ftp_connection.open
+    PgExport::Ftp::Connection.new(
+      host: config[:ftp_host],
+      user: config[:ftp_user],
+      password: config[:ftp_password],
+      logger: PgExport::BuildLogger.call(stream: $stdout, format: :plain)
+    ).open
     exit
   end
 end
 
 begin
   option_parser.parse!
-  pg_export.call
-rescue OptionParser::MissingArgument, PgExport::InvalidConfigurationError => e
-  using PgExport::ColourableString
+
+  PgExport.new(
+    dump_encryption_key: config[:dump_encryption_key],
+    ftp_host: config[:ftp_host],
+    ftp_user: config[:ftp_user],
+    ftp_password: config[:ftp_password],
+    logger_format: config[:logger_format],
+    interactive: config[:interactive]
+  ).call(
+    config[:database],
+    config[:keep_dumps]
+  )
+rescue OptionParser::MissingArgument, PgExport::PgExportError, ArgumentError => e
+  using PgExport::Roles::ColourableString
   puts "Error: #{e}".red
   puts option_parser
 rescue PgExport::PgDumpError => e

data/lib/pg_export/aes/base.rb

@@ -0,0 +1,47 @@
+require 'openssl'
+require 'pg_export/dump'
+
+class PgExport
+  module Aes
+    class Base
+      ALGORITHM = 'AES-128-CBC'.freeze
+
+      def initialize(key:, logger:)
+        @key, @logger = key, logger
+      end
+
+      def call(source_dump)
+        target_dump = Dump.new(name: target_dump_name, db_name: source_dump.db_name)
+        copy(from: source_dump, to: target_dump)
+        logger.info "Create #{target_dump}"
+        target_dump
+      end
+
+      private
+
+      attr_reader :key, :logger
+
+      def copy(from:, to:)
+        cipher.reset
+        to.open(:write) do |f|
+          from.each_chunk do |chunk|
+            f << cipher.update(chunk)
+          end
+          f << cipher.final
+        end
+        self
+      end
+
+      def cipher
+        @cipher ||= build_cipher
+      end
+
+      def build_cipher
+        OpenSSL::Cipher.new(ALGORITHM).tap do |cipher|
+          cipher.public_send(cipher_type)
+          cipher.key = key
+        end
+      end
+    end
+  end
+end

data/lib/pg_export/aes/decryptor.rb

@@ -0,0 +1,13 @@
+class PgExport
+  module Aes
+    class Decryptor < Base
+      def target_dump_name
+        'Dump'.freeze
+      end
+
+      def cipher_type
+        :decrypt
+      end
+    end
+  end
+end

data/lib/pg_export/aes/encryptor.rb

@@ -0,0 +1,13 @@
+class PgExport
+  module Aes
+    class Encryptor < Base
+      def target_dump_name
+        'Encrypted Dump'.freeze
+      end
+
+      def cipher_type
+        :encrypt
+      end
+    end
+  end
+end

data/lib/pg_export/aes.rb

@@ -0,0 +1,3 @@
+require_relative 'aes/base'
+require_relative 'aes/encryptor'
+require_relative 'aes/decryptor'

data/lib/pg_export/bash/adapter.rb

@@ -0,0 +1,31 @@
+require 'open3'
+require 'pg_export/errors'
+
+class PgExport
+  module Bash
+    class Adapter
+      def get(path, db_name)
+        popen("pg_dump -Fc --file #{path} #{db_name}") do |errors|
+          raise PgDumpError, errors unless errors.empty?
+        end
+      end
+
+      def persist(path, db_name)
+        popen("pg_restore -c -d #{db_name} #{path}") do |errors|
+          raise PgRestoreError, errors if /FATAL/ =~ errors
+        end
+      end
+
+      private
+
+      def popen(command)
+        Open3.popen3(command) do |_, _, err|
+          errors = err.read
+          yield errors
+        end
+
+        self
+      end
+    end
+  end
+end

data/lib/pg_export/bash/factory.rb

@@ -0,0 +1,23 @@
+require 'open3'
+require 'pg_export/dump'
+
+class PgExport
+  module Bash
+    class Factory
+      def initialize(adapter:, logger:)
+        @adapter, @logger = adapter, logger
+      end
+
+      def build_dump(db_name)
+        dump = Dump.new(name: 'Dump', db_name: db_name)
+        adapter.get(dump.path, dump.db_name)
+        logger.info "Create #{dump}"
+        dump
+      end
+
+      private
+
+      attr_reader :adapter, :logger
+    end
+  end
+end

data/lib/pg_export/bash/repository.rb

@@ -0,0 +1,18 @@
+class PgExport
+  module Bash
+    class Repository
+      def initialize(adapter:, logger:)
+        @adapter, @logger = adapter, logger
+      end
+
+      def persist(dump, db_name)
+        adapter.persist(dump.path, db_name)
+        logger.info "Persist #{dump} #{db_name} to #{adapter}"
+      end
+
+      private
+
+      attr_reader :adapter, :logger
+    end
+  end
+end

data/lib/pg_export/boot_container.rb

@@ -0,0 +1,69 @@
+require_relative 'build_logger'
+require_relative 'ftp/adapter'
+require_relative 'ftp/connection'
+require_relative 'ftp/repository'
+require_relative 'bash/adapter'
+require_relative 'bash/repository'
+require_relative 'bash/factory'
+require_relative 'aes'
+require_relative 'services/create_and_export_dump'
+
+class PgExport
+  class BootContainer
+    class << self
+      def call(config)
+        container = {}
+
+        boot_logger(container, config)
+        boot_aes(container, config)
+        boot_ftp(container, config)
+        boot_bash(container)
+        boot_services(container)
+
+        container
+      end
+
+      private
+
+      def boot_logger(container, config)
+        container[:logger] = BuildLogger.call(stream: $stdout, format: config[:logger_format])
+      end
+
+      def boot_aes(container, config)
+        container[:encryptor] = Aes::Encryptor.new(key: config[:dump_encryption_key], logger: container[:logger])
+        container[:decryptor] = Aes::Decryptor.new(key: config[:dump_encryption_key], logger: container[:logger])
+      end
+
+      def boot_ftp(container, config)
+        container[:ftp_connection] = Ftp::Connection.new(
+          host: config[:ftp_host],
+          user: config[:ftp_user],
+          password: config[:ftp_password],
+          logger: container[:logger]
+        )
+        container[:ftp_adapter] = Ftp::Adapter.new(connection: container[:ftp_connection])
+        container[:ftp_repository] = Ftp::Repository.new(adapter: container[:ftp_adapter], logger: container[:logger])
+      end
+
+      def boot_bash(container)
+        container[:bash_adapter] = Bash::Adapter.new
+        container[:bash_repository] = Bash::Repository.new(
+          adapter: container[:bash_adapter],
+          logger: container[:logger]
+        )
+        container[:bash_factory] = Bash::Factory.new(
+          adapter: container[:bash_adapter],
+          logger: container[:logger]
+        )
+      end
+
+      def boot_services(container)
+        container[:create_and_export_dump] = Services::CreateAndExportDump.new(
+          bash_factory: container[:bash_factory],
+          encryptor: container[:encryptor],
+          ftp_repository: container[:ftp_repository]
+        )
+      end
+    end
+  end
+end

data/lib/pg_export/build_logger.rb

@@ -0,0 +1,19 @@
+require 'logger'
+
+class PgExport
+  class BuildLogger
+    FORMATS = {
+      plain: ->(_, _, _, message) { "#{message}\n" },
+      muted: ->(_, _, _, _) {},
+      timestamped: lambda do |severity, datetime, progname, message|
+        "#{datetime} #{Process.pid} TID-#{Thread.current.object_id.to_s(36)}#{progname} #{severity}: #{message}\n"
+      end
+    }.freeze
+
+    def self.call(stream:, format:)
+      Logger.new(stream).tap do |logger|
+        logger.formatter = FORMATS.fetch(format)
+      end
+    end
+  end
+end

data/lib/pg_export/configuration.rb

@@ -1,56 +1,14 @@
-class PgExport
-  class Configuration
-    FIELD_REQUIRED = 'Field %s is required'.freeze
-    INVALID_ENCRYPTION_KEY_LENGTH = 'Dump encryption key should have exact 16 characters. Edit your DUMP_ENCRYPTION_KEY env variable.'.freeze
-
-    DEFAULTS = {
-      database: nil,
-      keep_dumps: ENV['KEEP_DUMPS'] || 10,
-      dump_encryption_key: ENV['DUMP_ENCRYPTION_KEY'],
-      ftp_host: ENV['BACKUP_FTP_HOST'],
-      ftp_user: ENV['BACKUP_FTP_USER'],
-      ftp_password: ENV['BACKUP_FTP_PASSWORD']
-    }.freeze
-
-    attr_accessor *DEFAULTS.keys
-
-    def initialize
-      DEFAULTS.each_pair do |key, value|
-        send("#{key}=", value)
-      end
-    end
-
-    def validate
-      DEFAULTS.keys.each do |field|
-        raise InvalidConfigurationError, FIELD_REQUIRED % field if public_send(field).nil?
-      end
-      raise InvalidConfigurationError, INVALID_ENCRYPTION_KEY_LENGTH unless dump_encryption_key.length == 16
-    end
+require 'dry-struct'
 
-    def ftp_params
-      {
-        host: ftp_host,
-        user: ftp_user,
-        password: ftp_password
-      }
-    end
-
-    def to_s
-      DEFAULTS.keys.map(&method(:print_attr))
-    end
-
-    private
-
-    def print_attr(key)
-      if %i(ftp_password dump_encryption_key).include?(key)
-        if public_send(key)
-          "#{key}: #{public_send(key)[0..2]}***\n"
-        else
-          "#{key}:\n"
-        end
-      else
-        "#{key}: #{public_send(key)}\n"
-      end
-    end
+class PgExport
+  class Configuration < Dry::Struct
+    include Dry::Types.module
+
+    attribute :dump_encryption_key, Strict::String.constrained(size: 16)
+    attribute :ftp_host,            Strict::String
+    attribute :ftp_user,            Strict::String
+    attribute :ftp_password,        Strict::String
+    attribute :logger_format,       Strict::Symbol.enum(:plain, :timestamped, :muted)
+    attribute :interactive,         Strict::Bool
   end
 end

data/lib/pg_export/{entities/dump.rb → dump.rb}

@@ -1,16 +1,24 @@
+require 'forwardable'
+require 'tempfile'
+
+require_relative 'roles/human_readable'
+
 class PgExport
   class Dump
+    TIMESTAMP = '_%Y%m%d_%H%M%S'.freeze
+
     extend Forwardable
-    include SizeHuman
+    include Roles::HumanReadable
 
     CHUNK_SIZE = (2**16).freeze
 
     def_delegators :file, :path, :read, :write, :<<, :rewind, :close, :size, :eof?
 
-    attr_reader :name
+    attr_reader :name, :db_name
 
-    def initialize(name)
-      @name = name
+    def initialize(name:, db_name:)
+      @name, @db_name = name, db_name
+      @timestamp = Time.now.strftime(TIMESTAMP)
     end
 
     def ext
@@ -31,12 +39,18 @@ class PgExport
       end
     end
 
+    def timestamped_name
+      db_name + timestamp + ext
+    end
+
     def to_s
       "#{name} (#{size_human})"
     end
 
     private
 
+    attr_reader :timestamp
+
     def file
       @file ||= Tempfile.new(file_name)
     end

data/lib/pg_export/errors.rb

@@ -1,5 +1,5 @@
 class PgExport
-  class InvalidConfigurationError < StandardError; end
-  class PgDumpError < StandardError; end
-  class PgRestoreError < StandardError; end
+  class PgExportError < StandardError; end
+  class PgRestoreError < PgExportError; end
+  class PgDumpError < PgExportError; end
 end

data/lib/pg_export/ftp/adapter.rb

@@ -0,0 +1,41 @@
+class PgExport
+  module Ftp
+    class Adapter
+      CHUNK_SIZE = (2**16).freeze
+
+      def initialize(connection:)
+        @connection = connection
+        @host = connection.host
+        ObjectSpace.define_finalizer(self, proc { connection.close })
+      end
+
+      def list(regex_string)
+        ftp.list(regex_string).map { |item| item.split(' ').last }.sort.reverse
+      end
+
+      def delete(filename)
+        ftp.delete(filename)
+      end
+
+      def persist(path, timestamped_name)
+        ftp.putbinaryfile(path, timestamped_name, CHUNK_SIZE)
+      end
+
+      def get(path, timestamped_name)
+        ftp.getbinaryfile(timestamped_name, path, CHUNK_SIZE)
+      end
+
+      def ftp
+        connection.ftp
+      end
+
+      def to_s
+        host
+      end
+
+      private
+
+      attr_reader :connection, :host
+    end
+  end
+end

data/lib/pg_export/ftp/connection.rb

@@ -0,0 +1,40 @@
+require 'net/ftp'
+
+class PgExport
+  module Ftp
+    class Connection
+      attr_reader :host
+
+      def initialize(host:, user:, password:, logger:)
+        @host, @user, @password, @logger = host, user, password, logger
+        open_ftp_thread
+      end
+
+      def ftp
+        open_ftp_thread.join
+        @ftp
+      end
+
+      def close
+        ftp.close
+        logger.info 'Close FTP'
+        self
+      end
+
+      private
+
+      attr_reader :user, :password, :logger
+
+      def open
+        @ftp = Net::FTP.new(host, user, password)
+        @ftp.passive = true
+        logger.info "Connect to #{host}"
+        self
+      end
+
+      def open_ftp_thread
+        @open_ftp_thread ||= Thread.new { open }
+      end
+    end
+  end
+end

data/lib/pg_export/ftp/repository.rb

@@ -0,0 +1,40 @@
+class PgExport
+  module Ftp
+    class Repository
+      def initialize(adapter:, logger:)
+        @adapter, @logger = adapter, logger
+      end
+
+      def persist(dump)
+        adapter.persist(dump.path, dump.timestamped_name)
+        logger.info "Persist #{dump} #{dump.timestamped_name} to #{adapter}"
+      end
+
+      def get(db_name)
+        dump = Dump.new(name: 'Encrypted Dump', db_name: db_name)
+        adapter.get(dump.path, dump.db_name)
+        logger.info "Get #{dump} #{db_name} from #{adapter}"
+        dump
+      end
+
+      def remove_old(name, keep)
+        find_by_name(name).drop(keep).each do |filename|
+          adapter.delete(filename)
+          logger.info "Remove #{filename} from #{adapter}"
+        end
+      end
+
+      def find_by_name(name)
+        adapter.list(name + '_*')
+      end
+
+      def all
+        adapter.list('*')
+      end
+
+      private
+
+      attr_reader :adapter, :logger
+    end
+  end
+end

data/lib/pg_export/roles/colourable_string.rb

@@ -0,0 +1,19 @@
+class PgExport
+  module Roles
+    module ColourableString
+      refine String do
+        def red
+          "\e[31m#{self}\e[0m"
+        end
+
+        def green
+          "\e[0;32;49m#{self}\e[0m"
+        end
+
+        def gray
+          "\e[37m#{self}\e[0m"
+        end
+      end
+    end
+  end
+end

data/lib/pg_export/roles/human_readable.rb

@@ -0,0 +1,17 @@
+class PgExport
+  module Roles
+    module HumanReadable
+      MAPPING = {
+        'B'  => 1024,
+        'kB' => 1024 * 1024,
+        'MB' => 1024 * 1024 * 1024,
+        'GB' => 1024 * 1024 * 1024 * 1024,
+        'TB' => 1024 * 1024 * 1024 * 1024 * 1024
+      }.freeze
+
+      def size_human
+        MAPPING.each_pair { |e, s| return "#{(size.to_f / (s / 1024)).round(2)}#{e}" if size < s }
+      end
+    end
+  end
+end