pg_export 0.4.1 → 0.5.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.rubocop.yml +5 -3
- data/.travis.yml +3 -4
- data/CHANGELOG.md +9 -0
- data/README.md +3 -3
- data/bin/pg_export +18 -27
- data/lib/pg_export.rb +31 -30
- data/lib/pg_export/configuration.rb +8 -5
- data/lib/pg_export/includable_modules/colourable_string.rb +17 -0
- data/lib/pg_export/{entities → includable_modules}/dump/size_human.rb +0 -0
- data/lib/pg_export/{interactive.rb → includable_modules/interactive.rb} +31 -17
- data/lib/pg_export/includable_modules/logging.rb +31 -0
- data/lib/pg_export/includable_modules/services_container.rb +41 -0
- data/lib/pg_export/services/aes.rb +18 -11
- data/lib/pg_export/services/aes/base.rb +26 -0
- data/lib/pg_export/services/aes/decryptor.rb +12 -0
- data/lib/pg_export/services/aes/encryptor.rb +12 -0
- data/lib/pg_export/services/bash_utils.rb +32 -0
- data/lib/pg_export/services/dump_storage.rb +12 -12
- data/lib/pg_export/services/{ftp_service.rb → ftp_adapter.rb} +8 -6
- data/lib/pg_export/services/ftp_connection.rb +28 -0
- data/lib/pg_export/version.rb +1 -1
- metadata +15 -11
- data/lib/pg_export/concurrency.rb +0 -21
- data/lib/pg_export/interactive/refinements/colourable_string.rb +0 -19
- data/lib/pg_export/logging.rb +0 -11
- data/lib/pg_export/services/ftp_service/connection.rb +0 -31
- data/lib/pg_export/services/utils.rb +0 -57
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 69c56bdaa52fa25112c93ffd284cd806ce6fa707
|
4
|
+
data.tar.gz: 285340dfcf4c20067d57e8d805868b453f7cbf5f
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 8d51c8854e8d2b4d17dd43c96e43d4e1a97af5c8653472e51898fa3018278b5126629d279e29c407ddf0233bb898da77e3f06456495604a63e2ac07f743b3ca2
|
7
|
+
data.tar.gz: be9ad50fb36708db2e6232168677fc9eb8f24f4491862d37feb73230e0bd3aa550ebd19b1f4def5c6475620be2b507b54b1abce891b327cada84a1d1afbe4a86
|
data/.rubocop.yml
CHANGED
@@ -3,12 +3,14 @@ AllCops:
|
|
3
3
|
Exclude:
|
4
4
|
- 'spec/spec_helper.rb'
|
5
5
|
|
6
|
+
Style/BlockLength:
|
7
|
+
Exclude:
|
8
|
+
- 'bin/pg_export'
|
9
|
+
- 'spec/**/*.rb'
|
10
|
+
|
6
11
|
Metrics/LineLength:
|
7
12
|
Max: 200
|
8
13
|
|
9
|
-
Metrics/AbcSize:
|
10
|
-
Max: 20
|
11
|
-
|
12
14
|
Lint/AmbiguousOperator:
|
13
15
|
Exclude:
|
14
16
|
- 'lib/pg_export/configuration.rb'
|
data/.travis.yml
CHANGED
@@ -2,9 +2,8 @@ sudo: false
|
|
2
2
|
language: ruby
|
3
3
|
rvm:
|
4
4
|
- 2.1.8
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
CODECLIMATE_REPO_TOKEN: db03e5968c5bcd68b12ca50f5d41ae07dd74fe80d4e1421d754e31c316e7477a
|
5
|
+
addons:
|
6
|
+
code_climate:
|
7
|
+
repo_token: db03e5968c5bcd68b12ca50f5d41ae07dd74fe80d4e1421d754e31c316e7477a
|
9
8
|
before_install: gem install bundler -v 1.13.3
|
10
9
|
after_success: codeclimate-test-reporter
|
data/CHANGELOG.md
ADDED
@@ -0,0 +1,9 @@
|
|
1
|
+
### 0.5.0 - 2017.03.11
|
2
|
+
|
3
|
+
- Add restriction on DUMP_ENCRYPTION_KEY, to be exactly 16 characters length
|
4
|
+
- Make interactive mode more verbose by adding more messages
|
5
|
+
- Fix concurrently opening ftp connection
|
6
|
+
- Add closing ftp connection while importing dump in interactive mode
|
7
|
+
- Fix Cipher deprecation warning
|
8
|
+
- Fix typos
|
9
|
+
- Improve code architecture
|
data/README.md
CHANGED
@@ -11,7 +11,7 @@ Can be used for backups or synchronizing databases between production and develo
|
|
11
11
|
|
12
12
|
Example:
|
13
13
|
|
14
|
-
pg_export --database database_name
|
14
|
+
pg_export --database database_name --keep 5
|
15
15
|
|
16
16
|
Above command will perform database dump, encrypt it, upload it to FTP and remove old dumps from FTP, keeping newest 5.
|
17
17
|
|
@@ -20,8 +20,8 @@ FTP connection params and encryption key are configured by env variables.
|
|
20
20
|
Features:
|
21
21
|
|
22
22
|
- uses shell command `pg_dump` and `pg_restore`
|
23
|
-
-
|
24
|
-
-
|
23
|
+
- encrypts dumps by OpenSSL AES-128-CBC
|
24
|
+
- configurable through env variables
|
25
25
|
- uses ruby tempfiles, so local dumps are garbage collected automatically
|
26
26
|
- easy restoring dumps through interactive mode
|
27
27
|
|
data/bin/pg_export
CHANGED
@@ -2,33 +2,35 @@
|
|
2
2
|
|
3
3
|
require 'optparse'
|
4
4
|
require 'ostruct'
|
5
|
-
require 'pg_export'
|
6
|
-
require 'pg_export/interactive'
|
7
5
|
|
8
|
-
|
9
|
-
|
10
|
-
|
6
|
+
require 'cli_spinnable'
|
7
|
+
|
8
|
+
require 'pg_export'
|
9
|
+
require 'pg_export/includable_modules/colourable_string'
|
10
|
+
require 'pg_export/includable_modules/interactive'
|
11
11
|
|
12
|
-
PgExport::
|
12
|
+
config = PgExport::ServicesContainer.config
|
13
|
+
pg_export = PgExport.new
|
13
14
|
|
14
|
-
options = OpenStruct.new
|
15
15
|
option_parser = OptionParser.new do |opts|
|
16
16
|
opts.banner = 'Usage: pg_export [options]'
|
17
17
|
|
18
18
|
opts.on('-d', '--database DATABASE', '[Required] Name of the database to export') do |database|
|
19
|
-
|
19
|
+
config.database = database
|
20
20
|
end
|
21
21
|
|
22
|
-
opts.on('-k', '--keep [KEEP]', Integer,
|
23
|
-
|
22
|
+
opts.on('-k', '--keep [KEEP]', Integer, "[Optional] Number of dump files to keep on FTP (default: #{config.keep_dumps})") do |keep|
|
23
|
+
config.keep_dumps = keep
|
24
24
|
end
|
25
25
|
|
26
26
|
opts.on('-t', '--timestamped', '[Optional] Enables log messages with timestamps') do
|
27
|
-
|
27
|
+
PgExport::Logging.format_timestamped
|
28
28
|
end
|
29
29
|
|
30
30
|
opts.on('-i', '--interactive', 'Interactive, command line mode, for restoring dumps into databases') do
|
31
|
-
|
31
|
+
PgExport::Logging.mute
|
32
|
+
config.database ||= 'undefined'
|
33
|
+
pg_export.extend(PgExport::Interactive)
|
32
34
|
end
|
33
35
|
|
34
36
|
opts.on('-h', '--help', 'Show this message') do
|
@@ -39,33 +41,22 @@ option_parser = OptionParser.new do |opts|
|
|
39
41
|
opts.separator "\nSetting can be verified by running following commands:"
|
40
42
|
|
41
43
|
opts.on('-c', '--configuration', 'Prints the configuration') do
|
42
|
-
puts
|
44
|
+
puts config.to_s
|
43
45
|
exit
|
44
46
|
end
|
45
47
|
|
46
48
|
opts.on('-f', '--ftp', 'Tries connecting to FTP to verify the connection') do
|
47
|
-
PgExport::
|
49
|
+
PgExport::ServicesContainer.ftp_connection.open
|
48
50
|
exit
|
49
51
|
end
|
50
52
|
end
|
51
53
|
|
52
54
|
begin
|
53
55
|
option_parser.parse!
|
54
|
-
|
55
|
-
options.database ||= 'undefined' if options.interactive
|
56
|
-
PgExport::Logging.logger.formatter = LOGS_TIMESTAMPED if options.timestamped
|
57
|
-
PgExport::Logging.logger.formatter = LOGS_MUTED if options.interactive
|
58
|
-
|
59
|
-
pg_export = PgExport.new do |config|
|
60
|
-
config.database = options.database if options.database
|
61
|
-
config.keep_dumps = options.keep if options.keep
|
62
|
-
end
|
63
|
-
pg_export.extend(PgExport::Interactive) if options.interactive
|
64
|
-
|
65
56
|
pg_export.call
|
66
|
-
|
67
57
|
rescue OptionParser::MissingArgument, PgExport::InvalidConfigurationError => e
|
68
|
-
|
58
|
+
using PgExport::ColourableString
|
59
|
+
puts "Error: #{e}".red
|
69
60
|
puts option_parser
|
70
61
|
rescue PgExport::PgDumpError => e
|
71
62
|
puts e
|
data/lib/pg_export.rb
CHANGED
@@ -6,63 +6,64 @@ require 'openssl'
|
|
6
6
|
require 'forwardable'
|
7
7
|
require 'open3'
|
8
8
|
|
9
|
-
require 'cli_spinnable'
|
10
|
-
|
11
9
|
require 'pg_export/version'
|
12
|
-
require 'pg_export/logging'
|
10
|
+
require 'pg_export/includable_modules/logging'
|
11
|
+
require 'pg_export/includable_modules/dump/size_human'
|
12
|
+
require 'pg_export/includable_modules/services_container'
|
13
13
|
require 'pg_export/errors'
|
14
14
|
require 'pg_export/configuration'
|
15
|
-
require 'pg_export/concurrency'
|
16
|
-
require 'pg_export/entities/dump/size_human'
|
17
15
|
require 'pg_export/entities/dump/base'
|
18
16
|
require 'pg_export/entities/plain_dump'
|
19
17
|
require 'pg_export/entities/encrypted_dump'
|
20
|
-
require 'pg_export/services/
|
21
|
-
require 'pg_export/services/
|
22
|
-
require 'pg_export/services/
|
18
|
+
require 'pg_export/services/ftp_adapter'
|
19
|
+
require 'pg_export/services/ftp_connection'
|
20
|
+
require 'pg_export/services/bash_utils'
|
23
21
|
require 'pg_export/services/dump_storage'
|
24
22
|
require 'pg_export/services/aes'
|
23
|
+
require 'pg_export/services/aes/base'
|
24
|
+
require 'pg_export/services/aes/encryptor'
|
25
|
+
require 'pg_export/services/aes/decryptor'
|
25
26
|
|
26
27
|
class PgExport
|
27
|
-
|
28
|
+
extend Forwardable
|
29
|
+
include ServicesContainer
|
30
|
+
|
31
|
+
def_delegators :services_container, :config, :bash_utils, :dump_storage, :ftp_connection, :encryptor, :decryptor
|
28
32
|
|
29
33
|
def initialize
|
30
|
-
@config = Configuration.new
|
31
34
|
yield config if block_given?
|
32
|
-
config.validate
|
33
35
|
end
|
34
36
|
|
35
37
|
def call
|
36
|
-
|
37
|
-
|
38
|
-
|
38
|
+
config.validate
|
39
|
+
concurrently do |threads|
|
40
|
+
threads << create_dump
|
41
|
+
threads << open_ftp_connection
|
39
42
|
end
|
40
43
|
dump_storage.upload(dump)
|
41
|
-
dump_storage.remove_old
|
44
|
+
dump_storage.remove_old
|
42
45
|
self
|
43
46
|
end
|
44
47
|
|
45
48
|
private
|
46
49
|
|
47
|
-
|
48
|
-
|
50
|
+
def concurrently
|
51
|
+
[].tap do |threads|
|
52
|
+
yield threads
|
53
|
+
end.each(&:join)
|
54
|
+
end
|
49
55
|
|
50
|
-
def
|
51
|
-
|
52
|
-
enc_dump = utils.encrypt(sql_dump)
|
53
|
-
self.dump = enc_dump
|
56
|
+
def dump
|
57
|
+
create_dump[:dump]
|
54
58
|
end
|
55
59
|
|
56
|
-
def
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
+
def create_dump
|
61
|
+
@create_dump ||= Thread.new do
|
62
|
+
Thread.current[:dump] = encryptor.call(bash_utils.create_dump)
|
63
|
+
end
|
60
64
|
end
|
61
65
|
|
62
|
-
def
|
63
|
-
|
64
|
-
Aes.encryptor(config.dump_encryption_key),
|
65
|
-
Aes.decryptor(config.dump_encryption_key)
|
66
|
-
)
|
66
|
+
def open_ftp_connection
|
67
|
+
Thread.new { ftp_connection.open }
|
67
68
|
end
|
68
69
|
end
|
@@ -1,5 +1,8 @@
|
|
1
1
|
class PgExport
|
2
2
|
class Configuration
|
3
|
+
FIELD_REQUIRED = 'Field %s is required'.freeze
|
4
|
+
INVALID_ENCRYPTION_KEY_LENGTH = 'Dump encryption key should have exact 16 characters. Edit your DUMP_ENCRYPTION_KEY env variable.'.freeze
|
5
|
+
|
3
6
|
DEFAULTS = {
|
4
7
|
database: nil,
|
5
8
|
keep_dumps: ENV['KEEP_DUMPS'] || 10,
|
@@ -19,9 +22,9 @@ class PgExport
|
|
19
22
|
|
20
23
|
def validate
|
21
24
|
DEFAULTS.keys.each do |field|
|
22
|
-
raise InvalidConfigurationError,
|
25
|
+
raise InvalidConfigurationError, FIELD_REQUIRED % field if public_send(field).nil?
|
23
26
|
end
|
24
|
-
raise InvalidConfigurationError,
|
27
|
+
raise InvalidConfigurationError, INVALID_ENCRYPTION_KEY_LENGTH unless dump_encryption_key.length == 16
|
25
28
|
end
|
26
29
|
|
27
30
|
def ftp_params
|
@@ -40,13 +43,13 @@ class PgExport
|
|
40
43
|
|
41
44
|
def print_attr(key)
|
42
45
|
if %i(ftp_password dump_encryption_key).include?(key)
|
43
|
-
if
|
44
|
-
"#{key}: #{
|
46
|
+
if public_send(key)
|
47
|
+
"#{key}: #{public_send(key)[0..2]}***\n"
|
45
48
|
else
|
46
49
|
"#{key}:\n"
|
47
50
|
end
|
48
51
|
else
|
49
|
-
"#{key}: #{
|
52
|
+
"#{key}: #{public_send(key)}\n"
|
50
53
|
end
|
51
54
|
end
|
52
55
|
end
|
File without changes
|
@@ -1,5 +1,3 @@
|
|
1
|
-
require 'pg_export/interactive/refinements/colourable_string'
|
2
|
-
|
3
1
|
class PgExport
|
4
2
|
module Interactive
|
5
3
|
include CliSpinnable
|
@@ -10,53 +8,69 @@ class PgExport
|
|
10
8
|
end
|
11
9
|
|
12
10
|
def call
|
13
|
-
|
11
|
+
initialize_connection
|
14
12
|
print_all_dumps
|
15
|
-
|
16
|
-
|
13
|
+
selected_dump = select_dump
|
14
|
+
dump = download_dump(selected_dump)
|
15
|
+
concurrently do |threads|
|
16
|
+
threads << Thread.new(dump) { restore_downloaded_dump(dump) }
|
17
|
+
threads << Thread.new { ftp_connection.close }
|
18
|
+
end
|
17
19
|
puts 'Success'.green
|
18
20
|
self
|
19
21
|
end
|
20
22
|
|
21
23
|
private
|
22
24
|
|
23
|
-
def
|
25
|
+
def initialize_connection
|
24
26
|
with_spinner do |cli|
|
25
27
|
cli.print 'Connecting to FTP'
|
26
|
-
|
28
|
+
ftp_connection.open
|
27
29
|
cli.tick
|
28
30
|
end
|
29
31
|
end
|
30
32
|
|
31
33
|
def print_all_dumps
|
32
|
-
dumps.each.with_index(
|
34
|
+
dumps.each.with_index(1) do |name, i|
|
33
35
|
print "(#{i}) "
|
34
36
|
puts name.to_s.gray
|
35
37
|
end
|
36
38
|
self
|
37
39
|
end
|
38
40
|
|
39
|
-
def
|
41
|
+
def select_dump
|
40
42
|
puts 'Which dump would you like to import?'
|
41
|
-
|
42
|
-
|
43
|
+
number = loop do
|
44
|
+
print "Type from 1 to #{dumps.count} (1): "
|
45
|
+
number = gets.chomp.to_i
|
46
|
+
break number if (1..dumps.count).cover?(number)
|
47
|
+
puts 'Invalid number. Please try again.'.red
|
48
|
+
end
|
49
|
+
|
50
|
+
dumps.fetch(number - 1)
|
51
|
+
end
|
52
|
+
|
53
|
+
def download_dump(name)
|
54
|
+
dump = nil
|
55
|
+
|
43
56
|
with_spinner do |cli|
|
44
|
-
cli.print
|
57
|
+
cli.print "Downloading dump #{name}"
|
45
58
|
encrypted_dump = dump_storage.download(name)
|
46
59
|
cli.print " (#{encrypted_dump.size_human})"
|
47
60
|
cli.tick
|
48
|
-
cli.print
|
49
|
-
|
61
|
+
cli.print "Decrypting dump #{name}"
|
62
|
+
dump = decryptor.call(encrypted_dump)
|
50
63
|
cli.print " (#{dump.size_human})"
|
51
64
|
cli.tick
|
52
65
|
end
|
53
|
-
|
66
|
+
|
67
|
+
dump
|
54
68
|
rescue OpenSSL::Cipher::CipherError => e
|
55
69
|
puts "Problem decrypting dump file: #{e}. Try again.".red
|
56
70
|
retry
|
57
71
|
end
|
58
72
|
|
59
|
-
def restore_downloaded_dump
|
73
|
+
def restore_downloaded_dump(dump)
|
60
74
|
puts 'To which database you would like to restore the downloaded dump?'
|
61
75
|
if config.database == 'undefined'
|
62
76
|
print 'Enter a local database name: '
|
@@ -67,7 +81,7 @@ class PgExport
|
|
67
81
|
database = database.empty? ? config.database : database
|
68
82
|
with_spinner do |cli|
|
69
83
|
cli.print "Restoring dump to #{database} database"
|
70
|
-
|
84
|
+
bash_utils.restore_dump(dump, database)
|
71
85
|
cli.tick
|
72
86
|
end
|
73
87
|
self
|
@@ -0,0 +1,31 @@
|
|
1
|
+
class PgExport
|
2
|
+
module Logging
|
3
|
+
FORMAT_PLAIN = ->(_, _, _, message) { "#{message}\n" }
|
4
|
+
FORMAT_TIMESTAMPED = ->(severity, datetime, progname, message) { "#{datetime} #{Process.pid} TID-#{Thread.current.object_id.to_s(36)}#{progname} #{severity}: #{message}\n" }
|
5
|
+
FORMAT_MUTED = ->(_, _, _, _) {}
|
6
|
+
|
7
|
+
class << self
|
8
|
+
def logger
|
9
|
+
@logger ||= Logger.new(STDOUT).tap do |logger|
|
10
|
+
logger.formatter = FORMAT_PLAIN
|
11
|
+
end
|
12
|
+
end
|
13
|
+
|
14
|
+
def format_default
|
15
|
+
logger.formatter = FORMAT_PLAIN
|
16
|
+
end
|
17
|
+
|
18
|
+
def format_timestamped
|
19
|
+
logger.formatter = FORMAT_TIMESTAMPED
|
20
|
+
end
|
21
|
+
|
22
|
+
def mute
|
23
|
+
logger.formatter = FORMAT_MUTED
|
24
|
+
end
|
25
|
+
end
|
26
|
+
|
27
|
+
def logger
|
28
|
+
Logging.logger
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end
|
@@ -0,0 +1,41 @@
|
|
1
|
+
class PgExport
|
2
|
+
module ServicesContainer
|
3
|
+
class << self
|
4
|
+
def config
|
5
|
+
@config ||= Configuration.new
|
6
|
+
end
|
7
|
+
|
8
|
+
def aes
|
9
|
+
@aes ||= Aes.new(config.dump_encryption_key)
|
10
|
+
end
|
11
|
+
|
12
|
+
def encryptor
|
13
|
+
@encryptor ||= aes.build_encryptor
|
14
|
+
end
|
15
|
+
|
16
|
+
def decryptor
|
17
|
+
@decryptor ||= aes.build_decryptor
|
18
|
+
end
|
19
|
+
|
20
|
+
def bash_utils
|
21
|
+
@bash_utils ||= BashUtils.new(config.database)
|
22
|
+
end
|
23
|
+
|
24
|
+
def ftp_connection
|
25
|
+
@ftp_connection ||= FtpConnection.new(config.ftp_params)
|
26
|
+
end
|
27
|
+
|
28
|
+
def ftp_adapter
|
29
|
+
@ftp_adapter ||= FtpAdapter.new(ftp_connection)
|
30
|
+
end
|
31
|
+
|
32
|
+
def dump_storage
|
33
|
+
@dump_storage ||= DumpStorage.new(ftp_adapter, config.database, config.keep_dumps)
|
34
|
+
end
|
35
|
+
end
|
36
|
+
|
37
|
+
def services_container
|
38
|
+
@services_container ||= ServicesContainer
|
39
|
+
end
|
40
|
+
end
|
41
|
+
end
|
@@ -1,21 +1,28 @@
|
|
1
1
|
class PgExport
|
2
2
|
class Aes
|
3
|
-
|
3
|
+
ALGORITHM = 'AES-128-CBC'.freeze
|
4
4
|
|
5
|
-
def
|
6
|
-
|
5
|
+
def initialize(key)
|
6
|
+
@key = key
|
7
7
|
end
|
8
8
|
|
9
|
-
def
|
10
|
-
|
9
|
+
def build_encryptor
|
10
|
+
Aes::Encryptor.new(cipher(:encrypt))
|
11
11
|
end
|
12
12
|
|
13
|
-
def
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
13
|
+
def build_decryptor
|
14
|
+
Aes::Decryptor.new(cipher(:decrypt))
|
15
|
+
end
|
16
|
+
|
17
|
+
private
|
18
|
+
|
19
|
+
attr_reader :key
|
20
|
+
|
21
|
+
def cipher(mode)
|
22
|
+
OpenSSL::Cipher.new(ALGORITHM).tap do |cipher|
|
23
|
+
cipher.public_send(mode.to_sym)
|
24
|
+
cipher.key = key
|
25
|
+
end
|
19
26
|
end
|
20
27
|
end
|
21
28
|
end
|
@@ -0,0 +1,26 @@
|
|
1
|
+
class PgExport
|
2
|
+
class Aes
|
3
|
+
class Base
|
4
|
+
include Logging
|
5
|
+
|
6
|
+
def initialize(cipher)
|
7
|
+
@cipher = cipher
|
8
|
+
end
|
9
|
+
|
10
|
+
private
|
11
|
+
|
12
|
+
def copy_using(cipher, from:, to:)
|
13
|
+
cipher.reset
|
14
|
+
to.open(:write) do |f|
|
15
|
+
from.each_chunk do |chunk|
|
16
|
+
f << cipher.update(chunk)
|
17
|
+
end
|
18
|
+
f << cipher.final
|
19
|
+
end
|
20
|
+
self
|
21
|
+
end
|
22
|
+
|
23
|
+
attr_reader :cipher
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
@@ -0,0 +1,32 @@
|
|
1
|
+
class PgExport
|
2
|
+
class BashUtils
|
3
|
+
include Logging
|
4
|
+
|
5
|
+
def initialize(database_name)
|
6
|
+
@database_name = database_name
|
7
|
+
end
|
8
|
+
|
9
|
+
def create_dump
|
10
|
+
dump = PlainDump.new
|
11
|
+
Open3.popen3("pg_dump -Fc --file #{dump.path} #{database_name}") do |_, _, err|
|
12
|
+
error = err.read
|
13
|
+
raise PgDumpError, error unless error.empty?
|
14
|
+
end
|
15
|
+
logger.info "Create #{dump}"
|
16
|
+
dump
|
17
|
+
end
|
18
|
+
|
19
|
+
def restore_dump(dump, restore_database_name)
|
20
|
+
Open3.popen3("pg_restore -c -d #{restore_database_name} #{dump.path}") do |_, _, err|
|
21
|
+
error = err.read
|
22
|
+
raise PgRestoreError, error if /FATAL/ =~ error
|
23
|
+
end
|
24
|
+
logger.info "Restore #{dump}"
|
25
|
+
self
|
26
|
+
end
|
27
|
+
|
28
|
+
private
|
29
|
+
|
30
|
+
attr_reader :database_name
|
31
|
+
end
|
32
|
+
end
|
@@ -5,41 +5,41 @@ class PgExport
|
|
5
5
|
TIMESTAMP = '_%Y%m%d_%H%M%S'.freeze
|
6
6
|
TIMESTAMP_REGEX = '[0-9]{8}_[0-9]{6}'.freeze
|
7
7
|
|
8
|
-
def initialize(
|
9
|
-
@
|
8
|
+
def initialize(ftp_adapter, name, keep)
|
9
|
+
@ftp_adapter, @name, @keep = ftp_adapter, name, keep
|
10
10
|
end
|
11
11
|
|
12
12
|
def upload(dump)
|
13
13
|
dump_name = timestamped_name(dump)
|
14
|
-
|
15
|
-
logger.info "Upload #{dump} #{dump_name} to #{
|
14
|
+
ftp_adapter.upload_file(dump.path, dump_name)
|
15
|
+
logger.info "Upload #{dump} #{dump_name} to #{ftp_adapter}"
|
16
16
|
end
|
17
17
|
|
18
18
|
def download(name)
|
19
19
|
dump = EncryptedDump.new
|
20
|
-
|
21
|
-
logger.info "Download #{dump} #{name} from #{
|
20
|
+
ftp_adapter.download_file(dump.path, name)
|
21
|
+
logger.info "Download #{dump} #{name} from #{ftp_adapter}"
|
22
22
|
dump
|
23
23
|
end
|
24
24
|
|
25
|
-
def remove_old
|
25
|
+
def remove_old
|
26
26
|
find_by_name(name).drop(keep.to_i).each do |filename|
|
27
|
-
|
28
|
-
logger.info "Remove #{filename} from #{
|
27
|
+
ftp_adapter.delete(filename)
|
28
|
+
logger.info "Remove #{filename} from #{ftp_adapter}"
|
29
29
|
end
|
30
30
|
end
|
31
31
|
|
32
32
|
def find_by_name(s)
|
33
|
-
|
33
|
+
ftp_adapter.list(s + '_*')
|
34
34
|
end
|
35
35
|
|
36
36
|
def all
|
37
|
-
|
37
|
+
ftp_adapter.list('*')
|
38
38
|
end
|
39
39
|
|
40
40
|
private
|
41
41
|
|
42
|
-
attr_reader :
|
42
|
+
attr_reader :ftp_adapter, :name, :keep
|
43
43
|
|
44
44
|
def timestamped_name(dump)
|
45
45
|
name + Time.now.strftime(TIMESTAMP) + dump.ext
|
@@ -1,11 +1,13 @@
|
|
1
1
|
class PgExport
|
2
|
-
class
|
2
|
+
class FtpAdapter
|
3
|
+
extend Forwardable
|
4
|
+
|
3
5
|
CHUNK_SIZE = (2**16).freeze
|
4
6
|
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
@
|
7
|
+
def_delegators :connection, :ftp, :host
|
8
|
+
|
9
|
+
def initialize(connection)
|
10
|
+
@connection = connection
|
9
11
|
ObjectSpace.define_finalizer(self, proc { connection.close })
|
10
12
|
end
|
11
13
|
|
@@ -31,6 +33,6 @@ class PgExport
|
|
31
33
|
|
32
34
|
private
|
33
35
|
|
34
|
-
attr_reader :
|
36
|
+
attr_reader :connection
|
35
37
|
end
|
36
38
|
end
|
@@ -0,0 +1,28 @@
|
|
1
|
+
class PgExport
|
2
|
+
class FtpConnection
|
3
|
+
include Logging
|
4
|
+
|
5
|
+
attr_reader :ftp, :host
|
6
|
+
|
7
|
+
def initialize(host:, user:, password:)
|
8
|
+
@host, @user, @password = host, user, password
|
9
|
+
end
|
10
|
+
|
11
|
+
def open
|
12
|
+
@ftp = Net::FTP.new(host, user, password)
|
13
|
+
@ftp.passive = true
|
14
|
+
logger.info "Connect to #{host}"
|
15
|
+
self
|
16
|
+
end
|
17
|
+
|
18
|
+
def close
|
19
|
+
@ftp.close
|
20
|
+
logger.info 'Close FTP'
|
21
|
+
self
|
22
|
+
end
|
23
|
+
|
24
|
+
private
|
25
|
+
|
26
|
+
attr_reader :user, :password
|
27
|
+
end
|
28
|
+
end
|
data/lib/pg_export/version.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: pg_export
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.5.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Krzysztof Maicher
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2017-03-11 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: cli_spinnable
|
@@ -137,6 +137,7 @@ files:
|
|
137
137
|
- ".rspec"
|
138
138
|
- ".rubocop.yml"
|
139
139
|
- ".travis.yml"
|
140
|
+
- CHANGELOG.md
|
140
141
|
- CODE_OF_CONDUCT.md
|
141
142
|
- Gemfile
|
142
143
|
- LICENSE.txt
|
@@ -146,21 +147,24 @@ files:
|
|
146
147
|
- bin/pg_export
|
147
148
|
- bin/setup
|
148
149
|
- lib/pg_export.rb
|
149
|
-
- lib/pg_export/concurrency.rb
|
150
150
|
- lib/pg_export/configuration.rb
|
151
151
|
- lib/pg_export/entities/dump/base.rb
|
152
|
-
- lib/pg_export/entities/dump/size_human.rb
|
153
152
|
- lib/pg_export/entities/encrypted_dump.rb
|
154
153
|
- lib/pg_export/entities/plain_dump.rb
|
155
154
|
- lib/pg_export/errors.rb
|
156
|
-
- lib/pg_export/
|
157
|
-
- lib/pg_export/
|
158
|
-
- lib/pg_export/
|
155
|
+
- lib/pg_export/includable_modules/colourable_string.rb
|
156
|
+
- lib/pg_export/includable_modules/dump/size_human.rb
|
157
|
+
- lib/pg_export/includable_modules/interactive.rb
|
158
|
+
- lib/pg_export/includable_modules/logging.rb
|
159
|
+
- lib/pg_export/includable_modules/services_container.rb
|
159
160
|
- lib/pg_export/services/aes.rb
|
161
|
+
- lib/pg_export/services/aes/base.rb
|
162
|
+
- lib/pg_export/services/aes/decryptor.rb
|
163
|
+
- lib/pg_export/services/aes/encryptor.rb
|
164
|
+
- lib/pg_export/services/bash_utils.rb
|
160
165
|
- lib/pg_export/services/dump_storage.rb
|
161
|
-
- lib/pg_export/services/
|
162
|
-
- lib/pg_export/services/
|
163
|
-
- lib/pg_export/services/utils.rb
|
166
|
+
- lib/pg_export/services/ftp_adapter.rb
|
167
|
+
- lib/pg_export/services/ftp_connection.rb
|
164
168
|
- lib/pg_export/version.rb
|
165
169
|
- pg_export.gemspec
|
166
170
|
homepage: https://github.com/maicher/pg_export
|
@@ -183,7 +187,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
183
187
|
version: '0'
|
184
188
|
requirements: []
|
185
189
|
rubyforge_project:
|
186
|
-
rubygems_version: 2.
|
190
|
+
rubygems_version: 2.6.8
|
187
191
|
signing_key:
|
188
192
|
specification_version: 4
|
189
193
|
summary: CLI for creating and exporting PostgreSQL dumps to FTP.
|
@@ -1,21 +0,0 @@
|
|
1
|
-
class PgExport
|
2
|
-
module Concurrency
|
3
|
-
class ThreadsArray < Array
|
4
|
-
def <<(job)
|
5
|
-
super Thread.new { job }
|
6
|
-
end
|
7
|
-
|
8
|
-
alias push <<
|
9
|
-
end
|
10
|
-
|
11
|
-
def self.included(*)
|
12
|
-
Thread.abort_on_exception = true
|
13
|
-
end
|
14
|
-
|
15
|
-
def concurrently
|
16
|
-
t = ThreadsArray.new
|
17
|
-
yield t
|
18
|
-
t.each(&:join)
|
19
|
-
end
|
20
|
-
end
|
21
|
-
end
|
@@ -1,19 +0,0 @@
|
|
1
|
-
class PgExport
|
2
|
-
module Interactive
|
3
|
-
module ColourableString
|
4
|
-
refine String do
|
5
|
-
def red
|
6
|
-
"\e[31m#{self}\e[0m"
|
7
|
-
end
|
8
|
-
|
9
|
-
def green
|
10
|
-
"\e[0;32;49m#{self}\e[0m"
|
11
|
-
end
|
12
|
-
|
13
|
-
def gray
|
14
|
-
"\e[37m#{self}\e[0m"
|
15
|
-
end
|
16
|
-
end
|
17
|
-
end
|
18
|
-
end
|
19
|
-
end
|
data/lib/pg_export/logging.rb
DELETED
@@ -1,31 +0,0 @@
|
|
1
|
-
class PgExport
|
2
|
-
class FtpService
|
3
|
-
class Connection
|
4
|
-
include Logging
|
5
|
-
|
6
|
-
attr_reader :ftp
|
7
|
-
|
8
|
-
def initialize(host:, user:, password:)
|
9
|
-
@host, @user, @password = host, user, password
|
10
|
-
open
|
11
|
-
end
|
12
|
-
|
13
|
-
def open
|
14
|
-
@ftp = Net::FTP.new(host, user, password)
|
15
|
-
@ftp.passive = true
|
16
|
-
logger.info "Connect to #{host}"
|
17
|
-
self
|
18
|
-
end
|
19
|
-
|
20
|
-
def close
|
21
|
-
@ftp.close
|
22
|
-
logger.info 'Close FTP'
|
23
|
-
self
|
24
|
-
end
|
25
|
-
|
26
|
-
private
|
27
|
-
|
28
|
-
attr_reader :host, :user, :password
|
29
|
-
end
|
30
|
-
end
|
31
|
-
end
|
@@ -1,57 +0,0 @@
|
|
1
|
-
class PgExport
|
2
|
-
class Utils
|
3
|
-
include Logging
|
4
|
-
|
5
|
-
def initialize(encryptor, decryptor)
|
6
|
-
@encryptor, @decryptor = encryptor, decryptor
|
7
|
-
end
|
8
|
-
|
9
|
-
def create_dump(database_name)
|
10
|
-
dump = PlainDump.new
|
11
|
-
Open3.popen3("pg_dump -Fc --file #{dump.path} #{database_name}") do |_, _, err|
|
12
|
-
error = err.read
|
13
|
-
raise PgDumpError, error unless error.empty?
|
14
|
-
end
|
15
|
-
logger.info "Create #{dump}"
|
16
|
-
dump
|
17
|
-
end
|
18
|
-
|
19
|
-
def restore_dump(dump, database_name)
|
20
|
-
Open3.popen3("pg_restore -c -d #{database_name} #{dump.path}") do |_, _, err|
|
21
|
-
error = err.read
|
22
|
-
raise PgRestoreError, error if /FATAL/ =~ error
|
23
|
-
end
|
24
|
-
logger.info "Restore #{dump}"
|
25
|
-
self
|
26
|
-
end
|
27
|
-
|
28
|
-
def encrypt(dump)
|
29
|
-
enc_dump = EncryptedDump.new
|
30
|
-
copy_using(encryptor, from: dump, to: enc_dump)
|
31
|
-
logger.info "Create #{enc_dump}"
|
32
|
-
enc_dump
|
33
|
-
end
|
34
|
-
|
35
|
-
def decrypt(enc_dump)
|
36
|
-
dump = PlainDump.new
|
37
|
-
copy_using(decryptor, from: enc_dump, to: dump)
|
38
|
-
logger.info "Create #{dump}"
|
39
|
-
dump
|
40
|
-
end
|
41
|
-
|
42
|
-
private
|
43
|
-
|
44
|
-
attr_reader :encryptor, :decryptor
|
45
|
-
|
46
|
-
def copy_using(aes, from:, to:)
|
47
|
-
aes.reset
|
48
|
-
to.open(:write) do |f|
|
49
|
-
from.each_chunk do |chunk|
|
50
|
-
f << aes.update(chunk)
|
51
|
-
end
|
52
|
-
f << aes.final
|
53
|
-
end
|
54
|
-
self
|
55
|
-
end
|
56
|
-
end
|
57
|
-
end
|