pg_export 0.4.1 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.rubocop.yml +5 -3
- data/.travis.yml +3 -4
- data/CHANGELOG.md +9 -0
- data/README.md +3 -3
- data/bin/pg_export +18 -27
- data/lib/pg_export.rb +31 -30
- data/lib/pg_export/configuration.rb +8 -5
- data/lib/pg_export/includable_modules/colourable_string.rb +17 -0
- data/lib/pg_export/{entities → includable_modules}/dump/size_human.rb +0 -0
- data/lib/pg_export/{interactive.rb → includable_modules/interactive.rb} +31 -17
- data/lib/pg_export/includable_modules/logging.rb +31 -0
- data/lib/pg_export/includable_modules/services_container.rb +41 -0
- data/lib/pg_export/services/aes.rb +18 -11
- data/lib/pg_export/services/aes/base.rb +26 -0
- data/lib/pg_export/services/aes/decryptor.rb +12 -0
- data/lib/pg_export/services/aes/encryptor.rb +12 -0
- data/lib/pg_export/services/bash_utils.rb +32 -0
- data/lib/pg_export/services/dump_storage.rb +12 -12
- data/lib/pg_export/services/{ftp_service.rb → ftp_adapter.rb} +8 -6
- data/lib/pg_export/services/ftp_connection.rb +28 -0
- data/lib/pg_export/version.rb +1 -1
- metadata +15 -11
- data/lib/pg_export/concurrency.rb +0 -21
- data/lib/pg_export/interactive/refinements/colourable_string.rb +0 -19
- data/lib/pg_export/logging.rb +0 -11
- data/lib/pg_export/services/ftp_service/connection.rb +0 -31
- data/lib/pg_export/services/utils.rb +0 -57
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 69c56bdaa52fa25112c93ffd284cd806ce6fa707
|
4
|
+
data.tar.gz: 285340dfcf4c20067d57e8d805868b453f7cbf5f
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 8d51c8854e8d2b4d17dd43c96e43d4e1a97af5c8653472e51898fa3018278b5126629d279e29c407ddf0233bb898da77e3f06456495604a63e2ac07f743b3ca2
|
7
|
+
data.tar.gz: be9ad50fb36708db2e6232168677fc9eb8f24f4491862d37feb73230e0bd3aa550ebd19b1f4def5c6475620be2b507b54b1abce891b327cada84a1d1afbe4a86
|
data/.rubocop.yml
CHANGED
@@ -3,12 +3,14 @@ AllCops:
|
|
3
3
|
Exclude:
|
4
4
|
- 'spec/spec_helper.rb'
|
5
5
|
|
6
|
+
Style/BlockLength:
|
7
|
+
Exclude:
|
8
|
+
- 'bin/pg_export'
|
9
|
+
- 'spec/**/*.rb'
|
10
|
+
|
6
11
|
Metrics/LineLength:
|
7
12
|
Max: 200
|
8
13
|
|
9
|
-
Metrics/AbcSize:
|
10
|
-
Max: 20
|
11
|
-
|
12
14
|
Lint/AmbiguousOperator:
|
13
15
|
Exclude:
|
14
16
|
- 'lib/pg_export/configuration.rb'
|
data/.travis.yml
CHANGED
@@ -2,9 +2,8 @@ sudo: false
|
|
2
2
|
language: ruby
|
3
3
|
rvm:
|
4
4
|
- 2.1.8
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
CODECLIMATE_REPO_TOKEN: db03e5968c5bcd68b12ca50f5d41ae07dd74fe80d4e1421d754e31c316e7477a
|
5
|
+
addons:
|
6
|
+
code_climate:
|
7
|
+
repo_token: db03e5968c5bcd68b12ca50f5d41ae07dd74fe80d4e1421d754e31c316e7477a
|
9
8
|
before_install: gem install bundler -v 1.13.3
|
10
9
|
after_success: codeclimate-test-reporter
|
data/CHANGELOG.md
ADDED
@@ -0,0 +1,9 @@
|
|
1
|
+
### 0.5.0 - 2017.03.11
|
2
|
+
|
3
|
+
- Add restriction on DUMP_ENCRYPTION_KEY, to be exactly 16 characters length
|
4
|
+
- Make interactive mode more verbose by adding more messages
|
5
|
+
- Fix concurrently opening ftp connection
|
6
|
+
- Add closing ftp connection while importing dump in interactive mode
|
7
|
+
- Fix Cipher deprecation warning
|
8
|
+
- Fix typos
|
9
|
+
- Improve code architecture
|
data/README.md
CHANGED
@@ -11,7 +11,7 @@ Can be used for backups or synchronizing databases between production and develo
|
|
11
11
|
|
12
12
|
Example:
|
13
13
|
|
14
|
-
pg_export --database database_name
|
14
|
+
pg_export --database database_name --keep 5
|
15
15
|
|
16
16
|
Above command will perform database dump, encrypt it, upload it to FTP and remove old dumps from FTP, keeping newest 5.
|
17
17
|
|
@@ -20,8 +20,8 @@ FTP connection params and encryption key are configured by env variables.
|
|
20
20
|
Features:
|
21
21
|
|
22
22
|
- uses shell command `pg_dump` and `pg_restore`
|
23
|
-
-
|
24
|
-
-
|
23
|
+
- encrypts dumps by OpenSSL AES-128-CBC
|
24
|
+
- configurable through env variables
|
25
25
|
- uses ruby tempfiles, so local dumps are garbage collected automatically
|
26
26
|
- easy restoring dumps through interactive mode
|
27
27
|
|
data/bin/pg_export
CHANGED
@@ -2,33 +2,35 @@
|
|
2
2
|
|
3
3
|
require 'optparse'
|
4
4
|
require 'ostruct'
|
5
|
-
require 'pg_export'
|
6
|
-
require 'pg_export/interactive'
|
7
5
|
|
8
|
-
|
9
|
-
|
10
|
-
|
6
|
+
require 'cli_spinnable'
|
7
|
+
|
8
|
+
require 'pg_export'
|
9
|
+
require 'pg_export/includable_modules/colourable_string'
|
10
|
+
require 'pg_export/includable_modules/interactive'
|
11
11
|
|
12
|
-
PgExport::
|
12
|
+
config = PgExport::ServicesContainer.config
|
13
|
+
pg_export = PgExport.new
|
13
14
|
|
14
|
-
options = OpenStruct.new
|
15
15
|
option_parser = OptionParser.new do |opts|
|
16
16
|
opts.banner = 'Usage: pg_export [options]'
|
17
17
|
|
18
18
|
opts.on('-d', '--database DATABASE', '[Required] Name of the database to export') do |database|
|
19
|
-
|
19
|
+
config.database = database
|
20
20
|
end
|
21
21
|
|
22
|
-
opts.on('-k', '--keep [KEEP]', Integer,
|
23
|
-
|
22
|
+
opts.on('-k', '--keep [KEEP]', Integer, "[Optional] Number of dump files to keep on FTP (default: #{config.keep_dumps})") do |keep|
|
23
|
+
config.keep_dumps = keep
|
24
24
|
end
|
25
25
|
|
26
26
|
opts.on('-t', '--timestamped', '[Optional] Enables log messages with timestamps') do
|
27
|
-
|
27
|
+
PgExport::Logging.format_timestamped
|
28
28
|
end
|
29
29
|
|
30
30
|
opts.on('-i', '--interactive', 'Interactive, command line mode, for restoring dumps into databases') do
|
31
|
-
|
31
|
+
PgExport::Logging.mute
|
32
|
+
config.database ||= 'undefined'
|
33
|
+
pg_export.extend(PgExport::Interactive)
|
32
34
|
end
|
33
35
|
|
34
36
|
opts.on('-h', '--help', 'Show this message') do
|
@@ -39,33 +41,22 @@ option_parser = OptionParser.new do |opts|
|
|
39
41
|
opts.separator "\nSetting can be verified by running following commands:"
|
40
42
|
|
41
43
|
opts.on('-c', '--configuration', 'Prints the configuration') do
|
42
|
-
puts
|
44
|
+
puts config.to_s
|
43
45
|
exit
|
44
46
|
end
|
45
47
|
|
46
48
|
opts.on('-f', '--ftp', 'Tries connecting to FTP to verify the connection') do
|
47
|
-
PgExport::
|
49
|
+
PgExport::ServicesContainer.ftp_connection.open
|
48
50
|
exit
|
49
51
|
end
|
50
52
|
end
|
51
53
|
|
52
54
|
begin
|
53
55
|
option_parser.parse!
|
54
|
-
|
55
|
-
options.database ||= 'undefined' if options.interactive
|
56
|
-
PgExport::Logging.logger.formatter = LOGS_TIMESTAMPED if options.timestamped
|
57
|
-
PgExport::Logging.logger.formatter = LOGS_MUTED if options.interactive
|
58
|
-
|
59
|
-
pg_export = PgExport.new do |config|
|
60
|
-
config.database = options.database if options.database
|
61
|
-
config.keep_dumps = options.keep if options.keep
|
62
|
-
end
|
63
|
-
pg_export.extend(PgExport::Interactive) if options.interactive
|
64
|
-
|
65
56
|
pg_export.call
|
66
|
-
|
67
57
|
rescue OptionParser::MissingArgument, PgExport::InvalidConfigurationError => e
|
68
|
-
|
58
|
+
using PgExport::ColourableString
|
59
|
+
puts "Error: #{e}".red
|
69
60
|
puts option_parser
|
70
61
|
rescue PgExport::PgDumpError => e
|
71
62
|
puts e
|
data/lib/pg_export.rb
CHANGED
@@ -6,63 +6,64 @@ require 'openssl'
|
|
6
6
|
require 'forwardable'
|
7
7
|
require 'open3'
|
8
8
|
|
9
|
-
require 'cli_spinnable'
|
10
|
-
|
11
9
|
require 'pg_export/version'
|
12
|
-
require 'pg_export/logging'
|
10
|
+
require 'pg_export/includable_modules/logging'
|
11
|
+
require 'pg_export/includable_modules/dump/size_human'
|
12
|
+
require 'pg_export/includable_modules/services_container'
|
13
13
|
require 'pg_export/errors'
|
14
14
|
require 'pg_export/configuration'
|
15
|
-
require 'pg_export/concurrency'
|
16
|
-
require 'pg_export/entities/dump/size_human'
|
17
15
|
require 'pg_export/entities/dump/base'
|
18
16
|
require 'pg_export/entities/plain_dump'
|
19
17
|
require 'pg_export/entities/encrypted_dump'
|
20
|
-
require 'pg_export/services/
|
21
|
-
require 'pg_export/services/
|
22
|
-
require 'pg_export/services/
|
18
|
+
require 'pg_export/services/ftp_adapter'
|
19
|
+
require 'pg_export/services/ftp_connection'
|
20
|
+
require 'pg_export/services/bash_utils'
|
23
21
|
require 'pg_export/services/dump_storage'
|
24
22
|
require 'pg_export/services/aes'
|
23
|
+
require 'pg_export/services/aes/base'
|
24
|
+
require 'pg_export/services/aes/encryptor'
|
25
|
+
require 'pg_export/services/aes/decryptor'
|
25
26
|
|
26
27
|
class PgExport
|
27
|
-
|
28
|
+
extend Forwardable
|
29
|
+
include ServicesContainer
|
30
|
+
|
31
|
+
def_delegators :services_container, :config, :bash_utils, :dump_storage, :ftp_connection, :encryptor, :decryptor
|
28
32
|
|
29
33
|
def initialize
|
30
|
-
@config = Configuration.new
|
31
34
|
yield config if block_given?
|
32
|
-
config.validate
|
33
35
|
end
|
34
36
|
|
35
37
|
def call
|
36
|
-
|
37
|
-
|
38
|
-
|
38
|
+
config.validate
|
39
|
+
concurrently do |threads|
|
40
|
+
threads << create_dump
|
41
|
+
threads << open_ftp_connection
|
39
42
|
end
|
40
43
|
dump_storage.upload(dump)
|
41
|
-
dump_storage.remove_old
|
44
|
+
dump_storage.remove_old
|
42
45
|
self
|
43
46
|
end
|
44
47
|
|
45
48
|
private
|
46
49
|
|
47
|
-
|
48
|
-
|
50
|
+
def concurrently
|
51
|
+
[].tap do |threads|
|
52
|
+
yield threads
|
53
|
+
end.each(&:join)
|
54
|
+
end
|
49
55
|
|
50
|
-
def
|
51
|
-
|
52
|
-
enc_dump = utils.encrypt(sql_dump)
|
53
|
-
self.dump = enc_dump
|
56
|
+
def dump
|
57
|
+
create_dump[:dump]
|
54
58
|
end
|
55
59
|
|
56
|
-
def
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
+
def create_dump
|
61
|
+
@create_dump ||= Thread.new do
|
62
|
+
Thread.current[:dump] = encryptor.call(bash_utils.create_dump)
|
63
|
+
end
|
60
64
|
end
|
61
65
|
|
62
|
-
def
|
63
|
-
|
64
|
-
Aes.encryptor(config.dump_encryption_key),
|
65
|
-
Aes.decryptor(config.dump_encryption_key)
|
66
|
-
)
|
66
|
+
def open_ftp_connection
|
67
|
+
Thread.new { ftp_connection.open }
|
67
68
|
end
|
68
69
|
end
|
@@ -1,5 +1,8 @@
|
|
1
1
|
class PgExport
|
2
2
|
class Configuration
|
3
|
+
FIELD_REQUIRED = 'Field %s is required'.freeze
|
4
|
+
INVALID_ENCRYPTION_KEY_LENGTH = 'Dump encryption key should have exact 16 characters. Edit your DUMP_ENCRYPTION_KEY env variable.'.freeze
|
5
|
+
|
3
6
|
DEFAULTS = {
|
4
7
|
database: nil,
|
5
8
|
keep_dumps: ENV['KEEP_DUMPS'] || 10,
|
@@ -19,9 +22,9 @@ class PgExport
|
|
19
22
|
|
20
23
|
def validate
|
21
24
|
DEFAULTS.keys.each do |field|
|
22
|
-
raise InvalidConfigurationError,
|
25
|
+
raise InvalidConfigurationError, FIELD_REQUIRED % field if public_send(field).nil?
|
23
26
|
end
|
24
|
-
raise InvalidConfigurationError,
|
27
|
+
raise InvalidConfigurationError, INVALID_ENCRYPTION_KEY_LENGTH unless dump_encryption_key.length == 16
|
25
28
|
end
|
26
29
|
|
27
30
|
def ftp_params
|
@@ -40,13 +43,13 @@ class PgExport
|
|
40
43
|
|
41
44
|
def print_attr(key)
|
42
45
|
if %i(ftp_password dump_encryption_key).include?(key)
|
43
|
-
if
|
44
|
-
"#{key}: #{
|
46
|
+
if public_send(key)
|
47
|
+
"#{key}: #{public_send(key)[0..2]}***\n"
|
45
48
|
else
|
46
49
|
"#{key}:\n"
|
47
50
|
end
|
48
51
|
else
|
49
|
-
"#{key}: #{
|
52
|
+
"#{key}: #{public_send(key)}\n"
|
50
53
|
end
|
51
54
|
end
|
52
55
|
end
|
File without changes
|
@@ -1,5 +1,3 @@
|
|
1
|
-
require 'pg_export/interactive/refinements/colourable_string'
|
2
|
-
|
3
1
|
class PgExport
|
4
2
|
module Interactive
|
5
3
|
include CliSpinnable
|
@@ -10,53 +8,69 @@ class PgExport
|
|
10
8
|
end
|
11
9
|
|
12
10
|
def call
|
13
|
-
|
11
|
+
initialize_connection
|
14
12
|
print_all_dumps
|
15
|
-
|
16
|
-
|
13
|
+
selected_dump = select_dump
|
14
|
+
dump = download_dump(selected_dump)
|
15
|
+
concurrently do |threads|
|
16
|
+
threads << Thread.new(dump) { restore_downloaded_dump(dump) }
|
17
|
+
threads << Thread.new { ftp_connection.close }
|
18
|
+
end
|
17
19
|
puts 'Success'.green
|
18
20
|
self
|
19
21
|
end
|
20
22
|
|
21
23
|
private
|
22
24
|
|
23
|
-
def
|
25
|
+
def initialize_connection
|
24
26
|
with_spinner do |cli|
|
25
27
|
cli.print 'Connecting to FTP'
|
26
|
-
|
28
|
+
ftp_connection.open
|
27
29
|
cli.tick
|
28
30
|
end
|
29
31
|
end
|
30
32
|
|
31
33
|
def print_all_dumps
|
32
|
-
dumps.each.with_index(
|
34
|
+
dumps.each.with_index(1) do |name, i|
|
33
35
|
print "(#{i}) "
|
34
36
|
puts name.to_s.gray
|
35
37
|
end
|
36
38
|
self
|
37
39
|
end
|
38
40
|
|
39
|
-
def
|
41
|
+
def select_dump
|
40
42
|
puts 'Which dump would you like to import?'
|
41
|
-
|
42
|
-
|
43
|
+
number = loop do
|
44
|
+
print "Type from 1 to #{dumps.count} (1): "
|
45
|
+
number = gets.chomp.to_i
|
46
|
+
break number if (1..dumps.count).cover?(number)
|
47
|
+
puts 'Invalid number. Please try again.'.red
|
48
|
+
end
|
49
|
+
|
50
|
+
dumps.fetch(number - 1)
|
51
|
+
end
|
52
|
+
|
53
|
+
def download_dump(name)
|
54
|
+
dump = nil
|
55
|
+
|
43
56
|
with_spinner do |cli|
|
44
|
-
cli.print
|
57
|
+
cli.print "Downloading dump #{name}"
|
45
58
|
encrypted_dump = dump_storage.download(name)
|
46
59
|
cli.print " (#{encrypted_dump.size_human})"
|
47
60
|
cli.tick
|
48
|
-
cli.print
|
49
|
-
|
61
|
+
cli.print "Decrypting dump #{name}"
|
62
|
+
dump = decryptor.call(encrypted_dump)
|
50
63
|
cli.print " (#{dump.size_human})"
|
51
64
|
cli.tick
|
52
65
|
end
|
53
|
-
|
66
|
+
|
67
|
+
dump
|
54
68
|
rescue OpenSSL::Cipher::CipherError => e
|
55
69
|
puts "Problem decrypting dump file: #{e}. Try again.".red
|
56
70
|
retry
|
57
71
|
end
|
58
72
|
|
59
|
-
def restore_downloaded_dump
|
73
|
+
def restore_downloaded_dump(dump)
|
60
74
|
puts 'To which database you would like to restore the downloaded dump?'
|
61
75
|
if config.database == 'undefined'
|
62
76
|
print 'Enter a local database name: '
|
@@ -67,7 +81,7 @@ class PgExport
|
|
67
81
|
database = database.empty? ? config.database : database
|
68
82
|
with_spinner do |cli|
|
69
83
|
cli.print "Restoring dump to #{database} database"
|
70
|
-
|
84
|
+
bash_utils.restore_dump(dump, database)
|
71
85
|
cli.tick
|
72
86
|
end
|
73
87
|
self
|
@@ -0,0 +1,31 @@
|
|
1
|
+
class PgExport
|
2
|
+
module Logging
|
3
|
+
FORMAT_PLAIN = ->(_, _, _, message) { "#{message}\n" }
|
4
|
+
FORMAT_TIMESTAMPED = ->(severity, datetime, progname, message) { "#{datetime} #{Process.pid} TID-#{Thread.current.object_id.to_s(36)}#{progname} #{severity}: #{message}\n" }
|
5
|
+
FORMAT_MUTED = ->(_, _, _, _) {}
|
6
|
+
|
7
|
+
class << self
|
8
|
+
def logger
|
9
|
+
@logger ||= Logger.new(STDOUT).tap do |logger|
|
10
|
+
logger.formatter = FORMAT_PLAIN
|
11
|
+
end
|
12
|
+
end
|
13
|
+
|
14
|
+
def format_default
|
15
|
+
logger.formatter = FORMAT_PLAIN
|
16
|
+
end
|
17
|
+
|
18
|
+
def format_timestamped
|
19
|
+
logger.formatter = FORMAT_TIMESTAMPED
|
20
|
+
end
|
21
|
+
|
22
|
+
def mute
|
23
|
+
logger.formatter = FORMAT_MUTED
|
24
|
+
end
|
25
|
+
end
|
26
|
+
|
27
|
+
def logger
|
28
|
+
Logging.logger
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end
|
@@ -0,0 +1,41 @@
|
|
1
|
+
class PgExport
|
2
|
+
module ServicesContainer
|
3
|
+
class << self
|
4
|
+
def config
|
5
|
+
@config ||= Configuration.new
|
6
|
+
end
|
7
|
+
|
8
|
+
def aes
|
9
|
+
@aes ||= Aes.new(config.dump_encryption_key)
|
10
|
+
end
|
11
|
+
|
12
|
+
def encryptor
|
13
|
+
@encryptor ||= aes.build_encryptor
|
14
|
+
end
|
15
|
+
|
16
|
+
def decryptor
|
17
|
+
@decryptor ||= aes.build_decryptor
|
18
|
+
end
|
19
|
+
|
20
|
+
def bash_utils
|
21
|
+
@bash_utils ||= BashUtils.new(config.database)
|
22
|
+
end
|
23
|
+
|
24
|
+
def ftp_connection
|
25
|
+
@ftp_connection ||= FtpConnection.new(config.ftp_params)
|
26
|
+
end
|
27
|
+
|
28
|
+
def ftp_adapter
|
29
|
+
@ftp_adapter ||= FtpAdapter.new(ftp_connection)
|
30
|
+
end
|
31
|
+
|
32
|
+
def dump_storage
|
33
|
+
@dump_storage ||= DumpStorage.new(ftp_adapter, config.database, config.keep_dumps)
|
34
|
+
end
|
35
|
+
end
|
36
|
+
|
37
|
+
def services_container
|
38
|
+
@services_container ||= ServicesContainer
|
39
|
+
end
|
40
|
+
end
|
41
|
+
end
|
@@ -1,21 +1,28 @@
|
|
1
1
|
class PgExport
|
2
2
|
class Aes
|
3
|
-
|
3
|
+
ALGORITHM = 'AES-128-CBC'.freeze
|
4
4
|
|
5
|
-
def
|
6
|
-
|
5
|
+
def initialize(key)
|
6
|
+
@key = key
|
7
7
|
end
|
8
8
|
|
9
|
-
def
|
10
|
-
|
9
|
+
def build_encryptor
|
10
|
+
Aes::Encryptor.new(cipher(:encrypt))
|
11
11
|
end
|
12
12
|
|
13
|
-
def
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
13
|
+
def build_decryptor
|
14
|
+
Aes::Decryptor.new(cipher(:decrypt))
|
15
|
+
end
|
16
|
+
|
17
|
+
private
|
18
|
+
|
19
|
+
attr_reader :key
|
20
|
+
|
21
|
+
def cipher(mode)
|
22
|
+
OpenSSL::Cipher.new(ALGORITHM).tap do |cipher|
|
23
|
+
cipher.public_send(mode.to_sym)
|
24
|
+
cipher.key = key
|
25
|
+
end
|
19
26
|
end
|
20
27
|
end
|
21
28
|
end
|
@@ -0,0 +1,26 @@
|
|
1
|
+
class PgExport
|
2
|
+
class Aes
|
3
|
+
class Base
|
4
|
+
include Logging
|
5
|
+
|
6
|
+
def initialize(cipher)
|
7
|
+
@cipher = cipher
|
8
|
+
end
|
9
|
+
|
10
|
+
private
|
11
|
+
|
12
|
+
def copy_using(cipher, from:, to:)
|
13
|
+
cipher.reset
|
14
|
+
to.open(:write) do |f|
|
15
|
+
from.each_chunk do |chunk|
|
16
|
+
f << cipher.update(chunk)
|
17
|
+
end
|
18
|
+
f << cipher.final
|
19
|
+
end
|
20
|
+
self
|
21
|
+
end
|
22
|
+
|
23
|
+
attr_reader :cipher
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
@@ -0,0 +1,32 @@
|
|
1
|
+
class PgExport
|
2
|
+
class BashUtils
|
3
|
+
include Logging
|
4
|
+
|
5
|
+
def initialize(database_name)
|
6
|
+
@database_name = database_name
|
7
|
+
end
|
8
|
+
|
9
|
+
def create_dump
|
10
|
+
dump = PlainDump.new
|
11
|
+
Open3.popen3("pg_dump -Fc --file #{dump.path} #{database_name}") do |_, _, err|
|
12
|
+
error = err.read
|
13
|
+
raise PgDumpError, error unless error.empty?
|
14
|
+
end
|
15
|
+
logger.info "Create #{dump}"
|
16
|
+
dump
|
17
|
+
end
|
18
|
+
|
19
|
+
def restore_dump(dump, restore_database_name)
|
20
|
+
Open3.popen3("pg_restore -c -d #{restore_database_name} #{dump.path}") do |_, _, err|
|
21
|
+
error = err.read
|
22
|
+
raise PgRestoreError, error if /FATAL/ =~ error
|
23
|
+
end
|
24
|
+
logger.info "Restore #{dump}"
|
25
|
+
self
|
26
|
+
end
|
27
|
+
|
28
|
+
private
|
29
|
+
|
30
|
+
attr_reader :database_name
|
31
|
+
end
|
32
|
+
end
|
@@ -5,41 +5,41 @@ class PgExport
|
|
5
5
|
TIMESTAMP = '_%Y%m%d_%H%M%S'.freeze
|
6
6
|
TIMESTAMP_REGEX = '[0-9]{8}_[0-9]{6}'.freeze
|
7
7
|
|
8
|
-
def initialize(
|
9
|
-
@
|
8
|
+
def initialize(ftp_adapter, name, keep)
|
9
|
+
@ftp_adapter, @name, @keep = ftp_adapter, name, keep
|
10
10
|
end
|
11
11
|
|
12
12
|
def upload(dump)
|
13
13
|
dump_name = timestamped_name(dump)
|
14
|
-
|
15
|
-
logger.info "Upload #{dump} #{dump_name} to #{
|
14
|
+
ftp_adapter.upload_file(dump.path, dump_name)
|
15
|
+
logger.info "Upload #{dump} #{dump_name} to #{ftp_adapter}"
|
16
16
|
end
|
17
17
|
|
18
18
|
def download(name)
|
19
19
|
dump = EncryptedDump.new
|
20
|
-
|
21
|
-
logger.info "Download #{dump} #{name} from #{
|
20
|
+
ftp_adapter.download_file(dump.path, name)
|
21
|
+
logger.info "Download #{dump} #{name} from #{ftp_adapter}"
|
22
22
|
dump
|
23
23
|
end
|
24
24
|
|
25
|
-
def remove_old
|
25
|
+
def remove_old
|
26
26
|
find_by_name(name).drop(keep.to_i).each do |filename|
|
27
|
-
|
28
|
-
logger.info "Remove #{filename} from #{
|
27
|
+
ftp_adapter.delete(filename)
|
28
|
+
logger.info "Remove #{filename} from #{ftp_adapter}"
|
29
29
|
end
|
30
30
|
end
|
31
31
|
|
32
32
|
def find_by_name(s)
|
33
|
-
|
33
|
+
ftp_adapter.list(s + '_*')
|
34
34
|
end
|
35
35
|
|
36
36
|
def all
|
37
|
-
|
37
|
+
ftp_adapter.list('*')
|
38
38
|
end
|
39
39
|
|
40
40
|
private
|
41
41
|
|
42
|
-
attr_reader :
|
42
|
+
attr_reader :ftp_adapter, :name, :keep
|
43
43
|
|
44
44
|
def timestamped_name(dump)
|
45
45
|
name + Time.now.strftime(TIMESTAMP) + dump.ext
|
@@ -1,11 +1,13 @@
|
|
1
1
|
class PgExport
|
2
|
-
class
|
2
|
+
class FtpAdapter
|
3
|
+
extend Forwardable
|
4
|
+
|
3
5
|
CHUNK_SIZE = (2**16).freeze
|
4
6
|
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
@
|
7
|
+
def_delegators :connection, :ftp, :host
|
8
|
+
|
9
|
+
def initialize(connection)
|
10
|
+
@connection = connection
|
9
11
|
ObjectSpace.define_finalizer(self, proc { connection.close })
|
10
12
|
end
|
11
13
|
|
@@ -31,6 +33,6 @@ class PgExport
|
|
31
33
|
|
32
34
|
private
|
33
35
|
|
34
|
-
attr_reader :
|
36
|
+
attr_reader :connection
|
35
37
|
end
|
36
38
|
end
|
@@ -0,0 +1,28 @@
|
|
1
|
+
class PgExport
|
2
|
+
class FtpConnection
|
3
|
+
include Logging
|
4
|
+
|
5
|
+
attr_reader :ftp, :host
|
6
|
+
|
7
|
+
def initialize(host:, user:, password:)
|
8
|
+
@host, @user, @password = host, user, password
|
9
|
+
end
|
10
|
+
|
11
|
+
def open
|
12
|
+
@ftp = Net::FTP.new(host, user, password)
|
13
|
+
@ftp.passive = true
|
14
|
+
logger.info "Connect to #{host}"
|
15
|
+
self
|
16
|
+
end
|
17
|
+
|
18
|
+
def close
|
19
|
+
@ftp.close
|
20
|
+
logger.info 'Close FTP'
|
21
|
+
self
|
22
|
+
end
|
23
|
+
|
24
|
+
private
|
25
|
+
|
26
|
+
attr_reader :user, :password
|
27
|
+
end
|
28
|
+
end
|
data/lib/pg_export/version.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: pg_export
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.5.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Krzysztof Maicher
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2017-03-11 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: cli_spinnable
|
@@ -137,6 +137,7 @@ files:
|
|
137
137
|
- ".rspec"
|
138
138
|
- ".rubocop.yml"
|
139
139
|
- ".travis.yml"
|
140
|
+
- CHANGELOG.md
|
140
141
|
- CODE_OF_CONDUCT.md
|
141
142
|
- Gemfile
|
142
143
|
- LICENSE.txt
|
@@ -146,21 +147,24 @@ files:
|
|
146
147
|
- bin/pg_export
|
147
148
|
- bin/setup
|
148
149
|
- lib/pg_export.rb
|
149
|
-
- lib/pg_export/concurrency.rb
|
150
150
|
- lib/pg_export/configuration.rb
|
151
151
|
- lib/pg_export/entities/dump/base.rb
|
152
|
-
- lib/pg_export/entities/dump/size_human.rb
|
153
152
|
- lib/pg_export/entities/encrypted_dump.rb
|
154
153
|
- lib/pg_export/entities/plain_dump.rb
|
155
154
|
- lib/pg_export/errors.rb
|
156
|
-
- lib/pg_export/
|
157
|
-
- lib/pg_export/
|
158
|
-
- lib/pg_export/
|
155
|
+
- lib/pg_export/includable_modules/colourable_string.rb
|
156
|
+
- lib/pg_export/includable_modules/dump/size_human.rb
|
157
|
+
- lib/pg_export/includable_modules/interactive.rb
|
158
|
+
- lib/pg_export/includable_modules/logging.rb
|
159
|
+
- lib/pg_export/includable_modules/services_container.rb
|
159
160
|
- lib/pg_export/services/aes.rb
|
161
|
+
- lib/pg_export/services/aes/base.rb
|
162
|
+
- lib/pg_export/services/aes/decryptor.rb
|
163
|
+
- lib/pg_export/services/aes/encryptor.rb
|
164
|
+
- lib/pg_export/services/bash_utils.rb
|
160
165
|
- lib/pg_export/services/dump_storage.rb
|
161
|
-
- lib/pg_export/services/
|
162
|
-
- lib/pg_export/services/
|
163
|
-
- lib/pg_export/services/utils.rb
|
166
|
+
- lib/pg_export/services/ftp_adapter.rb
|
167
|
+
- lib/pg_export/services/ftp_connection.rb
|
164
168
|
- lib/pg_export/version.rb
|
165
169
|
- pg_export.gemspec
|
166
170
|
homepage: https://github.com/maicher/pg_export
|
@@ -183,7 +187,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
183
187
|
version: '0'
|
184
188
|
requirements: []
|
185
189
|
rubyforge_project:
|
186
|
-
rubygems_version: 2.
|
190
|
+
rubygems_version: 2.6.8
|
187
191
|
signing_key:
|
188
192
|
specification_version: 4
|
189
193
|
summary: CLI for creating and exporting PostgreSQL dumps to FTP.
|
@@ -1,21 +0,0 @@
|
|
1
|
-
class PgExport
|
2
|
-
module Concurrency
|
3
|
-
class ThreadsArray < Array
|
4
|
-
def <<(job)
|
5
|
-
super Thread.new { job }
|
6
|
-
end
|
7
|
-
|
8
|
-
alias push <<
|
9
|
-
end
|
10
|
-
|
11
|
-
def self.included(*)
|
12
|
-
Thread.abort_on_exception = true
|
13
|
-
end
|
14
|
-
|
15
|
-
def concurrently
|
16
|
-
t = ThreadsArray.new
|
17
|
-
yield t
|
18
|
-
t.each(&:join)
|
19
|
-
end
|
20
|
-
end
|
21
|
-
end
|
@@ -1,19 +0,0 @@
|
|
1
|
-
class PgExport
|
2
|
-
module Interactive
|
3
|
-
module ColourableString
|
4
|
-
refine String do
|
5
|
-
def red
|
6
|
-
"\e[31m#{self}\e[0m"
|
7
|
-
end
|
8
|
-
|
9
|
-
def green
|
10
|
-
"\e[0;32;49m#{self}\e[0m"
|
11
|
-
end
|
12
|
-
|
13
|
-
def gray
|
14
|
-
"\e[37m#{self}\e[0m"
|
15
|
-
end
|
16
|
-
end
|
17
|
-
end
|
18
|
-
end
|
19
|
-
end
|
data/lib/pg_export/logging.rb
DELETED
@@ -1,31 +0,0 @@
|
|
1
|
-
class PgExport
|
2
|
-
class FtpService
|
3
|
-
class Connection
|
4
|
-
include Logging
|
5
|
-
|
6
|
-
attr_reader :ftp
|
7
|
-
|
8
|
-
def initialize(host:, user:, password:)
|
9
|
-
@host, @user, @password = host, user, password
|
10
|
-
open
|
11
|
-
end
|
12
|
-
|
13
|
-
def open
|
14
|
-
@ftp = Net::FTP.new(host, user, password)
|
15
|
-
@ftp.passive = true
|
16
|
-
logger.info "Connect to #{host}"
|
17
|
-
self
|
18
|
-
end
|
19
|
-
|
20
|
-
def close
|
21
|
-
@ftp.close
|
22
|
-
logger.info 'Close FTP'
|
23
|
-
self
|
24
|
-
end
|
25
|
-
|
26
|
-
private
|
27
|
-
|
28
|
-
attr_reader :host, :user, :password
|
29
|
-
end
|
30
|
-
end
|
31
|
-
end
|
@@ -1,57 +0,0 @@
|
|
1
|
-
class PgExport
|
2
|
-
class Utils
|
3
|
-
include Logging
|
4
|
-
|
5
|
-
def initialize(encryptor, decryptor)
|
6
|
-
@encryptor, @decryptor = encryptor, decryptor
|
7
|
-
end
|
8
|
-
|
9
|
-
def create_dump(database_name)
|
10
|
-
dump = PlainDump.new
|
11
|
-
Open3.popen3("pg_dump -Fc --file #{dump.path} #{database_name}") do |_, _, err|
|
12
|
-
error = err.read
|
13
|
-
raise PgDumpError, error unless error.empty?
|
14
|
-
end
|
15
|
-
logger.info "Create #{dump}"
|
16
|
-
dump
|
17
|
-
end
|
18
|
-
|
19
|
-
def restore_dump(dump, database_name)
|
20
|
-
Open3.popen3("pg_restore -c -d #{database_name} #{dump.path}") do |_, _, err|
|
21
|
-
error = err.read
|
22
|
-
raise PgRestoreError, error if /FATAL/ =~ error
|
23
|
-
end
|
24
|
-
logger.info "Restore #{dump}"
|
25
|
-
self
|
26
|
-
end
|
27
|
-
|
28
|
-
def encrypt(dump)
|
29
|
-
enc_dump = EncryptedDump.new
|
30
|
-
copy_using(encryptor, from: dump, to: enc_dump)
|
31
|
-
logger.info "Create #{enc_dump}"
|
32
|
-
enc_dump
|
33
|
-
end
|
34
|
-
|
35
|
-
def decrypt(enc_dump)
|
36
|
-
dump = PlainDump.new
|
37
|
-
copy_using(decryptor, from: enc_dump, to: dump)
|
38
|
-
logger.info "Create #{dump}"
|
39
|
-
dump
|
40
|
-
end
|
41
|
-
|
42
|
-
private
|
43
|
-
|
44
|
-
attr_reader :encryptor, :decryptor
|
45
|
-
|
46
|
-
def copy_using(aes, from:, to:)
|
47
|
-
aes.reset
|
48
|
-
to.open(:write) do |f|
|
49
|
-
from.each_chunk do |chunk|
|
50
|
-
f << aes.update(chunk)
|
51
|
-
end
|
52
|
-
f << aes.final
|
53
|
-
end
|
54
|
-
self
|
55
|
-
end
|
56
|
-
end
|
57
|
-
end
|