pg_export 0.2.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.rubocop.yml +1 -5
- data/README.md +25 -16
- data/bin/pg_export +2 -3
- data/lib/pg_export.rb +24 -36
- data/lib/pg_export/concurrency.rb +21 -0
- data/lib/pg_export/configuration.rb +0 -2
- data/lib/pg_export/entities/compressed_dump.rb +19 -0
- data/lib/pg_export/entities/dump/base.rb +32 -0
- data/lib/pg_export/entities/dump/size_human.rb +15 -0
- data/lib/pg_export/entities/sql_dump.rb +15 -0
- data/lib/pg_export/errors.rb +1 -3
- data/lib/pg_export/services/dump_storage.rb +37 -0
- data/lib/pg_export/{ftp_service.rb → services/ftp_service.rb} +10 -3
- data/lib/pg_export/{ftp_service → services/ftp_service}/connection.rb +0 -0
- data/lib/pg_export/services/utils.rb +34 -0
- data/lib/pg_export/version.rb +1 -1
- data/pg_export.gemspec +5 -6
- metadata +28 -28
- data/lib/pg_export/actions.rb +0 -5
- data/lib/pg_export/actions/compress_dump.rb +0 -39
- data/lib/pg_export/actions/create_dump.rb +0 -35
- data/lib/pg_export/actions/remove_old_dumps.rb +0 -25
- data/lib/pg_export/actions/remove_old_dumps_from_ftp.rb +0 -22
- data/lib/pg_export/actions/send_dump_to_ftp.rb +0 -19
- data/lib/pg_export/dump.rb +0 -61
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: ed370ed90634fe7db175d5e9f93ffb9d423dad07
|
4
|
+
data.tar.gz: 901040b143cef0587fcf9d14f9a677fbbf8fcb54
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: ae0a892e79679675d220fc0a91b9c579e4083f9e1c5791c66bd0ab4fcb6d4097da8c242567a0be935d61ab084d7b392f05a4bbb19ac1f63564aee36e8441fae9
|
7
|
+
data.tar.gz: 19fd6f4cd3eb192dd8b6a0a93f27badb704c9116fdb1ed6c9fd54efb08214e18f801b54d1a6ee5286606fe13453adff294cfcf2b91a6f1438807d6e7ff0047d1
|
data/.rubocop.yml
CHANGED
@@ -19,11 +19,7 @@ Lint/NestedMethodDefinition:
|
|
19
19
|
|
20
20
|
Lint/HandleExceptions:
|
21
21
|
Exclude:
|
22
|
-
- 'spec/
|
23
|
-
|
24
|
-
Lint/AssignmentInCondition:
|
25
|
-
Exclude:
|
26
|
-
- 'lib/pg_export/actions/compress_dump.rb'
|
22
|
+
- 'spec/services/utils_spec.rb'
|
27
23
|
|
28
24
|
Lint/AmbiguousOperator:
|
29
25
|
Exclude:
|
data/README.md
CHANGED
@@ -1,15 +1,26 @@
|
|
1
1
|
# PgExport
|
2
2
|
|
3
|
-
CLI for exporting PostgreSQL
|
3
|
+
CLI for creating and exporting PostgreSQL dumps to FTP.
|
4
4
|
|
5
5
|
Can be used for backups or synchronizing databases between production and development environments.
|
6
6
|
|
7
|
-
|
7
|
+
Example:
|
8
|
+
|
9
|
+
pg_export --database database_name -keep 5
|
10
|
+
|
11
|
+
Above command will perform database dump, gzip it, upload it to FTP and remove old dumps from FTP, keeping newest 5.
|
12
|
+
|
13
|
+
FTP connection params are configured by env variables.
|
14
|
+
|
15
|
+
Features:
|
16
|
+
|
17
|
+
- uses shell command pg_dump
|
18
|
+
- no external gem dependencies
|
19
|
+
- uses ruby tempfiles, so local dumps are garbage collected automatically
|
8
20
|
|
9
21
|
## Dependencies
|
10
22
|
|
11
23
|
* Ruby >= 2.1
|
12
|
-
* PostgreSQL >= 9.1
|
13
24
|
* $ pg_dump
|
14
25
|
|
15
26
|
## Installation
|
@@ -33,12 +44,14 @@ Or install it yourself as:
|
|
33
44
|
$ pg_export -h
|
34
45
|
|
35
46
|
Usage: pg_export [options]
|
36
|
-
-c, --configuration Prints the configuration
|
37
|
-
-f, --ftp Tries connecting to FTP to verify the connection
|
38
47
|
-d, --database DATABASE [Required] Name of the database to export
|
39
|
-
-k, --keep [KEEP] [Optional] Number of dump files to keep
|
48
|
+
-k, --keep [KEEP] [Optional] Number of dump files to keep on FTP (default: 10)
|
40
49
|
-t, --timestamped [Optional] Enables log messages with timestamps
|
41
50
|
-h, --help Show this message
|
51
|
+
|
52
|
+
Setting can be verified by running following commands:
|
53
|
+
-c, --configuration Prints the configuration
|
54
|
+
-f, --ftp Tries connecting to FTP to verify the connection
|
42
55
|
|
43
56
|
## How to start
|
44
57
|
|
@@ -54,17 +67,13 @@ Password cannot include `#` sign, [more info](http://serverfault.com/questions/5
|
|
54
67
|
__Step 2.__ Configure other variables (optional).
|
55
68
|
|
56
69
|
# /etc/environment
|
57
|
-
|
58
|
-
KEEP_DUMPS=3 # default: 10
|
59
|
-
KEEP_FTP_DUMPS=5 # default: 10
|
70
|
+
KEEP_DUMPS=5 # default: 10
|
60
71
|
|
61
|
-
__Step 3.__ Print the configuration to verify
|
72
|
+
__Step 3.__ Print the configuration to verify whether env variables has been loaded.
|
62
73
|
|
63
74
|
$ pg_export --configuration
|
64
75
|
=> database:
|
65
|
-
|
66
|
-
keep_dumps: 3
|
67
|
-
keep_ftp_dumps: 5
|
76
|
+
keep_dumps: 5
|
68
77
|
ftp_host: yourftp.example.com
|
69
78
|
ftp_user: user
|
70
79
|
ftp_password: pass***
|
@@ -78,10 +87,10 @@ __Step 4.__ Try connecting to FTP to verify the connection.
|
|
78
87
|
__Step 5.__ Perform database export.
|
79
88
|
|
80
89
|
$ pg_export -d your_database
|
81
|
-
=> Dump
|
82
|
-
|
90
|
+
=> Create Dump Tempfile (1.36MB)
|
91
|
+
Create Compressed Dump Tempfile (1.34MB)
|
83
92
|
Connect to yourftp.example.com
|
84
|
-
Export your_database_20161020_125747.gz to
|
93
|
+
Export Compressed Dump Tempfile (1.34MB) your_database_20161020_125747.gz to yourftp.example.com
|
85
94
|
Close FTP
|
86
95
|
|
87
96
|
## Development
|
data/bin/pg_export
CHANGED
@@ -15,7 +15,7 @@ option_parser = OptionParser.new do |opts|
|
|
15
15
|
options.database = database
|
16
16
|
end
|
17
17
|
|
18
|
-
opts.on('-k', '--keep [KEEP]', Integer, '[Optional] Number of dump files to keep
|
18
|
+
opts.on('-k', '--keep [KEEP]', Integer, '[Optional] Number of dump files to keep on FTP (default: 10)') do |keep|
|
19
19
|
options.keep = keep
|
20
20
|
end
|
21
21
|
|
@@ -50,11 +50,10 @@ begin
|
|
50
50
|
PgExport.new do |config|
|
51
51
|
config.database = options.database if options.database
|
52
52
|
config.keep_dumps = options.keep if options.keep
|
53
|
-
config.keep_ftp_dumps = options.keep if options.keep
|
54
53
|
end.call
|
55
54
|
rescue OptionParser::MissingArgument, PgExport::InvalidConfigurationError => e
|
56
55
|
puts "Error: #{e}"
|
57
56
|
puts option_parser
|
58
|
-
rescue PgExport::
|
57
|
+
rescue PgExport::PgDumpError => e
|
59
58
|
puts e
|
60
59
|
end
|
data/lib/pg_export.rb
CHANGED
@@ -1,22 +1,24 @@
|
|
1
1
|
require 'logger'
|
2
|
-
require '
|
3
|
-
require 'fileutils'
|
2
|
+
require 'tempfile'
|
4
3
|
require 'zlib'
|
5
4
|
require 'net/ftp'
|
6
5
|
|
7
|
-
require 'pg'
|
8
|
-
|
9
6
|
require 'pg_export/version'
|
10
7
|
require 'pg_export/logging'
|
11
|
-
require 'pg_export/configuration'
|
12
8
|
require 'pg_export/errors'
|
13
|
-
require 'pg_export/
|
14
|
-
require 'pg_export/
|
15
|
-
require 'pg_export/
|
16
|
-
require 'pg_export/
|
9
|
+
require 'pg_export/configuration'
|
10
|
+
require 'pg_export/concurrency'
|
11
|
+
require 'pg_export/entities/dump/size_human'
|
12
|
+
require 'pg_export/entities/dump/base'
|
13
|
+
require 'pg_export/entities/sql_dump'
|
14
|
+
require 'pg_export/entities/compressed_dump'
|
15
|
+
require 'pg_export/services/ftp_service'
|
16
|
+
require 'pg_export/services/ftp_service/connection'
|
17
|
+
require 'pg_export/services/utils'
|
18
|
+
require 'pg_export/services/dump_storage'
|
17
19
|
|
18
20
|
class PgExport
|
19
|
-
include
|
21
|
+
include Concurrency
|
20
22
|
|
21
23
|
def initialize
|
22
24
|
@config = Configuration.new
|
@@ -25,42 +27,28 @@ class PgExport
|
|
25
27
|
end
|
26
28
|
|
27
29
|
def call
|
28
|
-
initialize_dump
|
29
30
|
concurrently do |job|
|
30
|
-
job <<
|
31
|
-
job <<
|
31
|
+
job << create_dump
|
32
|
+
job << initialize_dump_storage
|
32
33
|
end
|
33
|
-
|
34
|
+
dump_storage.upload(dump)
|
35
|
+
dump_storage.remove_old(keep: config.keep_dumps)
|
34
36
|
self
|
35
37
|
end
|
36
38
|
|
37
39
|
private
|
38
40
|
|
39
41
|
attr_reader :config
|
40
|
-
attr_accessor :
|
41
|
-
|
42
|
-
def initialize_dump
|
43
|
-
self.dump = Dump.new(config.database, config.dumpfile_dir)
|
44
|
-
end
|
45
|
-
|
46
|
-
def initialize_ftp_service
|
47
|
-
self.ftp_service = FtpService.new(config.ftp_params)
|
48
|
-
end
|
49
|
-
|
50
|
-
def concurrently
|
51
|
-
t = []
|
52
|
-
yield t
|
53
|
-
t.each(&:join)
|
54
|
-
end
|
42
|
+
attr_accessor :dump, :dump_storage
|
55
43
|
|
56
|
-
def
|
57
|
-
|
58
|
-
|
59
|
-
RemoveOldDumps.new(dump, config.keep_dumps).call
|
44
|
+
def initialize_dump_storage
|
45
|
+
ftp_service = FtpService.new(config.ftp_params)
|
46
|
+
self.dump_storage = DumpStorage.new(ftp_service, config.database)
|
60
47
|
end
|
61
48
|
|
62
|
-
def
|
63
|
-
|
64
|
-
|
49
|
+
def create_dump
|
50
|
+
sql_dump = Utils.create_dump(config.database)
|
51
|
+
compressed_dump = Utils.compress(sql_dump)
|
52
|
+
self.dump = compressed_dump
|
65
53
|
end
|
66
54
|
end
|
@@ -0,0 +1,21 @@
|
|
1
|
+
class PgExport
|
2
|
+
module Concurrency
|
3
|
+
class ThreadsArray < Array
|
4
|
+
def <<(job)
|
5
|
+
super Thread.new { job }
|
6
|
+
end
|
7
|
+
|
8
|
+
alias push <<
|
9
|
+
end
|
10
|
+
|
11
|
+
def self.included(*)
|
12
|
+
Thread.abort_on_exception = true
|
13
|
+
end
|
14
|
+
|
15
|
+
def concurrently
|
16
|
+
t = ThreadsArray.new
|
17
|
+
yield t
|
18
|
+
t.each(&:join)
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
@@ -2,9 +2,7 @@ class PgExport
|
|
2
2
|
class Configuration
|
3
3
|
DEFAULTS = {
|
4
4
|
database: nil,
|
5
|
-
dumpfile_dir: ENV['DUMPFILE_DIR'] || 'tmp/dumps',
|
6
5
|
keep_dumps: ENV['KEEP_DUMPS'] || 10,
|
7
|
-
keep_ftp_dumps: ENV['KEEP_FTP_DUMPS'] || 10,
|
8
6
|
ftp_host: ENV['BACKUP_FTP_HOST'],
|
9
7
|
ftp_user: ENV['BACKUP_FTP_USER'],
|
10
8
|
ftp_password: ENV['BACKUP_FTP_PASSWORD']
|
@@ -0,0 +1,19 @@
|
|
1
|
+
class PgExport
|
2
|
+
class CompressedDump < Dump::Base
|
3
|
+
def name
|
4
|
+
'Compressed Dump'
|
5
|
+
end
|
6
|
+
|
7
|
+
def ext
|
8
|
+
'.gz'
|
9
|
+
end
|
10
|
+
|
11
|
+
def open(operation_type, &block)
|
12
|
+
case operation_type.to_sym
|
13
|
+
when :read then Zlib::GzipReader.open(path, &block)
|
14
|
+
when :write then Zlib::GzipWriter.open(path, &block)
|
15
|
+
else raise ArgumentError, 'Operation type can be only :read or :write'
|
16
|
+
end
|
17
|
+
end
|
18
|
+
end
|
19
|
+
end
|
@@ -0,0 +1,32 @@
|
|
1
|
+
class PgExport
|
2
|
+
module Dump
|
3
|
+
class Base
|
4
|
+
extend Forwardable
|
5
|
+
include SizeHuman
|
6
|
+
|
7
|
+
CHUNK_SIZE = (2**16).freeze
|
8
|
+
|
9
|
+
def_delegators :file, :path, :read, :write, :rewind, :size, :eof?
|
10
|
+
|
11
|
+
def initialize
|
12
|
+
@file = Tempfile.new('dump')
|
13
|
+
end
|
14
|
+
|
15
|
+
def ext
|
16
|
+
raise 'Overwrite it'
|
17
|
+
end
|
18
|
+
|
19
|
+
def read_chunk
|
20
|
+
raise 'Overwrite it'
|
21
|
+
end
|
22
|
+
|
23
|
+
def to_s
|
24
|
+
"#{name || self.class} #{file.class} (#{size_human})"
|
25
|
+
end
|
26
|
+
|
27
|
+
private
|
28
|
+
|
29
|
+
attr_reader :file
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
@@ -0,0 +1,15 @@
|
|
1
|
+
class PgExport
|
2
|
+
module Dump
|
3
|
+
module SizeHuman
|
4
|
+
def size_human
|
5
|
+
{
|
6
|
+
'B' => 1024,
|
7
|
+
'kB' => 1024 * 1024,
|
8
|
+
'MB' => 1024 * 1024 * 1024,
|
9
|
+
'GB' => 1024 * 1024 * 1024 * 1024,
|
10
|
+
'TB' => 1024 * 1024 * 1024 * 1024 * 1024
|
11
|
+
}.each_pair { |e, s| return "#{(size.to_f / (s / 1024)).round(2)}#{e}" if size < s }
|
12
|
+
end
|
13
|
+
end
|
14
|
+
end
|
15
|
+
end
|
data/lib/pg_export/errors.rb
CHANGED
@@ -1,6 +1,4 @@
|
|
1
1
|
class PgExport
|
2
|
-
class DependencyRequiredError < StandardError; end
|
3
|
-
class DatabaseDoesNotExistError < StandardError; end
|
4
|
-
class DumpFileDoesNotExistError < StandardError; end
|
5
2
|
class InvalidConfigurationError < StandardError; end
|
3
|
+
class PgDumpError < StandardError; end
|
6
4
|
end
|
@@ -0,0 +1,37 @@
|
|
1
|
+
class PgExport
|
2
|
+
class DumpStorage
|
3
|
+
include Logging
|
4
|
+
|
5
|
+
TIMESTAMP = '_%Y%m%d_%H%M%S'.freeze
|
6
|
+
TIMESTAMP_REGEX = '[0-9]{8}_[0-9]{6}'.freeze
|
7
|
+
|
8
|
+
def initialize(ftp_service, name)
|
9
|
+
@ftp_service, @name = ftp_service, name
|
10
|
+
end
|
11
|
+
|
12
|
+
def upload(dump)
|
13
|
+
dump_name = timestamped_name(dump)
|
14
|
+
ftp_service.upload_file(dump.path, dump_name)
|
15
|
+
logger.info "Export #{dump} #{dump_name} to #{ftp_service}"
|
16
|
+
end
|
17
|
+
|
18
|
+
def remove_old(keep:)
|
19
|
+
ftp_service.list(ftp_regex).drop(keep.to_i).each do |filename|
|
20
|
+
ftp_service.delete(filename)
|
21
|
+
logger.info "Remove #{filename} from FTP"
|
22
|
+
end
|
23
|
+
end
|
24
|
+
|
25
|
+
private
|
26
|
+
|
27
|
+
attr_reader :ftp_service, :name
|
28
|
+
|
29
|
+
def timestamped_name(dump)
|
30
|
+
name + Time.now.strftime(TIMESTAMP) + dump.ext
|
31
|
+
end
|
32
|
+
|
33
|
+
def ftp_regex
|
34
|
+
name + '_*'
|
35
|
+
end
|
36
|
+
end
|
37
|
+
end
|
@@ -1,6 +1,9 @@
|
|
1
1
|
class PgExport
|
2
2
|
class FtpService
|
3
|
+
CHUNK_SIZE = (2**16).freeze
|
4
|
+
|
3
5
|
def initialize(params)
|
6
|
+
@host = params.fetch(:host)
|
4
7
|
connection = Connection.new(params)
|
5
8
|
@ftp = connection.ftp
|
6
9
|
ObjectSpace.define_finalizer(self, proc { connection.close })
|
@@ -14,12 +17,16 @@ class PgExport
|
|
14
17
|
ftp.delete(filename)
|
15
18
|
end
|
16
19
|
|
17
|
-
def upload_file(path)
|
18
|
-
ftp.putbinaryfile(path.to_s)
|
20
|
+
def upload_file(path, name)
|
21
|
+
ftp.putbinaryfile(path.to_s, name, CHUNK_SIZE)
|
22
|
+
end
|
23
|
+
|
24
|
+
def to_s
|
25
|
+
host
|
19
26
|
end
|
20
27
|
|
21
28
|
private
|
22
29
|
|
23
|
-
attr_reader :ftp
|
30
|
+
attr_reader :ftp, :host
|
24
31
|
end
|
25
32
|
end
|
File without changes
|
@@ -0,0 +1,34 @@
|
|
1
|
+
class PgExport
|
2
|
+
class Utils
|
3
|
+
extend Logging
|
4
|
+
|
5
|
+
def self.create_dump(database_name)
|
6
|
+
dump = SqlDump.new
|
7
|
+
out = `pg_dump -Fc --file #{dump.path} #{database_name} 2>&1`
|
8
|
+
raise PgDumpError, out if /FATAL/ =~ out
|
9
|
+
logger.info "Create #{dump}"
|
10
|
+
dump
|
11
|
+
end
|
12
|
+
|
13
|
+
def self.compress(dump)
|
14
|
+
dump_gz = CompressedDump.new
|
15
|
+
dump_gz.open(:write) do |gz|
|
16
|
+
gz.write(dump.read_chunk) until dump.eof?
|
17
|
+
end
|
18
|
+
|
19
|
+
logger.info "Create #{dump_gz}"
|
20
|
+
dump_gz
|
21
|
+
end
|
22
|
+
|
23
|
+
def self.decompress(dump_gz)
|
24
|
+
dump = SqlDump.new
|
25
|
+
dump_gz.open(:read) do |gz|
|
26
|
+
dump.write(gz.readpartial(Dump::Base::CHUNK_SIZE)) until gz.eof?
|
27
|
+
end
|
28
|
+
dump.rewind
|
29
|
+
|
30
|
+
logger.info "Create #{dump}"
|
31
|
+
dump
|
32
|
+
end
|
33
|
+
end
|
34
|
+
end
|
data/lib/pg_export/version.rb
CHANGED
data/pg_export.gemspec
CHANGED
@@ -9,9 +9,9 @@ Gem::Specification.new do |spec|
|
|
9
9
|
spec.authors = ['Krzysztof Maicher']
|
10
10
|
spec.email = ['krzysztof.maicher@gmail.com']
|
11
11
|
|
12
|
-
spec.summary = 'CLI for
|
13
|
-
spec.description = "CLI for
|
14
|
-
Can be used for backups or synchronizing databases between production and development environments"
|
12
|
+
spec.summary = 'CLI for creating and exporting PostgreSQL dumps to FTP.'
|
13
|
+
spec.description = "CLI for creating and exporting PostgreSQL dumps to FTP.\
|
14
|
+
Can be used for backups or synchronizing databases between production and development environments."
|
15
15
|
spec.homepage = 'https://github.com/maicher/pg_export'
|
16
16
|
spec.license = 'MIT'
|
17
17
|
|
@@ -20,11 +20,10 @@ Gem::Specification.new do |spec|
|
|
20
20
|
spec.require_paths = ['lib']
|
21
21
|
spec.required_ruby_version = '>= 2.1.0'
|
22
22
|
|
23
|
-
spec.add_dependency 'pg', '>= 0.16'
|
24
|
-
|
25
23
|
spec.add_development_dependency 'bundler', '~> 1.10'
|
26
24
|
spec.add_development_dependency 'rubocop', '~> 0.44'
|
27
25
|
spec.add_development_dependency 'rake', '~> 10.0'
|
28
|
-
spec.add_development_dependency 'rspec', '~>3.4'
|
26
|
+
spec.add_development_dependency 'rspec', '~> 3.4'
|
27
|
+
spec.add_development_dependency 'pg', '~> 0.19'
|
29
28
|
spec.add_development_dependency 'pry'
|
30
29
|
end
|
metadata
CHANGED
@@ -1,29 +1,15 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: pg_export
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.3.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Krzysztof Maicher
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2016-10-
|
11
|
+
date: 2016-10-25 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
|
-
- !ruby/object:Gem::Dependency
|
14
|
-
name: pg
|
15
|
-
requirement: !ruby/object:Gem::Requirement
|
16
|
-
requirements:
|
17
|
-
- - ">="
|
18
|
-
- !ruby/object:Gem::Version
|
19
|
-
version: '0.16'
|
20
|
-
type: :runtime
|
21
|
-
prerelease: false
|
22
|
-
version_requirements: !ruby/object:Gem::Requirement
|
23
|
-
requirements:
|
24
|
-
- - ">="
|
25
|
-
- !ruby/object:Gem::Version
|
26
|
-
version: '0.16'
|
27
13
|
- !ruby/object:Gem::Dependency
|
28
14
|
name: bundler
|
29
15
|
requirement: !ruby/object:Gem::Requirement
|
@@ -80,6 +66,20 @@ dependencies:
|
|
80
66
|
- - "~>"
|
81
67
|
- !ruby/object:Gem::Version
|
82
68
|
version: '3.4'
|
69
|
+
- !ruby/object:Gem::Dependency
|
70
|
+
name: pg
|
71
|
+
requirement: !ruby/object:Gem::Requirement
|
72
|
+
requirements:
|
73
|
+
- - "~>"
|
74
|
+
- !ruby/object:Gem::Version
|
75
|
+
version: '0.19'
|
76
|
+
type: :development
|
77
|
+
prerelease: false
|
78
|
+
version_requirements: !ruby/object:Gem::Requirement
|
79
|
+
requirements:
|
80
|
+
- - "~>"
|
81
|
+
- !ruby/object:Gem::Version
|
82
|
+
version: '0.19'
|
83
83
|
- !ruby/object:Gem::Dependency
|
84
84
|
name: pry
|
85
85
|
requirement: !ruby/object:Gem::Requirement
|
@@ -94,9 +94,9 @@ dependencies:
|
|
94
94
|
- - ">="
|
95
95
|
- !ruby/object:Gem::Version
|
96
96
|
version: '0'
|
97
|
-
description: CLI for
|
97
|
+
description: CLI for creating and exporting PostgreSQL dumps to FTP. Can
|
98
98
|
be used for backups or synchronizing databases between production and development
|
99
|
-
environments
|
99
|
+
environments.
|
100
100
|
email:
|
101
101
|
- krzysztof.maicher@gmail.com
|
102
102
|
executables:
|
@@ -117,18 +117,18 @@ files:
|
|
117
117
|
- bin/pg_export
|
118
118
|
- bin/setup
|
119
119
|
- lib/pg_export.rb
|
120
|
-
- lib/pg_export/
|
121
|
-
- lib/pg_export/actions/compress_dump.rb
|
122
|
-
- lib/pg_export/actions/create_dump.rb
|
123
|
-
- lib/pg_export/actions/remove_old_dumps.rb
|
124
|
-
- lib/pg_export/actions/remove_old_dumps_from_ftp.rb
|
125
|
-
- lib/pg_export/actions/send_dump_to_ftp.rb
|
120
|
+
- lib/pg_export/concurrency.rb
|
126
121
|
- lib/pg_export/configuration.rb
|
127
|
-
- lib/pg_export/
|
122
|
+
- lib/pg_export/entities/compressed_dump.rb
|
123
|
+
- lib/pg_export/entities/dump/base.rb
|
124
|
+
- lib/pg_export/entities/dump/size_human.rb
|
125
|
+
- lib/pg_export/entities/sql_dump.rb
|
128
126
|
- lib/pg_export/errors.rb
|
129
|
-
- lib/pg_export/ftp_service.rb
|
130
|
-
- lib/pg_export/ftp_service/connection.rb
|
131
127
|
- lib/pg_export/logging.rb
|
128
|
+
- lib/pg_export/services/dump_storage.rb
|
129
|
+
- lib/pg_export/services/ftp_service.rb
|
130
|
+
- lib/pg_export/services/ftp_service/connection.rb
|
131
|
+
- lib/pg_export/services/utils.rb
|
132
132
|
- lib/pg_export/version.rb
|
133
133
|
- pg_export.gemspec
|
134
134
|
homepage: https://github.com/maicher/pg_export
|
@@ -154,5 +154,5 @@ rubyforge_project:
|
|
154
154
|
rubygems_version: 2.4.8
|
155
155
|
signing_key:
|
156
156
|
specification_version: 4
|
157
|
-
summary: CLI for
|
157
|
+
summary: CLI for creating and exporting PostgreSQL dumps to FTP.
|
158
158
|
test_files: []
|
data/lib/pg_export/actions.rb
DELETED
@@ -1,39 +0,0 @@
|
|
1
|
-
class PgExport
|
2
|
-
class CompressDump
|
3
|
-
include Logging
|
4
|
-
|
5
|
-
def initialize(dump)
|
6
|
-
@dump = dump
|
7
|
-
end
|
8
|
-
|
9
|
-
def call
|
10
|
-
validate_dumpfile_exists
|
11
|
-
compress_dumpfile
|
12
|
-
remove_dumpfile
|
13
|
-
self
|
14
|
-
end
|
15
|
-
|
16
|
-
private
|
17
|
-
|
18
|
-
attr_reader :dump
|
19
|
-
|
20
|
-
def validate_dumpfile_exists
|
21
|
-
File.exist?(dump.pathname) or raise DumpFileDoesNotExistError, "#{dump.pathname} does not exist"
|
22
|
-
end
|
23
|
-
|
24
|
-
def compress_dumpfile
|
25
|
-
Zlib::GzipWriter.open(dump.pathname_gz) do |gz|
|
26
|
-
File.open(dump.pathname) do |fp|
|
27
|
-
while chunk = fp.read(16 * 1024)
|
28
|
-
gz.write chunk
|
29
|
-
end
|
30
|
-
end
|
31
|
-
end
|
32
|
-
logger.info "Zip dump #{dump.basename_gz} (#{dump.size_gz})"
|
33
|
-
end
|
34
|
-
|
35
|
-
def remove_dumpfile
|
36
|
-
File.delete(dump.pathname)
|
37
|
-
end
|
38
|
-
end
|
39
|
-
end
|
@@ -1,35 +0,0 @@
|
|
1
|
-
class PgExport
|
2
|
-
class CreateDump
|
3
|
-
include Logging
|
4
|
-
|
5
|
-
def initialize(dump)
|
6
|
-
@dump = dump
|
7
|
-
end
|
8
|
-
|
9
|
-
def call
|
10
|
-
validate_pg_dump_exists
|
11
|
-
validate_db_exists(dump.database)
|
12
|
-
execute_dump_command
|
13
|
-
logger.info "Dump #{dump.database} to #{dump.pathname} (#{dump.size})"
|
14
|
-
end
|
15
|
-
|
16
|
-
private
|
17
|
-
|
18
|
-
attr_reader :dump
|
19
|
-
|
20
|
-
def validate_pg_dump_exists
|
21
|
-
out = `pg_dump -V`
|
22
|
-
/pg_dump \(PostgreSQL\)/ =~ out or raise DependencyRequiredError, 'Shell command "pg_dump" is required. Pleas install "pg_dump" and try again.'
|
23
|
-
end
|
24
|
-
|
25
|
-
def validate_db_exists(database)
|
26
|
-
PG.connect(dbname: database)
|
27
|
-
rescue PG::ConnectionBad => e
|
28
|
-
raise DatabaseDoesNotExistError, e
|
29
|
-
end
|
30
|
-
|
31
|
-
def execute_dump_command
|
32
|
-
`pg_dump -Fc --file #{dump.pathname} #{dump.database}`
|
33
|
-
end
|
34
|
-
end
|
35
|
-
end
|
@@ -1,25 +0,0 @@
|
|
1
|
-
class PgExport
|
2
|
-
class RemoveOldDumps
|
3
|
-
include Logging
|
4
|
-
|
5
|
-
def initialize(dump, keep_dumps_count)
|
6
|
-
@dump = dump
|
7
|
-
@keep_dumps_count = keep_dumps_count.to_i
|
8
|
-
end
|
9
|
-
|
10
|
-
def call
|
11
|
-
files.sort.reverse.drop(keep_dumps_count).each do |filename|
|
12
|
-
File.delete("#{dump.dirname}/#{filename}")
|
13
|
-
logger.info "Remove file #{filename}"
|
14
|
-
end
|
15
|
-
end
|
16
|
-
|
17
|
-
private
|
18
|
-
|
19
|
-
attr_reader :dump, :keep_dumps_count
|
20
|
-
|
21
|
-
def files
|
22
|
-
Dir.entries(dump.dirname).grep(dump.regexp)
|
23
|
-
end
|
24
|
-
end
|
25
|
-
end
|
@@ -1,22 +0,0 @@
|
|
1
|
-
class PgExport
|
2
|
-
class RemoveOldDumpsFromFtp
|
3
|
-
include Logging
|
4
|
-
|
5
|
-
def initialize(dump, ftp_service, keep_dumps)
|
6
|
-
@dump = dump
|
7
|
-
@ftp_service = ftp_service
|
8
|
-
@keep_dumps = keep_dumps.to_i
|
9
|
-
end
|
10
|
-
|
11
|
-
def call
|
12
|
-
ftp_service.list(dump.ftp_regexp).drop(keep_dumps).each do |filename|
|
13
|
-
ftp_service.delete(filename)
|
14
|
-
logger.info "Remove file #{filename} from FTP"
|
15
|
-
end
|
16
|
-
end
|
17
|
-
|
18
|
-
private
|
19
|
-
|
20
|
-
attr_accessor :dump, :ftp_service, :keep_dumps
|
21
|
-
end
|
22
|
-
end
|
@@ -1,19 +0,0 @@
|
|
1
|
-
class PgExport
|
2
|
-
class SendDumpToFtp
|
3
|
-
include Logging
|
4
|
-
|
5
|
-
def initialize(dump, ftp_service)
|
6
|
-
@dump = dump
|
7
|
-
@ftp_service = ftp_service
|
8
|
-
end
|
9
|
-
|
10
|
-
def call
|
11
|
-
ftp_service.upload_file(dump.pathname_gz)
|
12
|
-
logger.info "Export #{dump.basename_gz} to FTP"
|
13
|
-
end
|
14
|
-
|
15
|
-
private
|
16
|
-
|
17
|
-
attr_reader :dump, :ftp_service
|
18
|
-
end
|
19
|
-
end
|
data/lib/pg_export/dump.rb
DELETED
@@ -1,61 +0,0 @@
|
|
1
|
-
class PgExport
|
2
|
-
class Dump
|
3
|
-
TIMESTAMP_REGEX = '[0-9]{8}_[0-9]{6}'.freeze
|
4
|
-
|
5
|
-
attr_reader :database, :dir, :dirname, :basename, :basename_gz, :regexp, :ftp_regexp, :pathname, :pathname_gz
|
6
|
-
|
7
|
-
def initialize(a_database, a_dir)
|
8
|
-
@database = a_database
|
9
|
-
@dir = a_dir
|
10
|
-
@dirname = absolute_path(dir)
|
11
|
-
@basename = append_to_database(Time.now.strftime('%Y%m%d_%H%M%S'))
|
12
|
-
@basename_gz = basename + '.gz'
|
13
|
-
@regexp = Regexp.new(append_to_database(TIMESTAMP_REGEX))
|
14
|
-
@ftp_regexp = append_to_database('*')
|
15
|
-
@pathname = [dirname, basename].join('/')
|
16
|
-
@pathname_gz = pathname + '.gz'
|
17
|
-
create_dir_if_necessary
|
18
|
-
end
|
19
|
-
|
20
|
-
def size
|
21
|
-
file_size(pathname)
|
22
|
-
end
|
23
|
-
|
24
|
-
def size_gz
|
25
|
-
file_size(pathname_gz)
|
26
|
-
end
|
27
|
-
|
28
|
-
private
|
29
|
-
|
30
|
-
def file_size(path)
|
31
|
-
return unless File.exist?(path)
|
32
|
-
bytes2human(File.size(path))
|
33
|
-
end
|
34
|
-
|
35
|
-
def bytes2human(bytes)
|
36
|
-
{
|
37
|
-
'B' => 1024,
|
38
|
-
'kB' => 1024 * 1024,
|
39
|
-
'MB' => 1024 * 1024 * 1024,
|
40
|
-
'GB' => 1024 * 1024 * 1024 * 1024,
|
41
|
-
'TB' => 1024 * 1024 * 1024 * 1024 * 1024
|
42
|
-
}.each_pair { |e, s| return "#{(bytes.to_f / (s / 1024)).round(2)}#{e}" if bytes < s }
|
43
|
-
end
|
44
|
-
|
45
|
-
def create_dir_if_necessary
|
46
|
-
FileUtils.mkdir_p(dirname)
|
47
|
-
end
|
48
|
-
|
49
|
-
def append_to_database(segment)
|
50
|
-
[database, segment].join('_')
|
51
|
-
end
|
52
|
-
|
53
|
-
def absolute_path(dir)
|
54
|
-
if Pathname.new(dir).absolute?
|
55
|
-
dir
|
56
|
-
else
|
57
|
-
[Pathname.pwd, dir].join('/')
|
58
|
-
end
|
59
|
-
end
|
60
|
-
end
|
61
|
-
end
|