stellar-core-backup 0.0.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.gitignore +18 -0
- data/CONTRIBUTING.md +58 -0
- data/Gemfile +2 -0
- data/LICENSE.txt +202 -0
- data/README.md +92 -0
- data/Rakefile +9 -0
- data/bin/stellar-core-backup +46 -0
- data/config/sample.yaml +7 -0
- data/lib/stellar-core-backup.rb +21 -0
- data/lib/stellar-core-backup/cmd.rb +28 -0
- data/lib/stellar-core-backup/cmd_result.rb +14 -0
- data/lib/stellar-core-backup/config.rb +37 -0
- data/lib/stellar-core-backup/database.rb +67 -0
- data/lib/stellar-core-backup/filesystem.rb +62 -0
- data/lib/stellar-core-backup/job.rb +190 -0
- data/lib/stellar-core-backup/restore/database.rb +36 -0
- data/lib/stellar-core-backup/restore/filesystem.rb +36 -0
- data/lib/stellar-core-backup/s3.rb +64 -0
- data/lib/stellar-core-backup/tar.rb +37 -0
- data/lib/stellar-core-backup/utils.rb +130 -0
- data/lib/stellar-core-backup/version.rb +3 -0
- data/stellar-core-backup.gemspec +26 -0
- metadata +137 -0
@@ -0,0 +1,21 @@
|
|
1
|
+
require "stellar-core-backup/version"
|
2
|
+
require "contracts"
|
3
|
+
require "fileutils"
|
4
|
+
require "pg"
|
5
|
+
|
6
|
+
module StellarCoreBackup
|
7
|
+
autoload :Cmd, "stellar-core-backup/cmd"
|
8
|
+
autoload :CmdResult, "stellar-core-backup/cmd_result"
|
9
|
+
autoload :Config, "stellar-core-backup/config"
|
10
|
+
autoload :Database, "stellar-core-backup/database"
|
11
|
+
autoload :Filesystem, "stellar-core-backup/filesystem"
|
12
|
+
autoload :Job, "stellar-core-backup/job"
|
13
|
+
autoload :S3, "stellar-core-backup/s3"
|
14
|
+
autoload :Tar, "stellar-core-backup/tar"
|
15
|
+
autoload :Utils, "stellar-core-backup/utils"
|
16
|
+
|
17
|
+
module Restore
|
18
|
+
autoload :Database, "stellar-core-backup/restore/database"
|
19
|
+
autoload :Filesystem, "stellar-core-backup/restore/filesystem"
|
20
|
+
end
|
21
|
+
end
|
@@ -0,0 +1,28 @@
|
|
1
|
+
module StellarCoreBackup
|
2
|
+
class Cmd
|
3
|
+
include Contracts
|
4
|
+
|
5
|
+
Contract String => Any
|
6
|
+
def initialize(working_dir)
|
7
|
+
@working_dir = working_dir
|
8
|
+
end
|
9
|
+
|
10
|
+
Contract String, ArrayOf[String] => CmdResult
|
11
|
+
def run_and_capture(cmd, args)
|
12
|
+
Dir.chdir @working_dir do
|
13
|
+
stringArgs = args.map{|x| "#{x}"}.join(" ")
|
14
|
+
out = `#{cmd} #{stringArgs}`
|
15
|
+
CmdResult.new($?.exitstatus == 0, out)
|
16
|
+
end
|
17
|
+
end
|
18
|
+
|
19
|
+
Contract String, ArrayOf[String] => CmdResult
|
20
|
+
def run(cmd, args)
|
21
|
+
Dir.chdir @working_dir do
|
22
|
+
system(cmd, *args)
|
23
|
+
end
|
24
|
+
CmdResult.new($?.exitstatus == 0, nil)
|
25
|
+
end
|
26
|
+
|
27
|
+
end
|
28
|
+
end
|
@@ -0,0 +1,37 @@
|
|
1
|
+
require 'yaml'
|
2
|
+
|
3
|
+
module StellarCoreBackup
|
4
|
+
class Config
|
5
|
+
include Contracts
|
6
|
+
|
7
|
+
class ReadError < StandardError ; end
|
8
|
+
|
9
|
+
Contract String => Any
|
10
|
+
def initialize(config)
|
11
|
+
if (File.exists?(config)) then
|
12
|
+
@config = YAML.load_file(config)
|
13
|
+
else
|
14
|
+
raise ReadError
|
15
|
+
end
|
16
|
+
end
|
17
|
+
|
18
|
+
Contract None => Bool
|
19
|
+
def configured?()
|
20
|
+
unless @config.nil? || @config.empty?
|
21
|
+
return true
|
22
|
+
else
|
23
|
+
return false
|
24
|
+
end
|
25
|
+
end
|
26
|
+
|
27
|
+
Contract String => Any
|
28
|
+
def get(item)
|
29
|
+
if self.configured?() then
|
30
|
+
@config[item]
|
31
|
+
else
|
32
|
+
raise ReadError
|
33
|
+
end
|
34
|
+
end
|
35
|
+
|
36
|
+
end
|
37
|
+
end
|
@@ -0,0 +1,67 @@
|
|
1
|
+
require 'pg'
|
2
|
+
|
3
|
+
module StellarCoreBackup
|
4
|
+
class Database
|
5
|
+
include Contracts
|
6
|
+
|
7
|
+
attr_reader :dbname
|
8
|
+
|
9
|
+
Contract StellarCoreBackup::Config => Contracts::Any
|
10
|
+
def initialize(config)
|
11
|
+
@config = config
|
12
|
+
@working_dir = StellarCoreBackup::Utils.create_working_dir(@config.get('working_dir'))
|
13
|
+
@dbname = check_db_connection
|
14
|
+
end
|
15
|
+
|
16
|
+
public
|
17
|
+
def backup()
|
18
|
+
pg_dump()
|
19
|
+
end
|
20
|
+
|
21
|
+
private
|
22
|
+
#Contract nil, StellarCoreBackup::CmdResult
|
23
|
+
def pg_dump()
|
24
|
+
cmd = StellarCoreBackup::Cmd.new(@working_dir)
|
25
|
+
puts "info: connecting (#{get_db_details(@config.get('core_config')).gsub(/password=(.*)/, 'password=********')})"
|
26
|
+
pg_dump = cmd.run('pg_dump', ['--dbname', @dbname, '--format', 'd', '--file', "#{@working_dir}/core-db/"])
|
27
|
+
if pg_dump.success then
|
28
|
+
puts "info: database backup complete"
|
29
|
+
end
|
30
|
+
end
|
31
|
+
|
32
|
+
private
|
33
|
+
# check we have passwordless access to postgres
|
34
|
+
Contract nil => Contracts::Any
|
35
|
+
def check_db_connection()
|
36
|
+
# TODO move to config
|
37
|
+
dbname = get_db_details(@config.get('core_config'))
|
38
|
+
begin
|
39
|
+
conn = PG.connect(dbname)
|
40
|
+
conn.exec("SELECT * FROM pg_stat_activity") do |result|
|
41
|
+
# 2 => PGRES_TUPLES_OK
|
42
|
+
if result.result_status == 2 then
|
43
|
+
return dbname
|
44
|
+
end
|
45
|
+
end
|
46
|
+
rescue PG::Error
|
47
|
+
puts "error: failed to connect to db"
|
48
|
+
exit
|
49
|
+
end
|
50
|
+
end
|
51
|
+
|
52
|
+
private
|
53
|
+
Contract String => String
|
54
|
+
def get_db_details(config)
|
55
|
+
File.open(config,'r') do |fd|
|
56
|
+
fd.each_line do |line|
|
57
|
+
if (line[/^DATABASE=/]) then
|
58
|
+
# "postgresql://dbname=stellar user=stellar"
|
59
|
+
connection_str = /^DATABASE=(.*)/.match(line).captures[0].gsub!(/"postgresql:\/\/(.*)"$/,'\1')
|
60
|
+
return connection_str
|
61
|
+
end
|
62
|
+
end
|
63
|
+
end
|
64
|
+
end
|
65
|
+
|
66
|
+
end
|
67
|
+
end
|
@@ -0,0 +1,62 @@
|
|
1
|
+
module StellarCoreBackup
|
2
|
+
class Filesystem
|
3
|
+
include Contracts
|
4
|
+
|
5
|
+
class ReadError < StandardError ; end
|
6
|
+
|
7
|
+
attr_reader :core_data_dir
|
8
|
+
|
9
|
+
Contract StellarCoreBackup::Config => Contracts::Any
|
10
|
+
def initialize(config)
|
11
|
+
@config = config
|
12
|
+
@working_dir = StellarCoreBackup::Utils.create_working_dir(@config.get('working_dir'))
|
13
|
+
@core_data_dir = get_core_data_dir(@config.get('core_config'))
|
14
|
+
end
|
15
|
+
|
16
|
+
public
|
17
|
+
def backup()
|
18
|
+
create_core_data_tar()
|
19
|
+
end
|
20
|
+
|
21
|
+
private
|
22
|
+
#Contract nil, StellarCoreBackup::CmdResult
|
23
|
+
def create_core_data_tar()
|
24
|
+
if core_data_readable?() then
|
25
|
+
# create the tar balls
|
26
|
+
puts "info: creating filesystem backup"
|
27
|
+
Dir.chdir(@core_data_dir)
|
28
|
+
StellarCoreBackup::Tar.pack("#{@working_dir}/core-fs.tar", '.')
|
29
|
+
Dir.chdir(@working_dir)
|
30
|
+
end
|
31
|
+
end
|
32
|
+
|
33
|
+
private
|
34
|
+
# TODO: replace with Utils.readable ?
|
35
|
+
# check we have read access to the stellar-core data
|
36
|
+
Contract nil => Contracts::Any
|
37
|
+
def core_data_readable?()
|
38
|
+
if File.readable?("#{@core_data_dir}/stellar-core.lock") then
|
39
|
+
puts "info: processing #{@core_data_dir}"
|
40
|
+
return true
|
41
|
+
else
|
42
|
+
puts "error: can not access #{@core_data_dir}"
|
43
|
+
raise ReadError
|
44
|
+
end
|
45
|
+
end
|
46
|
+
|
47
|
+
private
|
48
|
+
Contract String => String
|
49
|
+
def get_core_data_dir(config)
|
50
|
+
File.open(config,'r') do |fd|
|
51
|
+
fd.each_line do |line|
|
52
|
+
if (line[/^BUCKET_DIR_PATH=/]) then
|
53
|
+
# "postgresql://dbname=stellar user=stellar"
|
54
|
+
core_data_dir = /^BUCKET_DIR_PATH="(.*)"$/.match(line).captures[0]
|
55
|
+
return core_data_dir
|
56
|
+
end
|
57
|
+
end
|
58
|
+
end
|
59
|
+
end
|
60
|
+
|
61
|
+
end
|
62
|
+
end
|
@@ -0,0 +1,190 @@
|
|
1
|
+
module StellarCoreBackup
|
2
|
+
class Job
|
3
|
+
|
4
|
+
class NoConfig < StandardError ; end
|
5
|
+
|
6
|
+
def initialize(**args)
|
7
|
+
if args.has_key?(:config) then
|
8
|
+
@config = StellarCoreBackup::Config.new(args[:config])
|
9
|
+
else
|
10
|
+
puts "info: no config provided"
|
11
|
+
raise NoConfig
|
12
|
+
end
|
13
|
+
|
14
|
+
# Set run time options
|
15
|
+
@verify = args[:verify] if args.has_key?(:verify)
|
16
|
+
@clean = args[:clean] if args.has_key?(:clean)
|
17
|
+
@listlen = args[:listlen]
|
18
|
+
|
19
|
+
# Set common run time parameters
|
20
|
+
@job_type = args[:type]
|
21
|
+
@gpg_key = @config.get('gpg_key')
|
22
|
+
@working_dir = StellarCoreBackup::Utils.create_working_dir(@config.get('working_dir'))
|
23
|
+
@cmd = StellarCoreBackup::Cmd.new(@working_dir)
|
24
|
+
@select = args[:select] if args.has_key?(:select)
|
25
|
+
@s3 = StellarCoreBackup::S3.new(@config)
|
26
|
+
|
27
|
+
# Set per operation type run time parameters
|
28
|
+
if args.has_key?(:type) then
|
29
|
+
case args[:type]
|
30
|
+
when 'backup'
|
31
|
+
puts 'info: backing up stellar-core'
|
32
|
+
@backup_dir = StellarCoreBackup::Utils.create_backup_dir(@config.get('backup_dir'))
|
33
|
+
@db = StellarCoreBackup::Database.new(@config)
|
34
|
+
@fs = StellarCoreBackup::Filesystem.new(@config)
|
35
|
+
when 'restore'
|
36
|
+
puts 'info: restoring stellar-core'
|
37
|
+
@db_restore = StellarCoreBackup::Restore::Database.new(@config)
|
38
|
+
@fs_restore = StellarCoreBackup::Restore::Filesystem.new(@config)
|
39
|
+
@utils = StellarCoreBackup::Utils.new(@config)
|
40
|
+
when 'getkey'
|
41
|
+
puts 'info: confirming public gpg key with key server'
|
42
|
+
when 'list'
|
43
|
+
puts "info: listing last #{@listlen} stellar-core backups"
|
44
|
+
end
|
45
|
+
end
|
46
|
+
end
|
47
|
+
|
48
|
+
def run()
|
49
|
+
case @job_type
|
50
|
+
when 'getkey'
|
51
|
+
begin
|
52
|
+
getkey = @cmd.run_and_capture('gpg', ['--keyserver', 'hkp://ipv4.pool.sks-keyservers.net', '--recv-key', @gpg_key, '2>&1'])
|
53
|
+
puts 'info: public gpg key installed'
|
54
|
+
if ! getkey.success then
|
55
|
+
puts "error: failed to get gpg key"
|
56
|
+
# dump the gpg output here for user level trouble shooting
|
57
|
+
puts "#{getkey.out}"
|
58
|
+
raise StandardError
|
59
|
+
end
|
60
|
+
rescue => e
|
61
|
+
puts e
|
62
|
+
end
|
63
|
+
when 'list'
|
64
|
+
begin
|
65
|
+
list=@s3.latest(@listlen)
|
66
|
+
puts "info: only #{list.length} backup files in bucket" if list.length < @listlen
|
67
|
+
puts list
|
68
|
+
rescue => e
|
69
|
+
puts e
|
70
|
+
end
|
71
|
+
when 'backup'
|
72
|
+
begin
|
73
|
+
puts 'info: stopping stellar-core'
|
74
|
+
# using sudo, if running as non root uid then you will need to configure sudoers
|
75
|
+
stop_core = @cmd.run_and_capture('sudo', ['/bin/systemctl', 'stop', 'stellar-core'])
|
76
|
+
# only proceed if core is stopped
|
77
|
+
if stop_core.success then
|
78
|
+
@db.backup
|
79
|
+
@fs.backup
|
80
|
+
if @verify then
|
81
|
+
create_hash_file = @cmd.run_and_capture('find', ['.', '-type', 'f', '!', '-name', 'SHA256SUMS', '|', 'xargs', 'sha256sum', '>', 'SHA256SUMS'])
|
82
|
+
if create_hash_file.success then
|
83
|
+
puts "info: sha sums file created"
|
84
|
+
else
|
85
|
+
puts 'error: error creating sha sums file'
|
86
|
+
raise StandardError
|
87
|
+
end
|
88
|
+
sign_hash_file = @cmd.run_and_capture('gpg', ['--local-user', @gpg_key, '--detach-sign', 'SHA256SUMS'])
|
89
|
+
if sign_hash_file.success then
|
90
|
+
puts "info: gpg signature created ok"
|
91
|
+
else
|
92
|
+
puts 'error: error signing sha256sum file'
|
93
|
+
raise StandardError
|
94
|
+
end
|
95
|
+
end
|
96
|
+
# create tar archive with fs, db backup files and if requested the file of shas256sums and corresponding gpg signature.
|
97
|
+
@backup = StellarCoreBackup::Utils.create_backup_tar(@working_dir, @backup_dir)
|
98
|
+
@s3.push(@backup)
|
99
|
+
else
|
100
|
+
puts 'error: can not stop stellar-core'
|
101
|
+
raise StandardError
|
102
|
+
end
|
103
|
+
|
104
|
+
# restart stellar-core post backup
|
105
|
+
puts 'info: starting stellar-core'
|
106
|
+
# using sudo, if running as non root uid then you will need to configure sudoers
|
107
|
+
start_core = @cmd.run_and_capture('sudo', ['/bin/systemctl', 'start', 'stellar-core'])
|
108
|
+
if start_core.success then
|
109
|
+
puts "info: stellar-core started"
|
110
|
+
else
|
111
|
+
puts 'error: can not start stellar-core'
|
112
|
+
raise StandardError
|
113
|
+
end
|
114
|
+
|
115
|
+
# clean up working_dir
|
116
|
+
StellarCoreBackup::Utils.cleanup(@working_dir)
|
117
|
+
rescue => e
|
118
|
+
puts e
|
119
|
+
# clean up working_dir
|
120
|
+
StellarCoreBackup::Utils.cleanup(@working_dir)
|
121
|
+
end
|
122
|
+
when 'restore'
|
123
|
+
begin
|
124
|
+
# confirm the bucket directory is set to be cleaned or is empty
|
125
|
+
# the fs_restore.core_data_dir_empty? throws an exception if it's not empty
|
126
|
+
if ! @clean then
|
127
|
+
@fs_restore.core_data_dir_empty?()
|
128
|
+
end
|
129
|
+
# using sudo, if running as non root uid then you will need to configure sudoers
|
130
|
+
stop_core = @cmd.run_and_capture('sudo', ['/bin/systemctl', 'stop', 'stellar-core'])
|
131
|
+
# only proceed if core is stopped
|
132
|
+
if stop_core.success then
|
133
|
+
# if no manual selection has been made, use the latest as derived from the s3.latest method
|
134
|
+
# this method returns an array so set @select to the first and only element
|
135
|
+
@select=@s3.latest(1)[0] if ! @select
|
136
|
+
@backup_archive = @s3.get(@select)
|
137
|
+
@utils.extract_backup(@backup_archive)
|
138
|
+
if @verify then
|
139
|
+
verify_hash_file = @cmd.run_and_capture('gpg', ['--local-user', @gpg_key, '--verify', 'SHA256SUMS.sig', 'SHA256SUMS', '2>&1'])
|
140
|
+
if verify_hash_file.success then
|
141
|
+
puts "info: gpg signature processed ok"
|
142
|
+
else
|
143
|
+
puts 'error: error verifying gpg signature'
|
144
|
+
raise StandardError
|
145
|
+
end
|
146
|
+
verify_sha_file_content = @cmd.run_and_capture('sha256sum', ['--status', '--strict', '-c', 'SHA256SUMS'])
|
147
|
+
if verify_sha_file_content.success then
|
148
|
+
puts "info: sha file sums match"
|
149
|
+
else
|
150
|
+
puts 'error: error processing sha256sum file'
|
151
|
+
raise StandardError
|
152
|
+
end
|
153
|
+
if StellarCoreBackup::Utils.confirm_shasums_definitive(@working_dir, @backup_archive) then
|
154
|
+
puts 'info: SHA256SUMS file list matches delivered archive'
|
155
|
+
else
|
156
|
+
puts 'error: unknown additional file(s) detected in archive'
|
157
|
+
raise StandardError
|
158
|
+
end
|
159
|
+
end
|
160
|
+
StellarCoreBackup::Utils.cleanbucket(@fs_restore.core_data_dir) if @clean
|
161
|
+
@fs_restore.restore(@backup_archive)
|
162
|
+
@db_restore.restore()
|
163
|
+
|
164
|
+
# restart stellar-core post restore
|
165
|
+
puts 'info: starting stellar-core'
|
166
|
+
# using sudo, if running as non root uid then you will need to configure sudoers
|
167
|
+
start_core = @cmd.run_and_capture('sudo', ['/bin/systemctl', 'start', 'stellar-core'])
|
168
|
+
if start_core.success then
|
169
|
+
puts "info: stellar-core started"
|
170
|
+
else
|
171
|
+
puts 'error: can not start stellar-core'
|
172
|
+
raise StandardError
|
173
|
+
end
|
174
|
+
else
|
175
|
+
puts 'error: can not stop stellar-core'
|
176
|
+
raise StandardError
|
177
|
+
end
|
178
|
+
|
179
|
+
# clean up working_dir
|
180
|
+
StellarCoreBackup::Utils.cleanup(@working_dir)
|
181
|
+
rescue => e
|
182
|
+
puts e
|
183
|
+
# clean up working_dir
|
184
|
+
StellarCoreBackup::Utils.cleanup(@working_dir)
|
185
|
+
end
|
186
|
+
end
|
187
|
+
end
|
188
|
+
|
189
|
+
end
|
190
|
+
end
|
@@ -0,0 +1,36 @@
|
|
1
|
+
require 'pg'
|
2
|
+
|
3
|
+
module StellarCoreBackup::Restore
|
4
|
+
class Database < StellarCoreBackup::Database
|
5
|
+
include Contracts
|
6
|
+
|
7
|
+
attr_reader :dbname
|
8
|
+
|
9
|
+
Contract StellarCoreBackup::Config => Contracts::Any
|
10
|
+
def initialize(config)
|
11
|
+
@config = config
|
12
|
+
@working_dir = StellarCoreBackup::Utils.create_working_dir(@config.get('working_dir'))
|
13
|
+
@dbname = check_db_connection
|
14
|
+
@cmd = StellarCoreBackup::Cmd.new(@working_dir)
|
15
|
+
end
|
16
|
+
|
17
|
+
public
|
18
|
+
Contract nil => nil
|
19
|
+
def restore()
|
20
|
+
puts "info: database restored" if pg_restore()
|
21
|
+
end
|
22
|
+
|
23
|
+
private
|
24
|
+
Contract nil => Bool
|
25
|
+
def pg_restore()
|
26
|
+
# we are restoring to public schema
|
27
|
+
pg_restore = @cmd.run('pg_restore', ['-n', 'public', '-c', '-d', @dbname, 'core-db/'])
|
28
|
+
if pg_restore.success then
|
29
|
+
return true
|
30
|
+
else
|
31
|
+
return false
|
32
|
+
end
|
33
|
+
end
|
34
|
+
|
35
|
+
end
|
36
|
+
end
|