stellar-core-backup 0.0.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,21 @@
1
+ require "stellar-core-backup/version"
2
+ require "contracts"
3
+ require "fileutils"
4
+ require "pg"
5
+
6
+ module StellarCoreBackup
7
+ autoload :Cmd, "stellar-core-backup/cmd"
8
+ autoload :CmdResult, "stellar-core-backup/cmd_result"
9
+ autoload :Config, "stellar-core-backup/config"
10
+ autoload :Database, "stellar-core-backup/database"
11
+ autoload :Filesystem, "stellar-core-backup/filesystem"
12
+ autoload :Job, "stellar-core-backup/job"
13
+ autoload :S3, "stellar-core-backup/s3"
14
+ autoload :Tar, "stellar-core-backup/tar"
15
+ autoload :Utils, "stellar-core-backup/utils"
16
+
17
+ module Restore
18
+ autoload :Database, "stellar-core-backup/restore/database"
19
+ autoload :Filesystem, "stellar-core-backup/restore/filesystem"
20
+ end
21
+ end
@@ -0,0 +1,28 @@
1
+ module StellarCoreBackup
2
+ class Cmd
3
+ include Contracts
4
+
5
+ Contract String => Any
6
+ def initialize(working_dir)
7
+ @working_dir = working_dir
8
+ end
9
+
10
+ Contract String, ArrayOf[String] => CmdResult
11
+ def run_and_capture(cmd, args)
12
+ Dir.chdir @working_dir do
13
+ stringArgs = args.map{|x| "#{x}"}.join(" ")
14
+ out = `#{cmd} #{stringArgs}`
15
+ CmdResult.new($?.exitstatus == 0, out)
16
+ end
17
+ end
18
+
19
+ Contract String, ArrayOf[String] => CmdResult
20
+ def run(cmd, args)
21
+ Dir.chdir @working_dir do
22
+ system(cmd, *args)
23
+ end
24
+ CmdResult.new($?.exitstatus == 0, nil)
25
+ end
26
+
27
+ end
28
+ end
@@ -0,0 +1,14 @@
1
+ module StellarCoreBackup
2
+ class CmdResult
3
+ include Contracts
4
+
5
+ attr_reader :success
6
+ attr_reader :out
7
+
8
+ Contract Bool, Maybe[String] => Any
9
+ def initialize(success, out = None)
10
+ @success = success
11
+ @out = out
12
+ end
13
+ end
14
+ end
@@ -0,0 +1,37 @@
1
+ require 'yaml'
2
+
3
+ module StellarCoreBackup
4
+ class Config
5
+ include Contracts
6
+
7
+ class ReadError < StandardError ; end
8
+
9
+ Contract String => Any
10
+ def initialize(config)
11
+ if (File.exists?(config)) then
12
+ @config = YAML.load_file(config)
13
+ else
14
+ raise ReadError
15
+ end
16
+ end
17
+
18
+ Contract None => Bool
19
+ def configured?()
20
+ unless @config.nil? || @config.empty?
21
+ return true
22
+ else
23
+ return false
24
+ end
25
+ end
26
+
27
+ Contract String => Any
28
+ def get(item)
29
+ if self.configured?() then
30
+ @config[item]
31
+ else
32
+ raise ReadError
33
+ end
34
+ end
35
+
36
+ end
37
+ end
@@ -0,0 +1,67 @@
1
+ require 'pg'
2
+
3
+ module StellarCoreBackup
4
+ class Database
5
+ include Contracts
6
+
7
+ attr_reader :dbname
8
+
9
+ Contract StellarCoreBackup::Config => Contracts::Any
10
+ def initialize(config)
11
+ @config = config
12
+ @working_dir = StellarCoreBackup::Utils.create_working_dir(@config.get('working_dir'))
13
+ @dbname = check_db_connection
14
+ end
15
+
16
+ public
17
+ def backup()
18
+ pg_dump()
19
+ end
20
+
21
+ private
22
+ #Contract nil, StellarCoreBackup::CmdResult
23
+ def pg_dump()
24
+ cmd = StellarCoreBackup::Cmd.new(@working_dir)
25
+ puts "info: connecting (#{get_db_details(@config.get('core_config')).gsub(/password=(.*)/, 'password=********')})"
26
+ pg_dump = cmd.run('pg_dump', ['--dbname', @dbname, '--format', 'd', '--file', "#{@working_dir}/core-db/"])
27
+ if pg_dump.success then
28
+ puts "info: database backup complete"
29
+ end
30
+ end
31
+
32
+ private
33
+ # check we have passwordless access to postgres
34
+ Contract nil => Contracts::Any
35
+ def check_db_connection()
36
+ # TODO move to config
37
+ dbname = get_db_details(@config.get('core_config'))
38
+ begin
39
+ conn = PG.connect(dbname)
40
+ conn.exec("SELECT * FROM pg_stat_activity") do |result|
41
+ # 2 => PGRES_TUPLES_OK
42
+ if result.result_status == 2 then
43
+ return dbname
44
+ end
45
+ end
46
+ rescue PG::Error
47
+ puts "error: failed to connect to db"
48
+ exit
49
+ end
50
+ end
51
+
52
+ private
53
+ Contract String => String
54
+ def get_db_details(config)
55
+ File.open(config,'r') do |fd|
56
+ fd.each_line do |line|
57
+ if (line[/^DATABASE=/]) then
58
+ # "postgresql://dbname=stellar user=stellar"
59
+ connection_str = /^DATABASE=(.*)/.match(line).captures[0].gsub!(/"postgresql:\/\/(.*)"$/,'\1')
60
+ return connection_str
61
+ end
62
+ end
63
+ end
64
+ end
65
+
66
+ end
67
+ end
@@ -0,0 +1,62 @@
1
+ module StellarCoreBackup
2
+ class Filesystem
3
+ include Contracts
4
+
5
+ class ReadError < StandardError ; end
6
+
7
+ attr_reader :core_data_dir
8
+
9
+ Contract StellarCoreBackup::Config => Contracts::Any
10
+ def initialize(config)
11
+ @config = config
12
+ @working_dir = StellarCoreBackup::Utils.create_working_dir(@config.get('working_dir'))
13
+ @core_data_dir = get_core_data_dir(@config.get('core_config'))
14
+ end
15
+
16
+ public
17
+ def backup()
18
+ create_core_data_tar()
19
+ end
20
+
21
+ private
22
+ #Contract nil, StellarCoreBackup::CmdResult
23
+ def create_core_data_tar()
24
+ if core_data_readable?() then
25
+ # create the tar balls
26
+ puts "info: creating filesystem backup"
27
+ Dir.chdir(@core_data_dir)
28
+ StellarCoreBackup::Tar.pack("#{@working_dir}/core-fs.tar", '.')
29
+ Dir.chdir(@working_dir)
30
+ end
31
+ end
32
+
33
+ private
34
+ # TODO: replace with Utils.readable ?
35
+ # check we have read access to the stellar-core data
36
+ Contract nil => Contracts::Any
37
+ def core_data_readable?()
38
+ if File.readable?("#{@core_data_dir}/stellar-core.lock") then
39
+ puts "info: processing #{@core_data_dir}"
40
+ return true
41
+ else
42
+ puts "error: can not access #{@core_data_dir}"
43
+ raise ReadError
44
+ end
45
+ end
46
+
47
+ private
48
+ Contract String => String
49
+ def get_core_data_dir(config)
50
+ File.open(config,'r') do |fd|
51
+ fd.each_line do |line|
52
+ if (line[/^BUCKET_DIR_PATH=/]) then
53
+ # "postgresql://dbname=stellar user=stellar"
54
+ core_data_dir = /^BUCKET_DIR_PATH="(.*)"$/.match(line).captures[0]
55
+ return core_data_dir
56
+ end
57
+ end
58
+ end
59
+ end
60
+
61
+ end
62
+ end
@@ -0,0 +1,190 @@
1
+ module StellarCoreBackup
2
+ class Job
3
+
4
+ class NoConfig < StandardError ; end
5
+
6
+ def initialize(**args)
7
+ if args.has_key?(:config) then
8
+ @config = StellarCoreBackup::Config.new(args[:config])
9
+ else
10
+ puts "info: no config provided"
11
+ raise NoConfig
12
+ end
13
+
14
+ # Set run time options
15
+ @verify = args[:verify] if args.has_key?(:verify)
16
+ @clean = args[:clean] if args.has_key?(:clean)
17
+ @listlen = args[:listlen]
18
+
19
+ # Set common run time parameters
20
+ @job_type = args[:type]
21
+ @gpg_key = @config.get('gpg_key')
22
+ @working_dir = StellarCoreBackup::Utils.create_working_dir(@config.get('working_dir'))
23
+ @cmd = StellarCoreBackup::Cmd.new(@working_dir)
24
+ @select = args[:select] if args.has_key?(:select)
25
+ @s3 = StellarCoreBackup::S3.new(@config)
26
+
27
+ # Set per operation type run time parameters
28
+ if args.has_key?(:type) then
29
+ case args[:type]
30
+ when 'backup'
31
+ puts 'info: backing up stellar-core'
32
+ @backup_dir = StellarCoreBackup::Utils.create_backup_dir(@config.get('backup_dir'))
33
+ @db = StellarCoreBackup::Database.new(@config)
34
+ @fs = StellarCoreBackup::Filesystem.new(@config)
35
+ when 'restore'
36
+ puts 'info: restoring stellar-core'
37
+ @db_restore = StellarCoreBackup::Restore::Database.new(@config)
38
+ @fs_restore = StellarCoreBackup::Restore::Filesystem.new(@config)
39
+ @utils = StellarCoreBackup::Utils.new(@config)
40
+ when 'getkey'
41
+ puts 'info: confirming public gpg key with key server'
42
+ when 'list'
43
+ puts "info: listing last #{@listlen} stellar-core backups"
44
+ end
45
+ end
46
+ end
47
+
48
+ def run()
49
+ case @job_type
50
+ when 'getkey'
51
+ begin
52
+ getkey = @cmd.run_and_capture('gpg', ['--keyserver', 'hkp://ipv4.pool.sks-keyservers.net', '--recv-key', @gpg_key, '2>&1'])
53
+ puts 'info: public gpg key installed'
54
+ if ! getkey.success then
55
+ puts "error: failed to get gpg key"
56
+ # dump the gpg output here for user level trouble shooting
57
+ puts "#{getkey.out}"
58
+ raise StandardError
59
+ end
60
+ rescue => e
61
+ puts e
62
+ end
63
+ when 'list'
64
+ begin
65
+ list=@s3.latest(@listlen)
66
+ puts "info: only #{list.length} backup files in bucket" if list.length < @listlen
67
+ puts list
68
+ rescue => e
69
+ puts e
70
+ end
71
+ when 'backup'
72
+ begin
73
+ puts 'info: stopping stellar-core'
74
+ # using sudo, if running as non root uid then you will need to configure sudoers
75
+ stop_core = @cmd.run_and_capture('sudo', ['/bin/systemctl', 'stop', 'stellar-core'])
76
+ # only proceed if core is stopped
77
+ if stop_core.success then
78
+ @db.backup
79
+ @fs.backup
80
+ if @verify then
81
+ create_hash_file = @cmd.run_and_capture('find', ['.', '-type', 'f', '!', '-name', 'SHA256SUMS', '|', 'xargs', 'sha256sum', '>', 'SHA256SUMS'])
82
+ if create_hash_file.success then
83
+ puts "info: sha sums file created"
84
+ else
85
+ puts 'error: error creating sha sums file'
86
+ raise StandardError
87
+ end
88
+ sign_hash_file = @cmd.run_and_capture('gpg', ['--local-user', @gpg_key, '--detach-sign', 'SHA256SUMS'])
89
+ if sign_hash_file.success then
90
+ puts "info: gpg signature created ok"
91
+ else
92
+ puts 'error: error signing sha256sum file'
93
+ raise StandardError
94
+ end
95
+ end
96
+ # create tar archive with fs, db backup files and if requested the file of shas256sums and corresponding gpg signature.
97
+ @backup = StellarCoreBackup::Utils.create_backup_tar(@working_dir, @backup_dir)
98
+ @s3.push(@backup)
99
+ else
100
+ puts 'error: can not stop stellar-core'
101
+ raise StandardError
102
+ end
103
+
104
+ # restart stellar-core post backup
105
+ puts 'info: starting stellar-core'
106
+ # using sudo, if running as non root uid then you will need to configure sudoers
107
+ start_core = @cmd.run_and_capture('sudo', ['/bin/systemctl', 'start', 'stellar-core'])
108
+ if start_core.success then
109
+ puts "info: stellar-core started"
110
+ else
111
+ puts 'error: can not start stellar-core'
112
+ raise StandardError
113
+ end
114
+
115
+ # clean up working_dir
116
+ StellarCoreBackup::Utils.cleanup(@working_dir)
117
+ rescue => e
118
+ puts e
119
+ # clean up working_dir
120
+ StellarCoreBackup::Utils.cleanup(@working_dir)
121
+ end
122
+ when 'restore'
123
+ begin
124
+ # confirm the bucket directory is set to be cleaned or is empty
125
+ # the fs_restore.core_data_dir_empty? throws an exception if it's not empty
126
+ if ! @clean then
127
+ @fs_restore.core_data_dir_empty?()
128
+ end
129
+ # using sudo, if running as non root uid then you will need to configure sudoers
130
+ stop_core = @cmd.run_and_capture('sudo', ['/bin/systemctl', 'stop', 'stellar-core'])
131
+ # only proceed if core is stopped
132
+ if stop_core.success then
133
+ # if no manual selection has been made, use the latest as derived from the s3.latest method
134
+ # this method returns an array so set @select to the first and only element
135
+ @select=@s3.latest(1)[0] if ! @select
136
+ @backup_archive = @s3.get(@select)
137
+ @utils.extract_backup(@backup_archive)
138
+ if @verify then
139
+ verify_hash_file = @cmd.run_and_capture('gpg', ['--local-user', @gpg_key, '--verify', 'SHA256SUMS.sig', 'SHA256SUMS', '2>&1'])
140
+ if verify_hash_file.success then
141
+ puts "info: gpg signature processed ok"
142
+ else
143
+ puts 'error: error verifying gpg signature'
144
+ raise StandardError
145
+ end
146
+ verify_sha_file_content = @cmd.run_and_capture('sha256sum', ['--status', '--strict', '-c', 'SHA256SUMS'])
147
+ if verify_sha_file_content.success then
148
+ puts "info: sha file sums match"
149
+ else
150
+ puts 'error: error processing sha256sum file'
151
+ raise StandardError
152
+ end
153
+ if StellarCoreBackup::Utils.confirm_shasums_definitive(@working_dir, @backup_archive) then
154
+ puts 'info: SHA256SUMS file list matches delivered archive'
155
+ else
156
+ puts 'error: unknown additional file(s) detected in archive'
157
+ raise StandardError
158
+ end
159
+ end
160
+ StellarCoreBackup::Utils.cleanbucket(@fs_restore.core_data_dir) if @clean
161
+ @fs_restore.restore(@backup_archive)
162
+ @db_restore.restore()
163
+
164
+ # restart stellar-core post restore
165
+ puts 'info: starting stellar-core'
166
+ # using sudo, if running as non root uid then you will need to configure sudoers
167
+ start_core = @cmd.run_and_capture('sudo', ['/bin/systemctl', 'start', 'stellar-core'])
168
+ if start_core.success then
169
+ puts "info: stellar-core started"
170
+ else
171
+ puts 'error: can not start stellar-core'
172
+ raise StandardError
173
+ end
174
+ else
175
+ puts 'error: can not stop stellar-core'
176
+ raise StandardError
177
+ end
178
+
179
+ # clean up working_dir
180
+ StellarCoreBackup::Utils.cleanup(@working_dir)
181
+ rescue => e
182
+ puts e
183
+ # clean up working_dir
184
+ StellarCoreBackup::Utils.cleanup(@working_dir)
185
+ end
186
+ end
187
+ end
188
+
189
+ end
190
+ end
@@ -0,0 +1,36 @@
1
+ require 'pg'
2
+
3
+ module StellarCoreBackup::Restore
4
+ class Database < StellarCoreBackup::Database
5
+ include Contracts
6
+
7
+ attr_reader :dbname
8
+
9
+ Contract StellarCoreBackup::Config => Contracts::Any
10
+ def initialize(config)
11
+ @config = config
12
+ @working_dir = StellarCoreBackup::Utils.create_working_dir(@config.get('working_dir'))
13
+ @dbname = check_db_connection
14
+ @cmd = StellarCoreBackup::Cmd.new(@working_dir)
15
+ end
16
+
17
+ public
18
+ Contract nil => nil
19
+ def restore()
20
+ puts "info: database restored" if pg_restore()
21
+ end
22
+
23
+ private
24
+ Contract nil => Bool
25
+ def pg_restore()
26
+ # we are restoring to public schema
27
+ pg_restore = @cmd.run('pg_restore', ['-n', 'public', '-c', '-d', @dbname, 'core-db/'])
28
+ if pg_restore.success then
29
+ return true
30
+ else
31
+ return false
32
+ end
33
+ end
34
+
35
+ end
36
+ end