bkp 1.0.0 → 1.0.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/Onafile +130 -0
- data/bin/bkp +1 -1
- data/lib/bkp.rb +0 -0
- data/lib/helpers.rb +233 -0
- data/lib/validations.rb +80 -0
- metadata +5 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 7f7f2dbb1125aa7389f550edd88efb0cc6f47ba7462a3897d67179f014f34abc
|
4
|
+
data.tar.gz: 7718b010f57402ed73891a50c210b676823069a2a40e884fb34c0e63a2e34fe6
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 946e793f94795068719875cd36f802035a53db26c9b04dcb3ad4805e41bf461d785bf6b7350a967c203646a847b3887bb69749a029367179e27430aac172c6fc
|
7
|
+
data.tar.gz: 8167b607e70e9e59e55212b9a711a31bb26ef0988dc04d9bf4094c25e58fcaa9876dc0f06f1829797c1a869024dadba1c5ed5720af71f907208c806378c3174f
|
data/Onafile
CHANGED
@@ -1,3 +1,133 @@
|
|
1
1
|
#!/usr/bin/env ruby
|
2
2
|
|
3
|
+
require 'yaml'
|
4
|
+
require 'time'
|
5
|
+
require 'tmpdir'
|
6
|
+
require 'json'
|
7
|
+
require 'fileutils'
|
8
|
+
require './lib/validations.rb'
|
9
|
+
require './lib/helpers.rb'
|
10
|
+
|
11
|
+
if ENV['OLDPWD']
|
12
|
+
Dir.chdir(ENV['OLDPWD'])
|
13
|
+
end
|
14
|
+
|
15
|
+
config_check
|
16
|
+
|
3
17
|
Ona.prompt = 'bkp'
|
18
|
+
|
19
|
+
Ona.resource(:backup, [
|
20
|
+
:background,
|
21
|
+
:bucket,
|
22
|
+
:date,
|
23
|
+
:name,
|
24
|
+
:owner,
|
25
|
+
:summary,
|
26
|
+
:ticket,
|
27
|
+
:path,
|
28
|
+
:directory
|
29
|
+
])
|
30
|
+
|
31
|
+
reload_manifests
|
32
|
+
|
33
|
+
Ona.action(
|
34
|
+
:regex => /(^)(ls)($)/,
|
35
|
+
:resource => :backup,
|
36
|
+
:text => "List backups.",
|
37
|
+
:example => "ls"
|
38
|
+
) do |items, command, regex|
|
39
|
+
items.sort do |a, b|
|
40
|
+
Time.parse(a.date) <=> Time.parse(b.date)
|
41
|
+
end.each_with_index do |item, id|
|
42
|
+
pretty_backup_list(id, item)
|
43
|
+
end
|
44
|
+
end
|
45
|
+
|
46
|
+
Ona.action(
|
47
|
+
:regex => /(^)(new)($)/,
|
48
|
+
:resource => :backup,
|
49
|
+
:text => "Creates a new backup config file.",
|
50
|
+
:example => "new"
|
51
|
+
) do |items, command, regex|
|
52
|
+
generate_template
|
53
|
+
end
|
54
|
+
|
55
|
+
Ona.action(
|
56
|
+
:regex => /(^)(check)(\s+)(.*)($)/,
|
57
|
+
:resource => :backup,
|
58
|
+
:text => "Validates a backup config file.",
|
59
|
+
:example => "check [FILENAME]"
|
60
|
+
) do |items, command, regex|
|
61
|
+
file = command.scan(regex)[0][3]
|
62
|
+
file.gsub!(/(\s+$)/, '')
|
63
|
+
load_file(file)
|
64
|
+
end
|
65
|
+
|
66
|
+
Ona.action(
|
67
|
+
:regex => /(^)(upload)(\s+)(.*)($)/,
|
68
|
+
:resource => :backup,
|
69
|
+
:text => "Uploads backups from config file.",
|
70
|
+
:example => "upload [FILENAME]"
|
71
|
+
) do |items, command, regex|
|
72
|
+
file = command.scan(regex)[0][3]
|
73
|
+
file.gsub!(/(\s+$)/, '')
|
74
|
+
backups = load_file(file)
|
75
|
+
unless Ona.confirm('Are you sure you want to upload these backups?', 'yes')
|
76
|
+
next
|
77
|
+
end
|
78
|
+
backups.each do |backup|
|
79
|
+
create_backup(backup)
|
80
|
+
end
|
81
|
+
end
|
82
|
+
|
83
|
+
Ona.action(
|
84
|
+
:regex => /(^)(sync)($)/,
|
85
|
+
:resource => :backup,
|
86
|
+
:text => "Reads remote backups so they can be evaluated locally.",
|
87
|
+
:example => "sync"
|
88
|
+
) do |items, command, regex|
|
89
|
+
download_manifests
|
90
|
+
reload_manifests
|
91
|
+
end
|
92
|
+
|
93
|
+
Ona.action(
|
94
|
+
:regex => /(^)(show)(\s+)(.*)($)/,
|
95
|
+
:resource => :backup,
|
96
|
+
:text => "Shows a backup config file.",
|
97
|
+
:example => "show [NUMBER]",
|
98
|
+
) do |items, command, regex|
|
99
|
+
queried_id = command.scan(regex)[0][3]
|
100
|
+
if queried_id =~ /\d+/
|
101
|
+
queried_id = queried_id.to_i
|
102
|
+
else
|
103
|
+
next
|
104
|
+
end
|
105
|
+
items.sort do |a, b|
|
106
|
+
Time.parse(a.date) <=> Time.parse(b.date)
|
107
|
+
end.each_with_index do |backup, id|
|
108
|
+
if id == queried_id
|
109
|
+
pretty_backup_body(id, backup)
|
110
|
+
end
|
111
|
+
end
|
112
|
+
end
|
113
|
+
|
114
|
+
Ona.action(
|
115
|
+
:regex => /(^)(download)(\s+)(.*)($)/,
|
116
|
+
:resource => :backup,
|
117
|
+
:text => 'Download a remote backup',
|
118
|
+
:example => "download [NUMBER]",
|
119
|
+
) do |items, command, regex|
|
120
|
+
queried_id = command.scan(regex)[0][3]
|
121
|
+
if queried_id =~ /\d+/
|
122
|
+
queried_id = queried_id.to_i
|
123
|
+
else
|
124
|
+
next
|
125
|
+
end
|
126
|
+
items.sort do |a, b|
|
127
|
+
Time.parse(a.date) <=> Time.parse(b.date)
|
128
|
+
end.each_with_index do |backup, id|
|
129
|
+
if id == queried_id
|
130
|
+
run_command('aws s3 cp ' + s3_path(backup) + ' .' )
|
131
|
+
end
|
132
|
+
end
|
133
|
+
end
|
data/bin/bkp
CHANGED
data/lib/bkp.rb
ADDED
File without changes
|
data/lib/helpers.rb
ADDED
@@ -0,0 +1,233 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
TEMPLATE = "---
|
4
|
+
-
|
5
|
+
# A short one line easy to understand summary of what the backup is for.
|
6
|
+
summary:
|
7
|
+
|
8
|
+
# Some context about the backup (why are we doing this?)
|
9
|
+
background: |-
|
10
|
+
This is a backup of the database for the website.
|
11
|
+
You can find the website at http://www.example.com
|
12
|
+
|
13
|
+
It is important to backup the database because it contains all the information for the website.
|
14
|
+
|
15
|
+
# The local diretory you want to backup
|
16
|
+
directory: ./mydata
|
17
|
+
|
18
|
+
# The date of the backup
|
19
|
+
date: '%<date>s'
|
20
|
+
|
21
|
+
# Who created the backup. (usually your name)
|
22
|
+
owner:
|
23
|
+
|
24
|
+
# The ticket url associated with the backup
|
25
|
+
ticket: ''
|
26
|
+
|
27
|
+
# The bucket where the backup will be stored
|
28
|
+
# (probably okay to keep as is)
|
29
|
+
bucket: %<bucket>s
|
30
|
+
|
31
|
+
# The path where all the backups are stored in the bucket
|
32
|
+
# Think about it this way: bucket/path/[backups live here]
|
33
|
+
# (probably okay to keep as is)
|
34
|
+
path: %<path>s
|
35
|
+
"
|
36
|
+
|
37
|
+
CONFIG_FILE = "#{ENV['HOME']}/.bkp/config.yml"
|
38
|
+
|
39
|
+
DEFAULT_CONFIG = "---
|
40
|
+
# The default bucket where all backups will be stored
|
41
|
+
bucket: ''
|
42
|
+
|
43
|
+
# The default path where all backups will be stored
|
44
|
+
# Think about it this way: bucket/path/[backups live here]
|
45
|
+
path: ''
|
46
|
+
"
|
47
|
+
|
48
|
+
def to_hyphen(string)
|
49
|
+
string = string.chomp
|
50
|
+
string.gsub!(/\W+/, ' ')
|
51
|
+
string.gsub!(/\s+/, '-')
|
52
|
+
string.gsub!(/^[-]+|[-]+$/, '')
|
53
|
+
string.downcase
|
54
|
+
end
|
55
|
+
|
56
|
+
def create_backup(backup)
|
57
|
+
Dir.mktmpdir do |dir|
|
58
|
+
# we keep the original directory when we tar and gzip the directory
|
59
|
+
run_command "cd #{backup['directory']} && cd .. && tar -czf #{dir}/#{backup['name']}.tar.gz #{backup['directory'].split('/').last}"
|
60
|
+
run_command "aws s3 cp #{dir}/#{backup['name']}.tar.gz s3://#{backup['bucket']}/#{backup['path']}/#{backup['name']}/#{backup['name']}.tar.gz"
|
61
|
+
|
62
|
+
manifest = "#{dir}/manifest.json"
|
63
|
+
|
64
|
+
File.open(manifest, 'w+') do |file|
|
65
|
+
file.write(backup.to_json)
|
66
|
+
end
|
67
|
+
|
68
|
+
run_command "aws s3 cp #{manifest} s3://#{backup['bucket']}/#{backup['path']}/#{backup['name']}/manifest.json"
|
69
|
+
end
|
70
|
+
end
|
71
|
+
|
72
|
+
def load_file(file)
|
73
|
+
unless File.exist?(file)
|
74
|
+
$stderr.puts "Backup File #{file.inspect} does not exist."
|
75
|
+
exit 1
|
76
|
+
end
|
77
|
+
|
78
|
+
backups = YAML.load_file(file)
|
79
|
+
|
80
|
+
backups.each_with_index do |backup, index|
|
81
|
+
validate_summary(backup['summary'], index)
|
82
|
+
validate_background(backup['background'], index)
|
83
|
+
validate_directory(backup['directory'], index)
|
84
|
+
validate_date(backup['date'], index)
|
85
|
+
validate_owner(backup['owner'], index)
|
86
|
+
validate_ticket(backup['ticket'], index)
|
87
|
+
validate_bucket(backup['bucket'], index)
|
88
|
+
validate_path(backup['path'], index)
|
89
|
+
date = Time.parse(backup['date'], index)
|
90
|
+
date = date.strftime('%Y-%m-%d')
|
91
|
+
backup['name'] = date + '-' + to_hyphen(backup['summary'])
|
92
|
+
end
|
93
|
+
end
|
94
|
+
|
95
|
+
def backup_list
|
96
|
+
config = YAML.load_file(CONFIG_FILE)
|
97
|
+
list = File.expand_path("~/.bkp/list.txt")
|
98
|
+
run_command "aws s3 ls s3://#{config['bucket']}/#{config['path']}/ > #{list}"
|
99
|
+
files = []
|
100
|
+
File.read(list).each_line do |line|
|
101
|
+
files << line.split(' ').last
|
102
|
+
end
|
103
|
+
files
|
104
|
+
end
|
105
|
+
|
106
|
+
def download_manifests
|
107
|
+
config = YAML.load_file(CONFIG_FILE)
|
108
|
+
backups = backup_list
|
109
|
+
FileUtils.mkdir_p(File.expand_path('~/.bkp/manifests'))
|
110
|
+
FileUtils.rm(Dir.glob(File.expand_path('~/.bkp/manifests/*.json')))
|
111
|
+
backups.each do |backup|
|
112
|
+
backup = backup.split('/').first
|
113
|
+
run_command "aws s3 cp s3://#{config['bucket']}/#{config['path']}/#{backup}/manifest.json ~/.bkp/manifests/#{backup}.json"
|
114
|
+
end
|
115
|
+
reload_manifests
|
116
|
+
end
|
117
|
+
|
118
|
+
def load_manifests
|
119
|
+
Dir.glob(File.expand_path('~/.bkp/manifests/*.json')).map do |file|
|
120
|
+
object = JSON.parse(File.read(file))
|
121
|
+
Ona.register(:backup) do |backup|
|
122
|
+
backup.name = object['name']
|
123
|
+
backup.summary = object['summary']
|
124
|
+
backup.background = object['background']
|
125
|
+
backup.directory = object['directory']
|
126
|
+
backup.date = object['date']
|
127
|
+
backup.owner = object['owner']
|
128
|
+
backup.ticket = object['ticket']
|
129
|
+
backup.bucket = object['bucket']
|
130
|
+
backup.path = object['path']
|
131
|
+
end
|
132
|
+
end
|
133
|
+
end
|
134
|
+
|
135
|
+
def unload_manifests
|
136
|
+
Ona.class_eval do
|
137
|
+
puts @resources[:backup][:entries] = []
|
138
|
+
end
|
139
|
+
end
|
140
|
+
|
141
|
+
def reload_manifests
|
142
|
+
unload_manifests
|
143
|
+
load_manifests
|
144
|
+
end
|
145
|
+
|
146
|
+
def config_check
|
147
|
+
return if File.exist?(CONFIG_FILE)
|
148
|
+
|
149
|
+
puts "Config file #{CONFIG_FILE.inspect} does not exist."
|
150
|
+
|
151
|
+
unless Ona.confirm('Okay to create a new config file in ~/.bkp/config.yml?', 'yes')
|
152
|
+
$stderr.puts 'Will exit this program now.'
|
153
|
+
exit 1
|
154
|
+
end
|
155
|
+
|
156
|
+
puts 'What is the name of the bucket where you want to store your backups?'
|
157
|
+
print 'S3 Bucket name> '
|
158
|
+
bucket = gets.chomp
|
159
|
+
|
160
|
+
puts "What is the 'path' where you want to store your backups?"
|
161
|
+
print 'S3 Bucket path> '
|
162
|
+
path = gets.chomp
|
163
|
+
|
164
|
+
config = YAML.load(DEFAULT_CONFIG)
|
165
|
+
config['bucket'] = bucket
|
166
|
+
config['path'] = path
|
167
|
+
FileUtils.mkdir_p(File.expand_path('~/.bkp'))
|
168
|
+
File.open(CONFIG_FILE, 'w+') do |file|
|
169
|
+
file.write(config.to_yaml)
|
170
|
+
end
|
171
|
+
end
|
172
|
+
|
173
|
+
# we use gsub because older rubies format method works differently.
|
174
|
+
def generate_template
|
175
|
+
config = YAML.load_file(CONFIG_FILE)
|
176
|
+
template = TEMPLATE.dup
|
177
|
+
template.gsub!('%<date>s', Time.now.to_s)
|
178
|
+
template.gsub!('%<bucket>s', config['bucket'])
|
179
|
+
template.gsub!('%<path>s', config['path'])
|
180
|
+
puts template
|
181
|
+
end
|
182
|
+
|
183
|
+
def run_command(command)
|
184
|
+
puts ''
|
185
|
+
puts "# Command: #{command.to_ansi.yellow.to_s}"
|
186
|
+
puts "# Executed at: #{Time.now.to_s}"
|
187
|
+
puts "# #{('=' * 76).to_ansi.cyan.to_s}"
|
188
|
+
system command
|
189
|
+
puts ''
|
190
|
+
end
|
191
|
+
|
192
|
+
def pretty_backup_list(id, backup)
|
193
|
+
pretty_id = id.to_s.to_s.rjust(5, ' ').to_ansi.cyan.to_s
|
194
|
+
pretty_name = 'name'.to_ansi.green.to_s
|
195
|
+
pretty_date = backup.date.to_ansi.yellow.to_s
|
196
|
+
puts "#{pretty_id} - [#{pretty_date}] #{pretty_name}: #{backup.name}"
|
197
|
+
end
|
198
|
+
|
199
|
+
def pretty_backup_body(id, backup)
|
200
|
+
puts ''
|
201
|
+
puts 'Summary:'.to_ansi.green.to_s
|
202
|
+
puts backup.summary.to_ansi.cyan.to_s
|
203
|
+
puts ''
|
204
|
+
puts 'Background:'.to_ansi.green.to_s
|
205
|
+
puts backup.background.to_ansi.cyan.to_s
|
206
|
+
puts ''
|
207
|
+
puts 'Date:'.to_ansi.green.to_s
|
208
|
+
puts backup.date.to_ansi.cyan.to_s
|
209
|
+
puts ''
|
210
|
+
puts 'Owner:'.to_ansi.green.to_s
|
211
|
+
puts backup.owner.to_ansi.cyan.to_s
|
212
|
+
puts ''
|
213
|
+
puts 'Ticket:'.to_ansi.green.to_s
|
214
|
+
puts backup.ticket.to_ansi.cyan.to_s
|
215
|
+
puts ''
|
216
|
+
puts 'Bucket:'.to_ansi.green.to_s
|
217
|
+
puts backup.bucket.to_ansi.cyan.to_s
|
218
|
+
puts ''
|
219
|
+
puts 'Path:'.to_ansi.green.to_s
|
220
|
+
puts backup.path.to_ansi.cyan.to_s
|
221
|
+
puts ''
|
222
|
+
puts 'Directory:'.to_ansi.green.to_s
|
223
|
+
puts backup.directory.to_ansi.cyan.to_s
|
224
|
+
puts ''
|
225
|
+
puts 'S3 Path:'.to_ansi.green.to_s
|
226
|
+
puts s3_path(backup).to_ansi.cyan.to_s
|
227
|
+
puts ''
|
228
|
+
|
229
|
+
end
|
230
|
+
|
231
|
+
def s3_path(backup)
|
232
|
+
's3://' + backup.bucket + '/' + backup.path + '/' + backup.name + '/' + backup.name + '.tar.gz'
|
233
|
+
end
|
data/lib/validations.rb
ADDED
@@ -0,0 +1,80 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
def validate_summary(summary, index)
|
4
|
+
if summary.nil? || summary.empty?
|
5
|
+
$stderr.puts "Backup(#{index}) summary is empty: #{summary.inspect}"
|
6
|
+
exit 1
|
7
|
+
end
|
8
|
+
|
9
|
+
if summary.size > 80
|
10
|
+
$stderr.puts "Backup(#{index}) summary is too long: #{summary.size} characters"
|
11
|
+
exit 1
|
12
|
+
end
|
13
|
+
end
|
14
|
+
|
15
|
+
def validate_background(background, index)
|
16
|
+
if background.nil? || background.empty?
|
17
|
+
$stderr.puts "Backup(#{index}) background is empty: #{background.inspect}"
|
18
|
+
exit 1
|
19
|
+
end
|
20
|
+
|
21
|
+
if background.size < 80
|
22
|
+
$stderr.puts "Backup(#{index}) background is too short: #{background.size} characters"
|
23
|
+
exit 1
|
24
|
+
end
|
25
|
+
end
|
26
|
+
|
27
|
+
def validate_directory(directory, index)
|
28
|
+
if directory.nil? || directory.empty?
|
29
|
+
$stderr.puts "Backup(#{index}) directory is empty: #{directory.inspect}"
|
30
|
+
exit 1
|
31
|
+
end
|
32
|
+
|
33
|
+
unless File.directory?(directory)
|
34
|
+
$stderr.puts "Backup(#{index}) directory does not exist: #{directory.inspect}"
|
35
|
+
exit 1
|
36
|
+
end
|
37
|
+
end
|
38
|
+
|
39
|
+
def validate_date(date, index)
|
40
|
+
if date.nil? || date.empty?
|
41
|
+
$stderr.puts "Backup(#{index}) date is empty: #{date.inspect}"
|
42
|
+
exit 1
|
43
|
+
end
|
44
|
+
|
45
|
+
begin
|
46
|
+
Time.parse(date)
|
47
|
+
rescue ArgumentError
|
48
|
+
$stderr.puts "Backup(#{index}) date is not a valid date: #{date.inspect}"
|
49
|
+
exit 1
|
50
|
+
end
|
51
|
+
end
|
52
|
+
|
53
|
+
def validate_owner(owner, index)
|
54
|
+
if owner.nil? || owner.empty?
|
55
|
+
$stderr.puts "Backup(#{index}) owner is empty: #{owner.inspect}"
|
56
|
+
exit 1
|
57
|
+
end
|
58
|
+
end
|
59
|
+
|
60
|
+
def validate_ticket(ticket, index)
|
61
|
+
if ticket.nil? || ticket.empty?
|
62
|
+
$stderr.puts "Backup(#{index}) ticket is empty: #{ticket.inspect}"
|
63
|
+
exit 1
|
64
|
+
end
|
65
|
+
end
|
66
|
+
|
67
|
+
def validate_bucket(bucket, index)
|
68
|
+
if bucket.nil? || bucket.empty?
|
69
|
+
$stderr.puts "Backup(#{index}) bucket is empty: #{bucket.inspect}"
|
70
|
+
exit 1
|
71
|
+
end
|
72
|
+
end
|
73
|
+
|
74
|
+
def validate_path(path, index)
|
75
|
+
if path.nil? || path.empty?
|
76
|
+
$stderr.puts "Backup(#{index}) path is empty: #{path.inspect}"
|
77
|
+
exit 1
|
78
|
+
end
|
79
|
+
end
|
80
|
+
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: bkp
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.0.
|
4
|
+
version: 1.0.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Kazuyoshi Tlacaelel
|
@@ -48,6 +48,9 @@ files:
|
|
48
48
|
- Gemfile
|
49
49
|
- Onafile
|
50
50
|
- bin/bkp
|
51
|
+
- lib/bkp.rb
|
52
|
+
- lib/helpers.rb
|
53
|
+
- lib/validations.rb
|
51
54
|
homepage: https://github.com/ktlacaelel/bkp
|
52
55
|
licenses:
|
53
56
|
- MIT
|
@@ -70,5 +73,5 @@ requirements: []
|
|
70
73
|
rubygems_version: 3.3.7
|
71
74
|
signing_key:
|
72
75
|
specification_version: 4
|
73
|
-
summary: bkp - Backup Management Tool
|
76
|
+
summary: bkp - Shell Backup Management Tool (for AWS S3 Bucket)
|
74
77
|
test_files: []
|