chef_backup 0.0.1.dev.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.gitignore +23 -0
- data/.kitchen.yml +30 -0
- data/.rubocop.yml +17 -0
- data/.travis.yml +6 -0
- data/Gemfile +4 -0
- data/Guardfile +22 -0
- data/LICENSE +13 -0
- data/README.md +33 -0
- data/Rakefile +44 -0
- data/chef_backup.gemspec +33 -0
- data/lib/chef_backup.rb +12 -0
- data/lib/chef_backup/config.rb +57 -0
- data/lib/chef_backup/data_map.rb +44 -0
- data/lib/chef_backup/exceptions.rb +12 -0
- data/lib/chef_backup/helpers.rb +159 -0
- data/lib/chef_backup/logger.rb +39 -0
- data/lib/chef_backup/runner.rb +136 -0
- data/lib/chef_backup/strategy.rb +29 -0
- data/lib/chef_backup/strategy/backup/custom.rb +7 -0
- data/lib/chef_backup/strategy/backup/ebs.rb +28 -0
- data/lib/chef_backup/strategy/backup/lvm.rb +42 -0
- data/lib/chef_backup/strategy/backup/object.rb +29 -0
- data/lib/chef_backup/strategy/backup/tar.rb +165 -0
- data/lib/chef_backup/strategy/restore/custom.rb +0 -0
- data/lib/chef_backup/strategy/restore/ebs.rb +0 -0
- data/lib/chef_backup/strategy/restore/lvm.rb +0 -0
- data/lib/chef_backup/strategy/restore/object.rb +0 -0
- data/lib/chef_backup/strategy/restore/tar.rb +125 -0
- data/lib/chef_backup/version.rb +4 -0
- data/spec/fixtures/chef-server-running.json +584 -0
- data/spec/spec_helper.rb +103 -0
- data/spec/unit/data_map_spec.rb +59 -0
- data/spec/unit/helpers_spec.rb +88 -0
- data/spec/unit/runner_spec.rb +185 -0
- data/spec/unit/shared_examples/helpers.rb +20 -0
- data/spec/unit/strategy/backup/lvm_spec.rb +0 -0
- data/spec/unit/strategy/backup/shared_examples/backup.rb +74 -0
- data/spec/unit/strategy/backup/tar_spec.rb +280 -0
- data/spec/unit/strategy/restore/lvm_spec.rb +0 -0
- data/spec/unit/strategy/restore/shared_examples/restore.rb +84 -0
- data/spec/unit/strategy/restore/tar_spec.rb +238 -0
- data/spec/unit/strategy_spec.rb +36 -0
- metadata +253 -0
@@ -0,0 +1,39 @@
|
|
1
|
+
require 'highline'
|
2
|
+
|
3
|
+
module ChefBackup
|
4
|
+
# Basic Logging Class
|
5
|
+
class Logger
|
6
|
+
def self.logger(logfile = nil)
|
7
|
+
@logger = nil if @logger && logfile && @logger.stdout != logfile
|
8
|
+
@logger ||= new(logfile)
|
9
|
+
end
|
10
|
+
|
11
|
+
def self.log(msg, level = :info)
|
12
|
+
logger.log(msg, level)
|
13
|
+
end
|
14
|
+
|
15
|
+
attr_accessor :stdout
|
16
|
+
|
17
|
+
def initialize(logfile = nil)
|
18
|
+
@stdout = logfile || $stdout
|
19
|
+
@highline = HighLine.new($stdin, @stdout)
|
20
|
+
end
|
21
|
+
|
22
|
+
def log(msg, level = :info)
|
23
|
+
case level
|
24
|
+
when :warn
|
25
|
+
msg = "WARNING: #{msg}"
|
26
|
+
@stdout.puts(color? ? @highline.color(msg, :yellow) : msg)
|
27
|
+
when :error
|
28
|
+
msg = "ERROR: #{msg}"
|
29
|
+
@stdout.puts(color? ? @highline.color(msg, :red) : msg)
|
30
|
+
else
|
31
|
+
@stdout.puts(msg)
|
32
|
+
end
|
33
|
+
end
|
34
|
+
|
35
|
+
def color?
|
36
|
+
@stdout.tty?
|
37
|
+
end
|
38
|
+
end
|
39
|
+
end
|
@@ -0,0 +1,136 @@
|
|
1
|
+
require 'fileutils'
|
2
|
+
require 'pathname'
|
3
|
+
|
4
|
+
module ChefBackup
|
5
|
+
# ChefBackup::Runner class initializes the strategy and runs the action
|
6
|
+
class Runner
|
7
|
+
include ChefBackup::Helpers
|
8
|
+
include ChefBackup::Exceptions
|
9
|
+
|
10
|
+
attr_reader :restore_param
|
11
|
+
|
12
|
+
#
|
13
|
+
# @param running_config [Hash] A hash of the private-chef-running.json
|
14
|
+
# or the CLI args for a restore
|
15
|
+
#
|
16
|
+
# @return [ChefBackup::Runner]
|
17
|
+
#
|
18
|
+
def initialize(running_config)
|
19
|
+
ChefBackup::Config.config = running_config
|
20
|
+
ChefBackup::Logger.logger(private_chef['backup']['logfile'] || nil)
|
21
|
+
end
|
22
|
+
|
23
|
+
#
|
24
|
+
# @return [TrueClass, FalseClass] Execute Chef Server backup
|
25
|
+
#
|
26
|
+
def backup
|
27
|
+
@backup ||= ChefBackup::Strategy.backup(backup_strategy)
|
28
|
+
@backup.backup
|
29
|
+
end
|
30
|
+
|
31
|
+
#
|
32
|
+
# @return [TrueClass, FalseClass] Execute Chef Server restore
|
33
|
+
#
|
34
|
+
def restore
|
35
|
+
@restore ||= ChefBackup::Strategy.restore(restore_strategy, restore_param)
|
36
|
+
@restore.restore
|
37
|
+
end
|
38
|
+
|
39
|
+
#
|
40
|
+
# @return [String] String name of the configured backup strategy
|
41
|
+
#
|
42
|
+
def backup_strategy
|
43
|
+
config['private_chef']['backup']['strategy']
|
44
|
+
end
|
45
|
+
|
46
|
+
#
|
47
|
+
# @return [String] String of the restore parameter. eg: EBS snapshot ID
|
48
|
+
# or a path to a tarball
|
49
|
+
#
|
50
|
+
def restore_param
|
51
|
+
config['restore_param']
|
52
|
+
end
|
53
|
+
|
54
|
+
#
|
55
|
+
# @return [String] A path to backup tarball or EBS snapshot ID
|
56
|
+
#
|
57
|
+
def restore_strategy
|
58
|
+
@restore_strategy ||= begin
|
59
|
+
if tarball?
|
60
|
+
unpack_tarball
|
61
|
+
manifest['strategy']
|
62
|
+
elsif ebs_snapshot?
|
63
|
+
'ebs'
|
64
|
+
else
|
65
|
+
fail InvalidStrategy, "#{restore_param} is not a valid backup"
|
66
|
+
end
|
67
|
+
end
|
68
|
+
end
|
69
|
+
|
70
|
+
#
|
71
|
+
# @return [TrueClass, FalseClass] Is the restore_param is a tarball?
|
72
|
+
#
|
73
|
+
def tarball?
|
74
|
+
file = Pathname.new(File.expand_path(restore_param))
|
75
|
+
file.exist? && file.extname == '.tgz'
|
76
|
+
end
|
77
|
+
|
78
|
+
#
|
79
|
+
# @return [TrueClass, FalseClass] Is the restore_param an EBS Snapshot ID?
|
80
|
+
#
|
81
|
+
def ebs_snapshot?
|
82
|
+
restore_param =~ /^snap-\h{8}$/
|
83
|
+
end
|
84
|
+
|
85
|
+
#
|
86
|
+
# @return [TrueClass, FalseClass] Expands tarball into restore directory
|
87
|
+
#
|
88
|
+
def unpack_tarball
|
89
|
+
file = File.expand_path(restore_param)
|
90
|
+
ensure_file!(file, InvalidTarball, "#{file} not found")
|
91
|
+
log "Expanding tarball: #{file}"
|
92
|
+
shell_out!("tar zxf #{file} -C #{restore_directory}")
|
93
|
+
end
|
94
|
+
|
95
|
+
#
|
96
|
+
# @return [String] The backup name from the restore param
|
97
|
+
#
|
98
|
+
def backup_name
|
99
|
+
if tarball?
|
100
|
+
Pathname.new(restore_param).basename.sub_ext('').to_s
|
101
|
+
elsif ebs_snapshot?
|
102
|
+
restore_param
|
103
|
+
end
|
104
|
+
end
|
105
|
+
|
106
|
+
#
|
107
|
+
# Sets the restore_dir in ChefBackup::Config and ensures the directory
|
108
|
+
# exists and is cleaned.
|
109
|
+
#
|
110
|
+
# @return [String] A path to the restore directory
|
111
|
+
#
|
112
|
+
def restore_directory
|
113
|
+
config['restore_dir'] ||= begin
|
114
|
+
dir_name = File.join(tmp_dir, backup_name)
|
115
|
+
if File.directory?(dir_name)
|
116
|
+
# clean restore directory if it exists
|
117
|
+
FileUtils.rm_r(Dir.glob("#{dir_name}/*"))
|
118
|
+
else
|
119
|
+
FileUtils.mkdir_p(dir_name)
|
120
|
+
end
|
121
|
+
dir_name
|
122
|
+
end
|
123
|
+
end
|
124
|
+
|
125
|
+
#
|
126
|
+
# @return [Hash] A parsed copy of the manifest.json in the backup tarball
|
127
|
+
#
|
128
|
+
def manifest
|
129
|
+
@manifest ||= begin
|
130
|
+
file = "#{restore_directory}/manifest.json"
|
131
|
+
ensure_file!(file, InvalidTarball, 'No manifest found in tarball')
|
132
|
+
JSON.parse(File.read(file))
|
133
|
+
end
|
134
|
+
end
|
135
|
+
end
|
136
|
+
end
|
@@ -0,0 +1,29 @@
|
|
1
|
+
# Copyright:: Copyright (c) 2014 Chef, Inc.
|
2
|
+
#
|
3
|
+
# All Rights Reserved
|
4
|
+
|
5
|
+
require 'chef_backup/strategy/backup/tar'
|
6
|
+
require 'chef_backup/strategy/backup/lvm'
|
7
|
+
require 'chef_backup/strategy/backup/ebs'
|
8
|
+
require 'chef_backup/strategy/backup/object'
|
9
|
+
require 'chef_backup/strategy/backup/custom'
|
10
|
+
require 'chef_backup/strategy/restore/tar'
|
11
|
+
require 'chef_backup/strategy/restore/lvm'
|
12
|
+
require 'chef_backup/strategy/restore/ebs'
|
13
|
+
require 'chef_backup/strategy/restore/object'
|
14
|
+
require 'chef_backup/strategy/restore/custom'
|
15
|
+
|
16
|
+
module ChefBackup
|
17
|
+
# ChefBackup::Strategy factory returns an ChefBackup::Strategy object
|
18
|
+
module Strategy
|
19
|
+
class << self
|
20
|
+
def backup(strategy)
|
21
|
+
const_get("#{strategy.capitalize}Backup").new
|
22
|
+
end
|
23
|
+
|
24
|
+
def restore(strategy, param)
|
25
|
+
const_get("#{strategy.capitalize}Restore").new(param)
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
@@ -0,0 +1,28 @@
|
|
1
|
+
# ChefBackup::Ebs class. To be used when/if Enterprise Chef ever supports EBS
|
2
|
+
# natively
|
3
|
+
class ChefBackup::Strategy::EbsBackup < ChefBackup::Strategy::TarBackup
|
4
|
+
# - ebs
|
5
|
+
# - Verify AWS credentials exist
|
6
|
+
# - Create backup manifest
|
7
|
+
# - Create backup dir on ebs volume
|
8
|
+
# - Copy /etc/opscode and manifest onto ebs volume
|
9
|
+
# - Take EBS snapshot
|
10
|
+
def backup
|
11
|
+
ensure_tmp_dir
|
12
|
+
verify_ebs
|
13
|
+
dump_db if pg_dump?
|
14
|
+
create_manifest
|
15
|
+
copy_opscode_config
|
16
|
+
take_ebs_snapshot
|
17
|
+
cleanup
|
18
|
+
end
|
19
|
+
|
20
|
+
def verify_ebs
|
21
|
+
end
|
22
|
+
|
23
|
+
def take_ebs_snapshot
|
24
|
+
end
|
25
|
+
|
26
|
+
def copy_opscode_config
|
27
|
+
end
|
28
|
+
end
|
@@ -0,0 +1,42 @@
|
|
1
|
+
# ChefBackup::Lvm class. Class used when Enterprise Chef's stateful services
|
2
|
+
# are on an LVM logical volume. This may allow us to not require a pg_dump,
|
3
|
+
# instead relying on pg_crash recovery. Not doing a pg_dump will greatly
|
4
|
+
# speed up backups.
|
5
|
+
class ChefBackup::Strategy::LvmBackup < ChefBackup::Strategy::TarBackup
|
6
|
+
# - verify config
|
7
|
+
# - Verify lv and vg existence
|
8
|
+
# - Warn if space is low in vg for an lvm snapshot
|
9
|
+
# - Ensure backup directory exists
|
10
|
+
# - Create temp backup dir
|
11
|
+
# - Do DB dump (if we have to)
|
12
|
+
# - Create backup manifest
|
13
|
+
# - Take LVM snapshot
|
14
|
+
# - Mount LVM snapshot
|
15
|
+
# - Create a gzipped tarball of all required files
|
16
|
+
# - db dump (if we have to)
|
17
|
+
# - /etc/opscode
|
18
|
+
# - Backup manifest
|
19
|
+
# - mounted LVM snapshot
|
20
|
+
# - Cleanup tmp directories
|
21
|
+
def backup
|
22
|
+
ensure_tmp_dir
|
23
|
+
verify_lvm
|
24
|
+
dump_db if pg_dump?
|
25
|
+
create_manifest
|
26
|
+
take_lvm_snapshot
|
27
|
+
mount_lvm_snapshot
|
28
|
+
create_tarball
|
29
|
+
cleanup
|
30
|
+
end
|
31
|
+
|
32
|
+
def verify_lvm
|
33
|
+
# verify lv and vg are available
|
34
|
+
# warn if vg space is low
|
35
|
+
end
|
36
|
+
|
37
|
+
def take_lvm_snapshot
|
38
|
+
end
|
39
|
+
|
40
|
+
def mount_lvm_snapshot
|
41
|
+
end
|
42
|
+
end
|
@@ -0,0 +1,29 @@
|
|
1
|
+
# ChefBackup::Object class. Used to backup all stateful services as an object
|
2
|
+
# and stored locally in a filesystem structure in JSON.
|
3
|
+
class ChefBackup::Strategy::ObjectBackup < ChefBackup::Strategy::TarBackup
|
4
|
+
# - object
|
5
|
+
# - Make tmp directory
|
6
|
+
# - Ensure backup directory exists
|
7
|
+
# - Warn if backups directory or tmp are low on space
|
8
|
+
# - knife-ec-backup into temp directory
|
9
|
+
# - Create backup manifest
|
10
|
+
# - Create gzipped tarball of all required files
|
11
|
+
# - knife ec backup dump
|
12
|
+
# - /etc/opscode
|
13
|
+
# - Backup manifest
|
14
|
+
# - Cleanup tmp directories
|
15
|
+
def backup
|
16
|
+
ensure_tmp_dir
|
17
|
+
verify_object
|
18
|
+
knife_ec_backup
|
19
|
+
create_manifest
|
20
|
+
create_tarball
|
21
|
+
cleanup
|
22
|
+
end
|
23
|
+
|
24
|
+
def verify_object
|
25
|
+
end
|
26
|
+
|
27
|
+
def knife_ec_backup
|
28
|
+
end
|
29
|
+
end
|
@@ -0,0 +1,165 @@
|
|
1
|
+
require 'fileutils'
|
2
|
+
require 'json'
|
3
|
+
require 'time'
|
4
|
+
|
5
|
+
# rubocop:disable IndentationWidth
|
6
|
+
module ChefBackup
|
7
|
+
module Strategy
|
8
|
+
# ChefBackup::Tar class. Used to backup Standalone and Tier Servers that aren't
|
9
|
+
# installed on LVM
|
10
|
+
class TarBackup
|
11
|
+
# rubocop:enable IndentationWidth
|
12
|
+
include ChefBackup::Helpers
|
13
|
+
include ChefBackup::Exceptions
|
14
|
+
|
15
|
+
attr_reader :backup_time
|
16
|
+
|
17
|
+
def initialize
|
18
|
+
@backup_time = Time.now.strftime('%Y-%m-%d-%H-%M-%S')
|
19
|
+
end
|
20
|
+
|
21
|
+
#
|
22
|
+
# Ensures existence of an export directory for the backup
|
23
|
+
#
|
24
|
+
# @return [String] A path to the export_dir
|
25
|
+
#
|
26
|
+
def export_dir
|
27
|
+
@export_dir ||= begin
|
28
|
+
dir =
|
29
|
+
if private_chef['backup']['export_dir']
|
30
|
+
private_chef['backup']['export_dir']
|
31
|
+
else
|
32
|
+
msg = ["backup['export_dir'] has not been set.",
|
33
|
+
'defaulting to: /var/opt/chef-backups'].join(' ')
|
34
|
+
log(msg, :warn)
|
35
|
+
'/var/opt/chef-backups'
|
36
|
+
end
|
37
|
+
FileUtils.mkdir_p(dir) unless File.directory?(dir)
|
38
|
+
dir
|
39
|
+
end
|
40
|
+
end
|
41
|
+
|
42
|
+
#
|
43
|
+
# Perform a pg_dump
|
44
|
+
#
|
45
|
+
# @return [TrueClass, FalseClass]
|
46
|
+
#
|
47
|
+
def dump_db
|
48
|
+
return true unless pg_dump?
|
49
|
+
pg_user = private_chef['postgresql']['username']
|
50
|
+
sql_file = "#{tmp_dir}/chef_backup-#{backup_time}.sql"
|
51
|
+
cmd = ['/opt/opscode/embedded/bin/chpst',
|
52
|
+
"-u #{pg_user}",
|
53
|
+
'/opt/opscode/embedded/bin/pg_dumpall',
|
54
|
+
"> #{sql_file}"
|
55
|
+
].join(' ')
|
56
|
+
log "Dumping Postgresql database to #{sql_file}"
|
57
|
+
shell_out!(cmd)
|
58
|
+
data_map.services['postgresql']['pg_dump_success'] = true
|
59
|
+
data_map.services['postgresql']['username'] = pg_user
|
60
|
+
true
|
61
|
+
end
|
62
|
+
|
63
|
+
def populate_data_map
|
64
|
+
stateful_services.each do |service|
|
65
|
+
next unless private_chef.key?(service)
|
66
|
+
data_map.add_service(service, private_chef[service]['data_dir'])
|
67
|
+
end
|
68
|
+
|
69
|
+
config_directories.each do |config|
|
70
|
+
data_map.add_config(config, "/etc/#{config}")
|
71
|
+
end
|
72
|
+
|
73
|
+
# Don't forget the upgrades!
|
74
|
+
if private_chef.key?('upgrades')
|
75
|
+
data_map.add_service('upgrades', private_chef['upgrades']['dir'])
|
76
|
+
end
|
77
|
+
end
|
78
|
+
|
79
|
+
def manifest
|
80
|
+
data_map.manifest
|
81
|
+
end
|
82
|
+
|
83
|
+
def write_manifest
|
84
|
+
log 'Writing backup manifest'
|
85
|
+
File.open("#{tmp_dir}/manifest.json", 'w') do |file|
|
86
|
+
file.write(JSON.pretty_generate(manifest))
|
87
|
+
end
|
88
|
+
end
|
89
|
+
|
90
|
+
def stateful_services
|
91
|
+
if private_chef.key?('drbd') && private_chef['drbd']['enable'] == true
|
92
|
+
['drbd']
|
93
|
+
else
|
94
|
+
%w(
|
95
|
+
rabbitmq
|
96
|
+
opscode-solr4
|
97
|
+
redis_lb
|
98
|
+
postgresql
|
99
|
+
bookshelf
|
100
|
+
)
|
101
|
+
end
|
102
|
+
end
|
103
|
+
|
104
|
+
def config_directories
|
105
|
+
%w(opscode) + enabled_addons
|
106
|
+
end
|
107
|
+
|
108
|
+
# The data_map is a working record of all of the data that is backed up.
|
109
|
+
def data_map
|
110
|
+
@data_map ||= ChefBackup::DataMap.new do |data|
|
111
|
+
data.backup_time = backup_time
|
112
|
+
data.strategy = strategy
|
113
|
+
end
|
114
|
+
end
|
115
|
+
|
116
|
+
def not_implemented
|
117
|
+
msg = "#{caller[0].split[1]} is not implemented for this strategy"
|
118
|
+
fail NotImplementedError, msg
|
119
|
+
end
|
120
|
+
|
121
|
+
def backup
|
122
|
+
log 'Starting Chef Server backup'
|
123
|
+
populate_data_map
|
124
|
+
if backend?
|
125
|
+
stop_chef_server(except: [:keepalived, :postgresql]) unless online?
|
126
|
+
dump_db
|
127
|
+
stop_service(:postgresql) unless online?
|
128
|
+
end
|
129
|
+
write_manifest
|
130
|
+
create_tarball
|
131
|
+
start_chef_server if backend? && !online?
|
132
|
+
export_tarball
|
133
|
+
cleanup
|
134
|
+
log 'Backup Complete!'
|
135
|
+
rescue => e
|
136
|
+
log "Something wen't terribly wrong, aborting backup", :error
|
137
|
+
log e.message, :error
|
138
|
+
cleanup
|
139
|
+
start_chef_server
|
140
|
+
raise e
|
141
|
+
end
|
142
|
+
|
143
|
+
def create_tarball
|
144
|
+
log 'Creating backup tarball'
|
145
|
+
cmd = [
|
146
|
+
"tar -czf #{tmp_dir}/chef-backup-#{backup_time}.tgz",
|
147
|
+
data_map.services.map { |_, v| v['data_dir'] }.compact.join(' '),
|
148
|
+
data_map.configs.map { |_, v| v['data_dir'] }.compact.join(' '),
|
149
|
+
Dir["#{tmp_dir}/*"].map { |f| File.basename(f) }.join(' ')
|
150
|
+
].join(' ').strip
|
151
|
+
|
152
|
+
res = shell_out(cmd, cwd: tmp_dir)
|
153
|
+
res
|
154
|
+
end
|
155
|
+
|
156
|
+
def export_tarball
|
157
|
+
log "Exporting tarball to #{export_dir}"
|
158
|
+
cmd = "rsync -chaz #{tmp_dir}/chef-backup-#{backup_time}.tgz #{export_dir}/"
|
159
|
+
|
160
|
+
res = shell_out(cmd)
|
161
|
+
res
|
162
|
+
end
|
163
|
+
end
|
164
|
+
end
|
165
|
+
end
|