chef_backup 0.0.1.dev.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (44) hide show
  1. checksums.yaml +7 -0
  2. data/.gitignore +23 -0
  3. data/.kitchen.yml +30 -0
  4. data/.rubocop.yml +17 -0
  5. data/.travis.yml +6 -0
  6. data/Gemfile +4 -0
  7. data/Guardfile +22 -0
  8. data/LICENSE +13 -0
  9. data/README.md +33 -0
  10. data/Rakefile +44 -0
  11. data/chef_backup.gemspec +33 -0
  12. data/lib/chef_backup.rb +12 -0
  13. data/lib/chef_backup/config.rb +57 -0
  14. data/lib/chef_backup/data_map.rb +44 -0
  15. data/lib/chef_backup/exceptions.rb +12 -0
  16. data/lib/chef_backup/helpers.rb +159 -0
  17. data/lib/chef_backup/logger.rb +39 -0
  18. data/lib/chef_backup/runner.rb +136 -0
  19. data/lib/chef_backup/strategy.rb +29 -0
  20. data/lib/chef_backup/strategy/backup/custom.rb +7 -0
  21. data/lib/chef_backup/strategy/backup/ebs.rb +28 -0
  22. data/lib/chef_backup/strategy/backup/lvm.rb +42 -0
  23. data/lib/chef_backup/strategy/backup/object.rb +29 -0
  24. data/lib/chef_backup/strategy/backup/tar.rb +165 -0
  25. data/lib/chef_backup/strategy/restore/custom.rb +0 -0
  26. data/lib/chef_backup/strategy/restore/ebs.rb +0 -0
  27. data/lib/chef_backup/strategy/restore/lvm.rb +0 -0
  28. data/lib/chef_backup/strategy/restore/object.rb +0 -0
  29. data/lib/chef_backup/strategy/restore/tar.rb +125 -0
  30. data/lib/chef_backup/version.rb +4 -0
  31. data/spec/fixtures/chef-server-running.json +584 -0
  32. data/spec/spec_helper.rb +103 -0
  33. data/spec/unit/data_map_spec.rb +59 -0
  34. data/spec/unit/helpers_spec.rb +88 -0
  35. data/spec/unit/runner_spec.rb +185 -0
  36. data/spec/unit/shared_examples/helpers.rb +20 -0
  37. data/spec/unit/strategy/backup/lvm_spec.rb +0 -0
  38. data/spec/unit/strategy/backup/shared_examples/backup.rb +74 -0
  39. data/spec/unit/strategy/backup/tar_spec.rb +280 -0
  40. data/spec/unit/strategy/restore/lvm_spec.rb +0 -0
  41. data/spec/unit/strategy/restore/shared_examples/restore.rb +84 -0
  42. data/spec/unit/strategy/restore/tar_spec.rb +238 -0
  43. data/spec/unit/strategy_spec.rb +36 -0
  44. metadata +253 -0
@@ -0,0 +1,39 @@
1
+ require 'highline'
2
+
3
+ module ChefBackup
4
+ # Basic Logging Class
5
+ class Logger
6
+ def self.logger(logfile = nil)
7
+ @logger = nil if @logger && logfile && @logger.stdout != logfile
8
+ @logger ||= new(logfile)
9
+ end
10
+
11
+ def self.log(msg, level = :info)
12
+ logger.log(msg, level)
13
+ end
14
+
15
+ attr_accessor :stdout
16
+
17
+ def initialize(logfile = nil)
18
+ @stdout = logfile || $stdout
19
+ @highline = HighLine.new($stdin, @stdout)
20
+ end
21
+
22
+ def log(msg, level = :info)
23
+ case level
24
+ when :warn
25
+ msg = "WARNING: #{msg}"
26
+ @stdout.puts(color? ? @highline.color(msg, :yellow) : msg)
27
+ when :error
28
+ msg = "ERROR: #{msg}"
29
+ @stdout.puts(color? ? @highline.color(msg, :red) : msg)
30
+ else
31
+ @stdout.puts(msg)
32
+ end
33
+ end
34
+
35
+ def color?
36
+ @stdout.tty?
37
+ end
38
+ end
39
+ end
@@ -0,0 +1,136 @@
1
+ require 'fileutils'
2
+ require 'pathname'
3
+
4
+ module ChefBackup
5
+ # ChefBackup::Runner class initializes the strategy and runs the action
6
+ class Runner
7
+ include ChefBackup::Helpers
8
+ include ChefBackup::Exceptions
9
+
10
+ attr_reader :restore_param
11
+
12
+ #
13
+ # @param running_config [Hash] A hash of the private-chef-running.json
14
+ # or the CLI args for a restore
15
+ #
16
+ # @return [ChefBackup::Runner]
17
+ #
18
+ def initialize(running_config)
19
+ ChefBackup::Config.config = running_config
20
+ ChefBackup::Logger.logger(private_chef['backup']['logfile'] || nil)
21
+ end
22
+
23
+ #
24
+ # @return [TrueClass, FalseClass] Execute Chef Server backup
25
+ #
26
+ def backup
27
+ @backup ||= ChefBackup::Strategy.backup(backup_strategy)
28
+ @backup.backup
29
+ end
30
+
31
+ #
32
+ # @return [TrueClass, FalseClass] Execute Chef Server restore
33
+ #
34
+ def restore
35
+ @restore ||= ChefBackup::Strategy.restore(restore_strategy, restore_param)
36
+ @restore.restore
37
+ end
38
+
39
+ #
40
+ # @return [String] String name of the configured backup strategy
41
+ #
42
+ def backup_strategy
43
+ config['private_chef']['backup']['strategy']
44
+ end
45
+
46
+ #
47
+ # @return [String] String of the restore parameter. eg: EBS snapshot ID
48
+ # or a path to a tarball
49
+ #
50
+ def restore_param
51
+ config['restore_param']
52
+ end
53
+
54
+ #
55
+ # @return [String] A path to backup tarball or EBS snapshot ID
56
+ #
57
+ def restore_strategy
58
+ @restore_strategy ||= begin
59
+ if tarball?
60
+ unpack_tarball
61
+ manifest['strategy']
62
+ elsif ebs_snapshot?
63
+ 'ebs'
64
+ else
65
+ fail InvalidStrategy, "#{restore_param} is not a valid backup"
66
+ end
67
+ end
68
+ end
69
+
70
+ #
71
+ # @return [TrueClass, FalseClass] Is the restore_param is a tarball?
72
+ #
73
+ def tarball?
74
+ file = Pathname.new(File.expand_path(restore_param))
75
+ file.exist? && file.extname == '.tgz'
76
+ end
77
+
78
+ #
79
+ # @return [TrueClass, FalseClass] Is the restore_param an EBS Snapshot ID?
80
+ #
81
+ def ebs_snapshot?
82
+ restore_param =~ /^snap-\h{8}$/
83
+ end
84
+
85
+ #
86
+ # @return [TrueClass, FalseClass] Expands tarball into restore directory
87
+ #
88
+ def unpack_tarball
89
+ file = File.expand_path(restore_param)
90
+ ensure_file!(file, InvalidTarball, "#{file} not found")
91
+ log "Expanding tarball: #{file}"
92
+ shell_out!("tar zxf #{file} -C #{restore_directory}")
93
+ end
94
+
95
+ #
96
+ # @return [String] The backup name from the restore param
97
+ #
98
+ def backup_name
99
+ if tarball?
100
+ Pathname.new(restore_param).basename.sub_ext('').to_s
101
+ elsif ebs_snapshot?
102
+ restore_param
103
+ end
104
+ end
105
+
106
+ #
107
+ # Sets the restore_dir in ChefBackup::Config and ensures the directory
108
+ # exists and is cleaned.
109
+ #
110
+ # @return [String] A path to the restore directory
111
+ #
112
+ def restore_directory
113
+ config['restore_dir'] ||= begin
114
+ dir_name = File.join(tmp_dir, backup_name)
115
+ if File.directory?(dir_name)
116
+ # clean restore directory if it exists
117
+ FileUtils.rm_r(Dir.glob("#{dir_name}/*"))
118
+ else
119
+ FileUtils.mkdir_p(dir_name)
120
+ end
121
+ dir_name
122
+ end
123
+ end
124
+
125
+ #
126
+ # @return [Hash] A parsed copy of the manifest.json in the backup tarball
127
+ #
128
+ def manifest
129
+ @manifest ||= begin
130
+ file = "#{restore_directory}/manifest.json"
131
+ ensure_file!(file, InvalidTarball, 'No manifest found in tarball')
132
+ JSON.parse(File.read(file))
133
+ end
134
+ end
135
+ end
136
+ end
@@ -0,0 +1,29 @@
1
+ # Copyright:: Copyright (c) 2014 Chef, Inc.
2
+ #
3
+ # All Rights Reserved
4
+
5
+ require 'chef_backup/strategy/backup/tar'
6
+ require 'chef_backup/strategy/backup/lvm'
7
+ require 'chef_backup/strategy/backup/ebs'
8
+ require 'chef_backup/strategy/backup/object'
9
+ require 'chef_backup/strategy/backup/custom'
10
+ require 'chef_backup/strategy/restore/tar'
11
+ require 'chef_backup/strategy/restore/lvm'
12
+ require 'chef_backup/strategy/restore/ebs'
13
+ require 'chef_backup/strategy/restore/object'
14
+ require 'chef_backup/strategy/restore/custom'
15
+
16
+ module ChefBackup
17
+ # ChefBackup::Strategy factory returns an ChefBackup::Strategy object
18
+ module Strategy
19
+ class << self
20
+ def backup(strategy)
21
+ const_get("#{strategy.capitalize}Backup").new
22
+ end
23
+
24
+ def restore(strategy, param)
25
+ const_get("#{strategy.capitalize}Restore").new(param)
26
+ end
27
+ end
28
+ end
29
+ end
@@ -0,0 +1,7 @@
1
+ # - custom
2
+ # - Verify that backup executable exists and is runnable
3
+ # - exec script
4
+ class ChefBackup::Strategy::CustomBackup < ChefBackup::Strategy::TarBackup
5
+ def backup
6
+ end
7
+ end
@@ -0,0 +1,28 @@
1
+ # ChefBackup::Ebs class. To be used when/if Enterprise Chef ever supports EBS
2
+ # natively
3
+ class ChefBackup::Strategy::EbsBackup < ChefBackup::Strategy::TarBackup
4
+ # - ebs
5
+ # - Verify AWS credentials exist
6
+ # - Create backup manifest
7
+ # - Create backup dir on ebs volume
8
+ # - Copy /etc/opscode and manifest onto ebs volume
9
+ # - Take EBS snapshot
10
+ def backup
11
+ ensure_tmp_dir
12
+ verify_ebs
13
+ dump_db if pg_dump?
14
+ create_manifest
15
+ copy_opscode_config
16
+ take_ebs_snapshot
17
+ cleanup
18
+ end
19
+
20
+ def verify_ebs
21
+ end
22
+
23
+ def take_ebs_snapshot
24
+ end
25
+
26
+ def copy_opscode_config
27
+ end
28
+ end
@@ -0,0 +1,42 @@
1
+ # ChefBackup::Lvm class. Class used when Enterprise Chef's stateful services
2
+ # are on an LVM logical volume. This may allow us to not require a pg_dump,
3
+ # instead relying on pg_crash recovery. Not doing a pg_dump will greatly
4
+ # speed up backups.
5
+ class ChefBackup::Strategy::LvmBackup < ChefBackup::Strategy::TarBackup
6
+ # - verify config
7
+ # - Verify lv and vg existence
8
+ # - Warn if space is low in vg for an lvm snapshot
9
+ # - Ensure backup directory exists
10
+ # - Create temp backup dir
11
+ # - Do DB dump (if we have to)
12
+ # - Create backup manifest
13
+ # - Take LVM snapshot
14
+ # - Mount LVM snapshot
15
+ # - Create a gzipped tarball of all required files
16
+ # - db dump (if we have to)
17
+ # - /etc/opscode
18
+ # - Backup manifest
19
+ # - mounted LVM snapshot
20
+ # - Cleanup tmp directories
21
+ def backup
22
+ ensure_tmp_dir
23
+ verify_lvm
24
+ dump_db if pg_dump?
25
+ create_manifest
26
+ take_lvm_snapshot
27
+ mount_lvm_snapshot
28
+ create_tarball
29
+ cleanup
30
+ end
31
+
32
+ def verify_lvm
33
+ # verify lv and vg are available
34
+ # warn if vg space is low
35
+ end
36
+
37
+ def take_lvm_snapshot
38
+ end
39
+
40
+ def mount_lvm_snapshot
41
+ end
42
+ end
@@ -0,0 +1,29 @@
1
+ # ChefBackup::Object class. Used to backup all stateful services as an object
2
+ # and stored locally in a filesystem structure in JSON.
3
+ class ChefBackup::Strategy::ObjectBackup < ChefBackup::Strategy::TarBackup
4
+ # - object
5
+ # - Make tmp directory
6
+ # - Ensure backup directory exists
7
+ # - Warn if backups directory or tmp are low on space
8
+ # - knife-ec-backup into temp directory
9
+ # - Create backup manifest
10
+ # - Create gzipped tarball of all required files
11
+ # - knife ec backup dump
12
+ # - /etc/opscode
13
+ # - Backup manifest
14
+ # - Cleanup tmp directories
15
+ def backup
16
+ ensure_tmp_dir
17
+ verify_object
18
+ knife_ec_backup
19
+ create_manifest
20
+ create_tarball
21
+ cleanup
22
+ end
23
+
24
+ def verify_object
25
+ end
26
+
27
+ def knife_ec_backup
28
+ end
29
+ end
@@ -0,0 +1,165 @@
1
+ require 'fileutils'
2
+ require 'json'
3
+ require 'time'
4
+
5
+ # rubocop:disable IndentationWidth
6
+ module ChefBackup
7
+ module Strategy
8
+ # ChefBackup::Tar class. Used to backup Standalone and Tier Servers that aren't
9
+ # installed on LVM
10
+ class TarBackup
11
+ # rubocop:enable IndentationWidth
12
+ include ChefBackup::Helpers
13
+ include ChefBackup::Exceptions
14
+
15
+ attr_reader :backup_time
16
+
17
+ def initialize
18
+ @backup_time = Time.now.strftime('%Y-%m-%d-%H-%M-%S')
19
+ end
20
+
21
+ #
22
+ # Ensures existence of an export directory for the backup
23
+ #
24
+ # @return [String] A path to the export_dir
25
+ #
26
+ def export_dir
27
+ @export_dir ||= begin
28
+ dir =
29
+ if private_chef['backup']['export_dir']
30
+ private_chef['backup']['export_dir']
31
+ else
32
+ msg = ["backup['export_dir'] has not been set.",
33
+ 'defaulting to: /var/opt/chef-backups'].join(' ')
34
+ log(msg, :warn)
35
+ '/var/opt/chef-backups'
36
+ end
37
+ FileUtils.mkdir_p(dir) unless File.directory?(dir)
38
+ dir
39
+ end
40
+ end
41
+
42
+ #
43
+ # Perform a pg_dump
44
+ #
45
+ # @return [TrueClass, FalseClass]
46
+ #
47
+ def dump_db
48
+ return true unless pg_dump?
49
+ pg_user = private_chef['postgresql']['username']
50
+ sql_file = "#{tmp_dir}/chef_backup-#{backup_time}.sql"
51
+ cmd = ['/opt/opscode/embedded/bin/chpst',
52
+ "-u #{pg_user}",
53
+ '/opt/opscode/embedded/bin/pg_dumpall',
54
+ "> #{sql_file}"
55
+ ].join(' ')
56
+ log "Dumping Postgresql database to #{sql_file}"
57
+ shell_out!(cmd)
58
+ data_map.services['postgresql']['pg_dump_success'] = true
59
+ data_map.services['postgresql']['username'] = pg_user
60
+ true
61
+ end
62
+
63
+ def populate_data_map
64
+ stateful_services.each do |service|
65
+ next unless private_chef.key?(service)
66
+ data_map.add_service(service, private_chef[service]['data_dir'])
67
+ end
68
+
69
+ config_directories.each do |config|
70
+ data_map.add_config(config, "/etc/#{config}")
71
+ end
72
+
73
+ # Don't forget the upgrades!
74
+ if private_chef.key?('upgrades')
75
+ data_map.add_service('upgrades', private_chef['upgrades']['dir'])
76
+ end
77
+ end
78
+
79
+ def manifest
80
+ data_map.manifest
81
+ end
82
+
83
+ def write_manifest
84
+ log 'Writing backup manifest'
85
+ File.open("#{tmp_dir}/manifest.json", 'w') do |file|
86
+ file.write(JSON.pretty_generate(manifest))
87
+ end
88
+ end
89
+
90
+ def stateful_services
91
+ if private_chef.key?('drbd') && private_chef['drbd']['enable'] == true
92
+ ['drbd']
93
+ else
94
+ %w(
95
+ rabbitmq
96
+ opscode-solr4
97
+ redis_lb
98
+ postgresql
99
+ bookshelf
100
+ )
101
+ end
102
+ end
103
+
104
+ def config_directories
105
+ %w(opscode) + enabled_addons
106
+ end
107
+
108
+ # The data_map is a working record of all of the data that is backed up.
109
+ def data_map
110
+ @data_map ||= ChefBackup::DataMap.new do |data|
111
+ data.backup_time = backup_time
112
+ data.strategy = strategy
113
+ end
114
+ end
115
+
116
+ def not_implemented
117
+ msg = "#{caller[0].split[1]} is not implemented for this strategy"
118
+ fail NotImplementedError, msg
119
+ end
120
+
121
+ def backup
122
+ log 'Starting Chef Server backup'
123
+ populate_data_map
124
+ if backend?
125
+ stop_chef_server(except: [:keepalived, :postgresql]) unless online?
126
+ dump_db
127
+ stop_service(:postgresql) unless online?
128
+ end
129
+ write_manifest
130
+ create_tarball
131
+ start_chef_server if backend? && !online?
132
+ export_tarball
133
+ cleanup
134
+ log 'Backup Complete!'
135
+ rescue => e
136
+ log "Something wen't terribly wrong, aborting backup", :error
137
+ log e.message, :error
138
+ cleanup
139
+ start_chef_server
140
+ raise e
141
+ end
142
+
143
+ def create_tarball
144
+ log 'Creating backup tarball'
145
+ cmd = [
146
+ "tar -czf #{tmp_dir}/chef-backup-#{backup_time}.tgz",
147
+ data_map.services.map { |_, v| v['data_dir'] }.compact.join(' '),
148
+ data_map.configs.map { |_, v| v['data_dir'] }.compact.join(' '),
149
+ Dir["#{tmp_dir}/*"].map { |f| File.basename(f) }.join(' ')
150
+ ].join(' ').strip
151
+
152
+ res = shell_out(cmd, cwd: tmp_dir)
153
+ res
154
+ end
155
+
156
+ def export_tarball
157
+ log "Exporting tarball to #{export_dir}"
158
+ cmd = "rsync -chaz #{tmp_dir}/chef-backup-#{backup_time}.tgz #{export_dir}/"
159
+
160
+ res = shell_out(cmd)
161
+ res
162
+ end
163
+ end
164
+ end
165
+ end