Backup_GEM_2 0.1.2
Sign up to get free protection for your applications and to get access to all the features.
- data/CHANGELOG +48 -0
- data/Rakefile +26 -0
- data/TODO +7 -0
- data/bin/backup +12 -0
- data/bin/commands.sh +2 -0
- data/doc/LICENSE-GPL.txt +280 -0
- data/doc/index.html +716 -0
- data/doc/styles.css +157 -0
- data/examples/global.rb +28 -0
- data/examples/mediawiki.rb +24 -0
- data/examples/mediawiki_numeric.rb +19 -0
- data/examples/s3.rb +35 -0
- data/lib/backup.rb +23 -0
- data/lib/backup/actor.rb +208 -0
- data/lib/backup/cli.rb +144 -0
- data/lib/backup/configuration.rb +137 -0
- data/lib/backup/date_parser.rb +37 -0
- data/lib/backup/extensions.rb +17 -0
- data/lib/backup/recipes/standard.rb +113 -0
- data/lib/backup/rotator.rb +219 -0
- data/lib/backup/s3_helpers.rb +140 -0
- data/lib/backup/ssh_helpers.rb +135 -0
- data/lib/backup/state_recorder.rb +21 -0
- data/tests/actor_test.rb +69 -0
- data/tests/cleanup.sh +2 -0
- data/tests/optional/s3_test.rb +40 -0
- data/tests/rotation_test.rb +31 -0
- data/tests/s3_test.rb +40 -0
- data/tests/ssh_test.rb +21 -0
- data/tests/tests_helper.rb +5 -0
- metadata +165 -0
data/lib/backup/cli.rb
ADDED
@@ -0,0 +1,144 @@
|
|
1
|
+
require 'optparse'
|
2
|
+
require 'backup'
|
3
|
+
require 'yaml'
|
4
|
+
|
5
|
+
module Backup
|
6
|
+
# The CLI class encapsulates the behavior of backup when it is invoked
|
7
|
+
# as a command-line utility.
|
8
|
+
class CLI
|
9
|
+
# Invoke capistrano using the ARGV array as the option parameters.
|
10
|
+
def self.execute!
|
11
|
+
new.execute!
|
12
|
+
end
|
13
|
+
|
14
|
+
# The array of (unparsed) command-line options
|
15
|
+
attr_reader :args
|
16
|
+
|
17
|
+
# The hash of (parsed) command-line options
|
18
|
+
attr_reader :options
|
19
|
+
|
20
|
+
# Docs for creating a new instance go here
|
21
|
+
def initialize(args = ARGV)
|
22
|
+
@args = args
|
23
|
+
@options = { :recipes => [], :actions => [],
|
24
|
+
:vars => {}, # :pre_vars => {},
|
25
|
+
:global => nil }
|
26
|
+
|
27
|
+
OptionParser.new do |opts|
|
28
|
+
opts.banner = "Usage: #{$0} [options]"
|
29
|
+
|
30
|
+
opts.separator ""
|
31
|
+
opts.separator "Recipe Options -----------------------"
|
32
|
+
opts.separator ""
|
33
|
+
|
34
|
+
opts.on("-r", "--recipe RECIPE ",
|
35
|
+
"A recipe file to load. Multiple recipes is DEPRECATED and not fully functional."
|
36
|
+
) { |value| @options[:recipes] << value }
|
37
|
+
|
38
|
+
opts.on("-s", "--set NAME=VALUE",
|
39
|
+
"Specify a variable and it's value to set. This",
|
40
|
+
"will be set after loading all recipe files."
|
41
|
+
) do |pair|
|
42
|
+
name, value = pair.split(/=/, 2)
|
43
|
+
@options[:vars][name.to_sym] = value
|
44
|
+
end
|
45
|
+
|
46
|
+
opts.on("-g", "--global RECIPE",
|
47
|
+
"Specify a specific file to load as the global file",
|
48
|
+
"for the recipes. By default the recipes load the",
|
49
|
+
"file +global.rb+ in the same directory."
|
50
|
+
) { |value| @options[:recipes] << value }
|
51
|
+
|
52
|
+
opts.on("-q", "--quiet",
|
53
|
+
"suppresses much of the output of backup, except",
|
54
|
+
"for error messages") { verbose(false) }
|
55
|
+
|
56
|
+
if args.empty?
|
57
|
+
puts opts
|
58
|
+
exit
|
59
|
+
else
|
60
|
+
opts.parse!(args)
|
61
|
+
end
|
62
|
+
|
63
|
+
check_options!
|
64
|
+
|
65
|
+
end
|
66
|
+
|
67
|
+
end
|
68
|
+
|
69
|
+
# Begin running Backup based on the configured options.
|
70
|
+
def execute!
|
71
|
+
#if !@options[:recipes].empty? # put backk
|
72
|
+
execute_recipes!
|
73
|
+
# elsif @options[:apply_to]
|
74
|
+
# execute_apply_to!
|
75
|
+
#end
|
76
|
+
end
|
77
|
+
|
78
|
+
|
79
|
+
private
|
80
|
+
def check_options!
|
81
|
+
# perform a sanity check
|
82
|
+
end
|
83
|
+
|
84
|
+
# Load the recipes specified by the options, and execute the actions
|
85
|
+
# specified.
|
86
|
+
def execute_recipes!
|
87
|
+
config = Backup::Configuration.new
|
88
|
+
#config.logger.level = options[:verbose]
|
89
|
+
#options[:pre_vars].each { |name, value| config.set(name, value) }
|
90
|
+
options[:vars].each { |name, value| config.set(name, value) }
|
91
|
+
|
92
|
+
# load the standard recipe definition
|
93
|
+
config.load "standard"
|
94
|
+
options[:recipes].each do |recipe|
|
95
|
+
global = options[:global] || File.dirname(recipe) + "/global.rb"
|
96
|
+
config.load global if File.exists? global # cache this?
|
97
|
+
end
|
98
|
+
|
99
|
+
options[:recipes].each_with_index do |recipe,i|
|
100
|
+
config.load(recipe)
|
101
|
+
$state = setup_saved_state(recipe, config)
|
102
|
+
warn "DEPRICATED: Using multiple recipes with one command is deprecated for the time being. Just run a different command if you want to do two recipes at the same time" if i > 0
|
103
|
+
|
104
|
+
end
|
105
|
+
|
106
|
+
#options[:vars].each { |name, value| config.set(name, value) }
|
107
|
+
|
108
|
+
actor = config.actor
|
109
|
+
actor.start_process! # eventually make more options, like the ability
|
110
|
+
# to run each action individually
|
111
|
+
#options[:actions].each { |action| actor.send action }
|
112
|
+
end
|
113
|
+
|
114
|
+
|
115
|
+
# Setup the persistant state using madeline
|
116
|
+
def setup_saved_state(recipe, config)
|
117
|
+
if defined? ::NO_NUMERIC_ROTATION
|
118
|
+
if :numeric == config[:rotation_mode]
|
119
|
+
puts "Missing Gem: :numeric :rotation mode is not valid unless you have madeleine installed. try: 'gem install madeleine'"
|
120
|
+
exit 1
|
121
|
+
end
|
122
|
+
return nil
|
123
|
+
end
|
124
|
+
|
125
|
+
saved_state_folder = config[:saved_state_folder] || File.join(config[:backup_path], ".backupgem_#{File.basename(recipe)}_state")
|
126
|
+
|
127
|
+
state = SnapshotMadeleine.new(saved_state_folder, YAML) do
|
128
|
+
StateRecorder.new
|
129
|
+
end
|
130
|
+
state.system.saved_state_folder = saved_state_folder
|
131
|
+
state
|
132
|
+
end
|
133
|
+
|
134
|
+
end
|
135
|
+
end
|
136
|
+
|
137
|
+
at_exit {
|
138
|
+
|
139
|
+
if !defined?(::NO_NUMERIC_ROTATION) && defined?($state)
|
140
|
+
$state.take_snapshot
|
141
|
+
$state.system.cleanup_snapshots
|
142
|
+
end
|
143
|
+
}
|
144
|
+
|
@@ -0,0 +1,137 @@
|
|
1
|
+
require 'backup/actor'
|
2
|
+
require 'backup/rotator'
|
3
|
+
|
4
|
+
module Backup
|
5
|
+
# Represents a specific Backup configuration. A Configuration instance
|
6
|
+
# may be used to load multiple recipe files, define and describe tasks,
|
7
|
+
# define roles, create an actor, and set configuration variables.
|
8
|
+
class Configuration
|
9
|
+
# The actor created for this configuration instance.
|
10
|
+
attr_reader :actor
|
11
|
+
|
12
|
+
# The logger instance defined for this configuration.
|
13
|
+
attr_reader :logger
|
14
|
+
|
15
|
+
# The load paths used for locating recipe files.
|
16
|
+
attr_reader :load_paths
|
17
|
+
|
18
|
+
# The hash of variables currently known by the configuration
|
19
|
+
attr_reader :variables
|
20
|
+
|
21
|
+
def initialize(actor_class=Actor) #:nodoc:
|
22
|
+
@actor = actor_class.new(self)
|
23
|
+
#@logger = Logger.new
|
24
|
+
@load_paths = [".", File.join(File.dirname(__FILE__), "recipes")]
|
25
|
+
@variables = {}
|
26
|
+
end
|
27
|
+
|
28
|
+
# Set a variable to the given value.
|
29
|
+
def set(variable, value=nil, &block)
|
30
|
+
# if the variable is uppercase, then we add it as a constant to the
|
31
|
+
# actor. This is to allow uppercase "variables" to be set and referenced
|
32
|
+
# in recipes.
|
33
|
+
if variable.to_s[0].between?(?A, ?Z)
|
34
|
+
klass = @actor.metaclass
|
35
|
+
klass.send(:remove_const, variable) if klass.const_defined?(variable)
|
36
|
+
klass.const_set(variable, value)
|
37
|
+
end
|
38
|
+
|
39
|
+
value = block if value.nil? && block_given?
|
40
|
+
@variables[variable] = value
|
41
|
+
end
|
42
|
+
|
43
|
+
alias :[]= :set
|
44
|
+
|
45
|
+
def [](variable)
|
46
|
+
# TODO have it raise if it doesn exist
|
47
|
+
@variables[variable]
|
48
|
+
end
|
49
|
+
|
50
|
+
# Require another file. This is identical to the standard require method,
|
51
|
+
# with the exception that it sets the reciever as the "current" configuration
|
52
|
+
# so that third-party task bundles can include themselves relative to
|
53
|
+
# that configuration.
|
54
|
+
def require(*args) #:nodoc:
|
55
|
+
original, Backup.configuration = Backup.configuration, self
|
56
|
+
super
|
57
|
+
ensure
|
58
|
+
# restore the original, so that require's can be nested
|
59
|
+
Backup.configuration = original
|
60
|
+
end
|
61
|
+
|
62
|
+
# Disclaimer: This method written by Jamis Buck. Taken directly from his
|
63
|
+
# excellent code Capistrano.
|
64
|
+
#
|
65
|
+
# Load a configuration file or string into this configuration.
|
66
|
+
#
|
67
|
+
# Usage:
|
68
|
+
#
|
69
|
+
# load("recipe"):
|
70
|
+
# Look for and load the contents of 'recipe.rb' into this
|
71
|
+
# configuration.
|
72
|
+
#
|
73
|
+
# load(:file => "recipe"):
|
74
|
+
# same as above
|
75
|
+
#
|
76
|
+
# load(:string => "set :scm, :subversion"):
|
77
|
+
# Load the given string as a configuration specification.
|
78
|
+
#
|
79
|
+
# load { ... }
|
80
|
+
# Load the block in the context of the configuration.
|
81
|
+
def load(*args, &block)
|
82
|
+
options = args.last.is_a?(Hash) ? args.pop : {}
|
83
|
+
args.each { |arg| load options.merge(:file => arg) }
|
84
|
+
return unless args.empty?
|
85
|
+
|
86
|
+
if block
|
87
|
+
raise "loading a block requires 0 parameters" unless args.empty?
|
88
|
+
load(options.merge(:proc => block))
|
89
|
+
|
90
|
+
elsif options[:file]
|
91
|
+
file = options[:file]
|
92
|
+
unless file[0] == ?/
|
93
|
+
load_paths.each do |path|
|
94
|
+
if File.file?(File.join(path, file))
|
95
|
+
file = File.join(path, file)
|
96
|
+
break
|
97
|
+
elsif File.file?(File.join(path, file) + ".rb")
|
98
|
+
file = File.join(path, file + ".rb")
|
99
|
+
break
|
100
|
+
end
|
101
|
+
end
|
102
|
+
end
|
103
|
+
load :string => File.read(file), :name => options[:name] || file
|
104
|
+
|
105
|
+
elsif options[:string]
|
106
|
+
#logger.trace "loading configuration #{options[:name] || "<eval>"}"
|
107
|
+
instance_eval(options[:string], options[:name] || "<eval>")
|
108
|
+
|
109
|
+
elsif options[:proc]
|
110
|
+
#logger.trace "loading configuration #{options[:proc].inspect}"
|
111
|
+
instance_eval(&options[:proc])
|
112
|
+
|
113
|
+
else
|
114
|
+
raise ArgumentError, "don't know how to load #{options.inspect}"
|
115
|
+
end
|
116
|
+
end
|
117
|
+
|
118
|
+
# Describe the next task to be defined. The given text will be attached to
|
119
|
+
# the next task that is defined and used as its description.
|
120
|
+
def desc(text)
|
121
|
+
@next_description = text
|
122
|
+
end
|
123
|
+
|
124
|
+
# Define a new task. If a description is active (see #desc), it is added to
|
125
|
+
# the options under the <tt>:desc</tt> key. This method ultimately
|
126
|
+
# delegates to Actor#define_task.
|
127
|
+
def action(name, options={}, &block)
|
128
|
+
# raise ArgumentError, "expected a block or method" unless block or options[:method] # ??
|
129
|
+
if @next_description
|
130
|
+
options = options.merge(:desc => @next_description)
|
131
|
+
@next_description = nil
|
132
|
+
end
|
133
|
+
actor.define_action(name, options, &block)
|
134
|
+
end
|
135
|
+
|
136
|
+
end
|
137
|
+
end
|
@@ -0,0 +1,37 @@
|
|
1
|
+
module Backup
|
2
|
+
class DateParser
|
3
|
+
|
4
|
+
def self.date_from(what)
|
5
|
+
DateParser.new.date_from(what)
|
6
|
+
end
|
7
|
+
|
8
|
+
# the test is going to be whatever is returned here .include? the day of
|
9
|
+
# today. so if we want to do something every day than this needs to return
|
10
|
+
# something that will lincde the righ daY:W
|
11
|
+
def date_from(what)
|
12
|
+
if what.kind_of?(Symbol)
|
13
|
+
return Runt::DIWeek.new( Time.num_from_day(what) ) if day_of_week?(what)
|
14
|
+
return Runt::REDay.new(0,0,24,01) if what == :daily
|
15
|
+
if what.to_s =~ /^last_/
|
16
|
+
what.to_s =~ /^last_(\w+)_of_the_month$/
|
17
|
+
day = $1
|
18
|
+
return Runt::DIMonth.new(Runt::Last, Time.num_from_day(day.intern))
|
19
|
+
end
|
20
|
+
end
|
21
|
+
raise "#{what} is not a valid time" unless what.respond_to?(:include?)
|
22
|
+
what
|
23
|
+
end
|
24
|
+
|
25
|
+
private
|
26
|
+
def day_of_week?(word)
|
27
|
+
days_of_week.include?(word.to_s.downcase[0..2])
|
28
|
+
end
|
29
|
+
|
30
|
+
def days_of_week
|
31
|
+
%w{mon tue wed thu fri sat sun}
|
32
|
+
end
|
33
|
+
|
34
|
+
end
|
35
|
+
end
|
36
|
+
|
37
|
+
|
@@ -0,0 +1,113 @@
|
|
1
|
+
#------------------------------------------------------------------------------
|
2
|
+
# Backup Global Settings
|
3
|
+
# @author: Nate Murray <nate@natemurray.com>
|
4
|
+
# @date: Mon Aug 28 07:28:22 PDT 2006
|
5
|
+
#
|
6
|
+
# The settings contained in this file will be global for all tasks and can be
|
7
|
+
# overridden locally.
|
8
|
+
#------------------------------------------------------------------------------
|
9
|
+
# require 'tmpdir'
|
10
|
+
|
11
|
+
# Sepcify sever settings
|
12
|
+
set :servers, %w{ localhost }
|
13
|
+
set :action_order, %w{ content compress encrypt deliver rotate cleanup }
|
14
|
+
|
15
|
+
# Name of the SSH user
|
16
|
+
set :ssh_user, ENV['USER']
|
17
|
+
|
18
|
+
# default port
|
19
|
+
set :port, 22 # todo, change to ssh_port
|
20
|
+
|
21
|
+
# Path to your SSH key
|
22
|
+
set :identity_key, ENV['HOME'] + "/.ssh/id_rsa"
|
23
|
+
|
24
|
+
# Set global actions
|
25
|
+
action :compress, :method => :tar_bz2
|
26
|
+
action :deliver, :method => :mv # action :deliver, :method => :scp
|
27
|
+
action :rotate, :method => :via_mv # action :rotate, :method => :via_ssh
|
28
|
+
# action :encrypt, :method => :gpg
|
29
|
+
|
30
|
+
# Specify a directory that backup can use as a temporary directory
|
31
|
+
# set :tmp_dir, Dir.tmpdir
|
32
|
+
set :tmp_dir, "/tmp"
|
33
|
+
|
34
|
+
# Options to be passed to gpg when encrypting
|
35
|
+
set :encrypt, false
|
36
|
+
set :gpg_encrypt_options, ""
|
37
|
+
|
38
|
+
# These settings specify the rotation variables
|
39
|
+
# Rotation method. Currently the only method is gfs, grandfather-father-son.
|
40
|
+
# Read more about that below
|
41
|
+
set :rotation_method, :gfs
|
42
|
+
|
43
|
+
# rotation mode - temporal or numeric. For instance
|
44
|
+
# temporal mode would continue to be the default and work with
|
45
|
+
# :son_promoted_on. The promotions are based on days. This works well for 1 backup per day.
|
46
|
+
# numeric works by promoting after every number of creations. This is better for multiple backups per day.
|
47
|
+
# numeric mode uses :sons_promoted_after
|
48
|
+
set :rotation_mode, :temporal
|
49
|
+
|
50
|
+
# :mon-sun
|
51
|
+
# :last_day_of_the_month # whatever son_promoted on son was, but the last of the month
|
52
|
+
# everything else you can define with a Runt object
|
53
|
+
# set :son_created_on, :every_day - if you dont want a son created dont run the program
|
54
|
+
# a backup is created every time the program is run
|
55
|
+
|
56
|
+
set :son_promoted_on, :fri
|
57
|
+
set :father_promoted_on, :last_fri_of_the_month
|
58
|
+
|
59
|
+
# more complex
|
60
|
+
# mon_wed_fri = Runt::DIWeek.new(Runt::Mon) |
|
61
|
+
# Runt::DIWeek.new(Runt::Wed) |
|
62
|
+
# Runt::DIWeek.new(Runt::Fri)
|
63
|
+
# set :son_promoted_on, mon_wed_fri
|
64
|
+
|
65
|
+
set :sons_to_keep, 14
|
66
|
+
set :fathers_to_keep, 6
|
67
|
+
set :grandfathers_to_keep, 6 # 6 months, by default
|
68
|
+
|
69
|
+
# These options are only used if :rotation_mode is :numeric.
|
70
|
+
# This is better if you are doing multiple backups per day.
|
71
|
+
# This setting says that every 14th son will be promoted to a father.
|
72
|
+
set :sons_promoted_after, 14
|
73
|
+
set :fathers_promoted_after, 6
|
74
|
+
|
75
|
+
# -------------------------
|
76
|
+
# Standard Actions
|
77
|
+
# -------------------------
|
78
|
+
action(:tar_bz2) do
|
79
|
+
name = c[:tmp_dir] + "/" + File.basename(last_result) + ".tar.bz2"
|
80
|
+
v = "v" if verbose
|
81
|
+
sh "tar -c#{v}jf #{name} #{last_result}"
|
82
|
+
name
|
83
|
+
end
|
84
|
+
|
85
|
+
action(:scp) do
|
86
|
+
# what should the default scp task be?
|
87
|
+
# scp the local file to the foreign directory. same name.
|
88
|
+
c[:servers].each do |server|
|
89
|
+
host = server =~ /localhost/ ? "" : "#{server}:"
|
90
|
+
sh "scp #{last_result} #{c[:ssh_user]}@#{host}#{c[:backup_path]}/"
|
91
|
+
end
|
92
|
+
c[:backup_path] + "/" + File.basename(last_result)
|
93
|
+
end
|
94
|
+
|
95
|
+
action(:mv) do
|
96
|
+
move last_result, c[:backup_path] # has to be move (not mv) to avoid infinite
|
97
|
+
# recursion
|
98
|
+
c[:backup_path] + "/" + File.basename(last_result)
|
99
|
+
end
|
100
|
+
|
101
|
+
action(:s3) do
|
102
|
+
s3 = S3Actor.new(c)
|
103
|
+
s3.put last_result
|
104
|
+
end
|
105
|
+
|
106
|
+
action(:encrypt) do
|
107
|
+
result = last_result
|
108
|
+
if c[:encrypt]
|
109
|
+
sh "gpg #{c[:gpg_encrypt_options]} --encrypt #{last_result}"
|
110
|
+
result = last_result + ".gpg" # ?
|
111
|
+
end
|
112
|
+
result
|
113
|
+
end
|