backupgemsteven 0.1.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,219 @@
1
+ require "rake"
2
+
3
+ module Backup
4
+ class Rotator
5
+ include FileUtils
6
+
7
+ attr_reader :actor
8
+ attr_reader :configuration
9
+ alias_method :c, :configuration
10
+
11
+ def initialize(actor)
12
+ @actor = actor
13
+ @configuration = actor.configuration
14
+ end
15
+
16
+ # Take the last result and rotate it via mv on the local machine
17
+ def rotate_via_mv(last_result)
18
+ # verify that each of the directories exist, grandfathers, fathers, sons
19
+ hierarchy.each { |m| verify_local_backup_directory_exists(m) }
20
+
21
+ where = place_in
22
+
23
+ # place todays backup into the specified directory with a timestamp.
24
+ newname = timestamped_prefix(last_result)
25
+ mv last_result, "#{where}/#{newname}"
26
+
27
+ cleanup_via_mv(where, how_many_to_keep_today)
28
+ update_state
29
+ end
30
+
31
+ def update_state
32
+ return unless :numeric == c[:rotation_mode]
33
+ where = find_goes_in_by_state
34
+
35
+ previous_sons = $state.system.sons_since_last_promotion || 0
36
+ previous_fathers = $state.system.fathers_since_last_promotion || 0
37
+
38
+ case where
39
+ when "sons"
40
+ $state.system.sons_since_last_promotion = previous_sons + 1
41
+ when "fathers"
42
+ $state.system.sons_since_last_promotion = 0
43
+ $state.system.fathers_since_last_promotion = previous_fathers + 1
44
+ when "grandfathers"
45
+ $state.system.sons_since_last_promotion = 0
46
+ $state.system.fathers_since_last_promotion = 0
47
+ end
48
+ end
49
+
50
+ def rotate_via_ssh(last_result)
51
+ ssh = Backup::SshActor.new(c)
52
+ ssh.connect
53
+ ssh.run "echo \"#{last_result}\""
54
+
55
+ hierarchy.each do |m|
56
+ dir = c[:backup_path] + "/" + m
57
+ ssh.verify_directory_exists(dir)
58
+ end
59
+
60
+ where = place_in
61
+
62
+ newname = timestamped_prefix(last_result)
63
+ ssh.run "mv #{last_result} #{where}/#{newname}"
64
+
65
+ ssh.cleanup_directory(where, how_many_to_keep_today)
66
+ ssh.close
67
+ end
68
+
69
+ # TODO
70
+ def rotate_via_ftp(last_result)
71
+ # ftp = Backup::FtpActor.new(c)
72
+ # ftp.connect
73
+ #
74
+ # hierarchy.each do |m|
75
+ # dir = c[:backup_path] + "/" + m
76
+ # ftp.verify_directory_exists(dir)
77
+ # end
78
+ #
79
+ # newname = timestamped_prefix(last_result)
80
+ # ftp.run "mv #{last_result} #{place_in}/#{newname}"
81
+ #
82
+ # ftp.cleanup_directory(place_in, how_many_to_keep_today)
83
+ # ftp.close
84
+ end
85
+
86
+ def rotate_via_s3(last_result)
87
+ s3 = Backup::ChunkingS3Actor.new(c)
88
+ s3.verify_rotation_hierarchy_exists(hierarchy)
89
+ index = s3.rotation
90
+ index[todays_generation] << last_result
91
+ s3.rotation = index
92
+ s3.cleanup(todays_generation, how_many_to_keep_today)
93
+ end
94
+
95
+ def create_sons_today?; is_today_a? :son_created_on; end
96
+ def promote_sons_today?; is_today_a? :son_promoted_on; end
97
+ def promote_fathers_today?; is_today_a? :father_promoted_on; end
98
+
99
+ # old ( offset_days % c[:son_promoted_on] ) == 0 ? true : false
100
+ # old ( offset_days % (c[:son_promoted_on] * c[:father_promoted_on]) ) == 0 ? true : false
101
+
102
+ private
103
+ def is_today_a?(symbol)
104
+ t = $test_time || Date.today
105
+ day = DateParser.date_from( c[symbol] )
106
+ day.include?( t )
107
+ end
108
+
109
+ def verify_local_backup_directory_exists(dir)
110
+ path = c[:backup_path]
111
+ full = path + "/" + dir
112
+ unless File.exists?(full)
113
+ mkdir_p full
114
+ end
115
+ end
116
+
117
+ #def offset_days
118
+ # t = $test_time || Time.now # todo, write a test to use this
119
+ # num_from_day = Time.num_from_day( c[:promote_on] )
120
+ # offset = ( t.days_since_epoch + num_from_day + 0)
121
+ # offset
122
+ #end
123
+
124
+ def cleanup_via_mv(where, num_keep)
125
+
126
+ files = Dir[where + "/*"].sort
127
+ diff = files.size - num_keep
128
+
129
+ 1.upto( diff ) do
130
+ extra = files.shift
131
+ rm extra
132
+ end
133
+ end
134
+
135
+ def hierarchy
136
+ %w{grandfathers fathers sons}
137
+ end
138
+
139
+ # figure out which generation today's backup is
140
+ public
141
+ def todays_generation
142
+ goes_in = promote_fathers_today? ? "grandfathers" : \
143
+ promote_sons_today? ? "fathers" : "sons"
144
+ end
145
+
146
+ def self.timestamped_prefix(name,time="%Y-%m-%d-%H-%M-%S")
147
+ time ||= "%Y-%m-%d-%H-%M-%S" # there has to be a better way to do this
148
+ # but it works.
149
+ newname = Time.now.strftime(time) + "_" + File.basename(name)
150
+ end
151
+
152
+ # Given +name+ returns a timestamped version of name.
153
+ def timestamped_prefix(name)
154
+ Backup::Rotator.timestamped_prefix(name,c[:timestamp])
155
+ end
156
+
157
+ private
158
+
159
+ # Decide where to place the next backup. g,f,or s. See the standard
160
+ # config for a description of the difference between numeric and temporal
161
+ # rotation mode
162
+ def place_in
163
+ if :numeric == c[:rotation_mode]
164
+ goes_in = find_goes_in_by_state
165
+ else
166
+ goes_in = todays_generation
167
+ end
168
+
169
+ place_in = c[:backup_path] + "/" + goes_in
170
+ end
171
+
172
+ def find_goes_in_by_state
173
+ previous_sons = $state.system.sons_since_last_promotion || 0
174
+ previous_fathers = $state.system.fathers_since_last_promotion || 0
175
+
176
+ #puts "in find goes in"
177
+ #puts "sons since last: " + $state.system.sons_since_last_promotion.to_s
178
+ #puts "fathers since last: " + $state.system.fathers_since_last_promotion.to_s
179
+
180
+ if previous_sons >= c[:sons_promoted_after]
181
+ if previous_fathers >= c[:fathers_promoted_after]
182
+ return "grandfathers"
183
+ end
184
+ return "fathers"
185
+ end
186
+ "sons"
187
+ end
188
+
189
+ # Returns the number of sons to keep. Looks for config values +:sons_to_keep+,
190
+ # +:son_promoted_on+. Default +14+.
191
+ def sons_to_keep
192
+ c[:sons_to_keep] || c[:son_promoted_on] || 14
193
+ end
194
+
195
+ # Returns the number of fathers to keep. Looks for config values +:fathers_to_keep+,
196
+ # +:fathers_promoted_on+. Default +6+.
197
+ def fathers_to_keep
198
+ c[:fathers_to_keep] || c[:father_promoted_on] || 6
199
+ end
200
+
201
+ # Returns the number of grandfathers to keep. Looks for config values
202
+ # +:grandfathers_to_keep+. Default +6+.
203
+ def gfathers_to_keep
204
+ c[:grandfathers_to_keep] || 6
205
+ end
206
+
207
+ # This method returns the number of how many to keep in whatever today
208
+ # goes in. Example: if today is a day to create a +son+ then this
209
+ # function returns the value of +sons_to_keep+. If today is a +father+
210
+ # then +fathers_to_keep+ etc.
211
+ def how_many_to_keep_today
212
+ goes_in = todays_generation
213
+ keep = goes_in =~ /^sons$/ ? sons_to_keep :
214
+ goes_in =~ /^fathers$/ ? fathers_to_keep :
215
+ goes_in =~ /^grandfathers$/ ? gfathers_to_keep : 14
216
+ end
217
+
218
+ end
219
+ end
@@ -0,0 +1,140 @@
1
+ require 'yaml'
2
+
3
+ module Backup
4
+ class S3Actor
5
+
6
+ attr_accessor :rotation
7
+
8
+ attr_reader :config
9
+ alias_method :c, :config
10
+
11
+ def initialize(config)
12
+ @config = config
13
+ @rotation_key = c[:rotation_object_key] ||= 'backup_rotation_index.yml'
14
+ @access_key = c[:aws_access] ||= ENV['AMAZON_ACCESS_KEY_ID']
15
+ @secret_key = c[:aws_secret] ||= ENV['AMAZON_SECRET_ACCESS_KEY']
16
+ @bucket_key = "#{@access_key}.#{c[:backup_path]}"
17
+ @s3 = RightAws::S3.new(@access_key, @secret_key)
18
+ @bucket = @s3.bucket(@bucket_key, true)
19
+ end
20
+
21
+ def rotation
22
+ key = @bucket.key(@rotation_key)
23
+ YAML::load(key.data) if key
24
+ end
25
+
26
+ def rotation=(index)
27
+ @bucket.put(@rotation_key, index.to_yaml)
28
+ index
29
+ end
30
+
31
+ # Send a file to s3
32
+ def put(last_result)
33
+ object_key = Rotator.timestamped_prefix(last_result)
34
+ puts "put: #{object_key}"
35
+ @bucket.put(object_key, open(last_result))
36
+ object_key
37
+ end
38
+
39
+ # Remove a file from s3
40
+ def delete(object_key)
41
+ puts "delete: #{object_key}"
42
+ @bucket.key(object_key).delete
43
+ end
44
+
45
+ # Make sure our rotation index exists and contains the hierarchy we're using.
46
+ # Create it if it does not exist
47
+ def verify_rotation_hierarchy_exists(hierarchy)
48
+ index = self.rotation
49
+ if index
50
+ verified_index = index.merge(init_rotation_index(hierarchy)) { |m,x,y| x ||= y }
51
+ unless (verified_index == index)
52
+ self.rotation = verified_index
53
+ end
54
+ else
55
+ self.rotation = init_rotation_index(hierarchy)
56
+ end
57
+ end
58
+
59
+ # Expire old objects
60
+ def cleanup(generation, keep)
61
+ puts "Cleaning up #{generation} #{keep}"
62
+
63
+ new_rotation = self.rotation
64
+ keys = new_rotation[generation]
65
+
66
+ diff = keys.size - keep
67
+
68
+ 1.upto( diff ) do
69
+ extra_key = keys.shift
70
+ delete extra_key
71
+ end
72
+
73
+ # store updated index
74
+ self.rotation = new_rotation
75
+ end
76
+
77
+ private
78
+
79
+ # Create a new index representing our backup hierarchy
80
+ def init_rotation_index(hierarchy)
81
+ hash = {}
82
+ hierarchy.each do |m|
83
+ hash[m] = Array.new
84
+ end
85
+ hash
86
+ end
87
+
88
+ end
89
+
90
+ class ChunkingS3Actor < S3Actor
91
+ DEFAULT_MAX_OBJECT_SIZE = 5368709120 # 5 * 2^30 = 5GB
92
+ DEFAULT_CHUNK_SIZE = 4294967296 # 4 * 2^30 = 4GB
93
+
94
+ def initialize(config)
95
+ super
96
+ @max_object_size = c[:max_object_size] ||= DEFAULT_MAX_OBJECT_SIZE
97
+ @chunk_size = c[:chunk_size] ||= DEFAULT_CHUNK_SIZE
98
+ end
99
+
100
+ # Send a file to s3
101
+ def put(last_result)
102
+ object_key = Rotator.timestamped_prefix(last_result)
103
+ puts "put: #{object_key}"
104
+ # determine if the file is too large
105
+ if File.stat(last_result).size > @max_object_size
106
+ # if so, split
107
+ split_command = "cd #{File.dirname(last_result)} && split -d -b #{@chunk_size} #{File.basename(last_result)} #{File.basename(last_result)}."
108
+ puts "split: #{split_command}"
109
+ system split_command
110
+ chunks = Dir.glob("#{last_result}.*")
111
+ # put each file in the split
112
+ chunks.each do |chunk|
113
+ chunk_index = chunk.sub(last_result,"")
114
+ chunk_key = "#{object_key}#{chunk_index}"
115
+ puts " #{chunk_key}"
116
+ @bucket.put(chunk_key, open(chunk))
117
+ end
118
+ else
119
+ @bucket.put(object_key, open(last_result))
120
+ end
121
+ object_key
122
+ end
123
+
124
+ # Remove a file from s3
125
+ def delete(object_key)
126
+ puts "delete: #{object_key}"
127
+ # determine if there are multiple objects with this key prefix
128
+ chunks = @bucket.keys(:prefix => object_key)
129
+ if chunks.size > 1
130
+ # delete them all
131
+ chunks.each do |chunk|
132
+ puts " #{chunk.name}"
133
+ chunk.delete
134
+ end
135
+ else
136
+ chunks.first.delete
137
+ end
138
+ end
139
+ end
140
+ end
@@ -0,0 +1,135 @@
1
+ # large portions borrowed from capistrano
2
+ require 'rubygems'
3
+ require 'net/ssh'
4
+
5
+ # TODO - add in a way to extend the belay script to work with this. thats a
6
+ # good idea for the belay script over all. write the kernal extensions, and the
7
+ # rake extensions. form there write an example third party extension.
8
+ module Backup
9
+ class Command
10
+ attr_reader :command, :options
11
+ attr_reader :actor
12
+
13
+ def initialize(server_name, command, callback, options, actor) #:nodoc:
14
+ @command = command.strip.gsub(/\r?\n/, "\\\n")
15
+ @callback = callback
16
+ @options = options
17
+ @actor = actor
18
+ @channels = open_channels
19
+ end
20
+
21
+ def process!
22
+ since = Time.now
23
+ loop do
24
+ active = 0
25
+ @channels.each do |ch|
26
+ next if ch[:closed]
27
+ active += 1
28
+ ch.connection.process(0)
29
+ end
30
+
31
+ break if active == 0
32
+ if Time.now - since >= 1
33
+ since = Time.now
34
+ @channels.each { |ch| ch.connection.ping! }
35
+ end
36
+ sleep 0.01 # a brief respite, to keep the CPU from going crazy
37
+ end
38
+
39
+ #logger.trace "command finished"
40
+
41
+ if failed = @channels.detect { |ch| ch[:status] != 0 }
42
+ raise "command #{@command.inspect} failed"
43
+ end
44
+
45
+ self
46
+ end
47
+
48
+ def open_channels
49
+ channel = actor.session.open_channel do |channel|
50
+ channel.request_pty do |ch, success|
51
+ if success
52
+ ch.exec command
53
+ ch.send_data options[:data] if options[:data]
54
+ else
55
+ ch.close
56
+ end
57
+ end
58
+
59
+ channel[:actor] = @actor
60
+
61
+ channel.on_data do |ch, data|
62
+ puts data
63
+ @callback[ch, :out, data] if @callback
64
+ end
65
+
66
+ channel.on_request "exit-status" do |ch, data|
67
+ ch[:status] = data.read_long
68
+ end
69
+
70
+ channel.on_close do |ch|
71
+ ch[:closed] = true
72
+ end
73
+ end
74
+ [channel]
75
+ end
76
+ end # end Class Command
77
+
78
+ class SshActor
79
+
80
+ #def self.new_for_ssh(server_name)
81
+ # a = new(server_name)
82
+ #end
83
+
84
+ attr_reader :session
85
+ attr_reader :config
86
+ alias_method :c, :config
87
+
88
+ def initialize(config)
89
+ @config = config
90
+ end
91
+
92
+ def connect
93
+ c[:servers].each do |server| # todo, make this actually work
94
+ @session = Net::SSH.start(
95
+ server,
96
+ c[:ssh_user],
97
+ :port => c[:port],
98
+ :host_key => "ssh-rsa",
99
+ :keys => [ c[:identity_key] ],
100
+ :auth_methods => %w{ publickey } )
101
+ end
102
+ end
103
+
104
+ def run(cmd, options={}, &block)
105
+ #logger.debug "executing #{cmd.strip.inspect}"
106
+ puts "executing #{cmd.strip.inspect}"
107
+ command = Command.new(@server_name, cmd, block, options, self)
108
+ command.process! # raises an exception if command fails on any server
109
+ end
110
+
111
+ def on_remote(&block)
112
+ connect
113
+ self.instance_eval(&block)
114
+ close
115
+ end
116
+
117
+ def close
118
+ @session.close
119
+ end
120
+
121
+ def verify_directory_exists(dir)
122
+ run "if [ -d '#{dir}' ]; then true; else mkdir -p '#{dir}'; fi"
123
+ end
124
+
125
+ def cleanup_directory(dir, keep)
126
+ puts "Cleaning up"
127
+ cleanup = <<-END
128
+ LOOK_IN="#{dir}"; COUNT=`ls -1 $LOOK_IN | wc -l`; MAX=#{keep}; if (( $COUNT > $MAX )); then let "OFFSET=$COUNT-$MAX"; i=1; for f in `ls -1 $LOOK_IN | sort`; do if (( $i <= $OFFSET )); then CMD="rm $LOOK_IN/$f"; echo $CMD; $CMD; fi; let "i=$i + 1"; done; else true; fi
129
+ END
130
+ run cleanup # todo make it so that even this can be overridden
131
+ end
132
+
133
+ end # end class SshActor
134
+
135
+ end # end Module Backup