Backup_GEM_2 0.1.2

data/lib/backup/rotator.rb

@@ -0,0 +1,219 @@
+require "rake"
+
+module Backup
+  class Rotator
+    include FileUtils
+
+    attr_reader :actor
+    attr_reader :configuration
+    alias_method :c, :configuration
+
+    def initialize(actor)
+      @actor = actor
+      @configuration = actor.configuration
+    end
+
+    # Take the last result and rotate it via mv on the local machine
+    def rotate_via_mv(last_result)
+      # verify that each of the directories exist, grandfathers, fathers, sons
+      hierarchy.each { |m| verify_local_backup_directory_exists(m) }
+
+      where = place_in
+
+      # place todays backup into the specified directory with a timestamp. 
+      newname = timestamped_prefix(last_result)
+      mv last_result, "#{where}/#{newname}"
+
+      cleanup_via_mv(where, how_many_to_keep_today)
+      update_state
+    end
+
+    def update_state
+      return unless :numeric == c[:rotation_mode] 
+      where = find_goes_in_by_state
+
+      previous_sons     = $state.system.sons_since_last_promotion    || 0
+      previous_fathers  = $state.system.fathers_since_last_promotion || 0
+
+      case where
+        when "sons"
+         $state.system.sons_since_last_promotion = previous_sons + 1
+        when "fathers"
+         $state.system.sons_since_last_promotion = 0
+         $state.system.fathers_since_last_promotion = previous_fathers + 1
+        when "grandfathers"
+         $state.system.sons_since_last_promotion = 0
+         $state.system.fathers_since_last_promotion = 0
+      end
+    end
+    
+    def rotate_via_ssh(last_result)
+      ssh = Backup::SshActor.new(c) 
+      ssh.connect 
+      ssh.run "echo \"#{last_result}\""
+
+      hierarchy.each do |m| 
+        dir = c[:backup_path] + "/" + m
+        ssh.verify_directory_exists(dir) 
+      end  
+
+      where = place_in
+
+      newname = timestamped_prefix(last_result)
+      ssh.run "mv #{last_result} #{where}/#{newname}"
+
+      ssh.cleanup_directory(where, how_many_to_keep_today)
+      ssh.close
+    end
+
+    # TODO
+    def rotate_via_ftp(last_result)
+#      ftp = Backup::FtpActor.new(c) 
+#      ftp.connect 
+#
+#      hierarchy.each do |m| 
+#        dir = c[:backup_path] + "/" + m
+#        ftp.verify_directory_exists(dir) 
+#      end  
+#
+#      newname = timestamped_prefix(last_result)
+#      ftp.run "mv #{last_result} #{place_in}/#{newname}"
+#
+#      ftp.cleanup_directory(place_in, how_many_to_keep_today)
+#      ftp.close
+    end
+
+    def rotate_via_s3(last_result)
+      s3 = Backup::ChunkingS3Actor.new(c)
+      s3.verify_rotation_hierarchy_exists(hierarchy)
+      index = s3.rotation
+      index[todays_generation] << last_result
+      s3.rotation = index
+      s3.cleanup(todays_generation, how_many_to_keep_today)
+    end
+
+    def create_sons_today?;     is_today_a? :son_created_on;     end
+    def promote_sons_today?;    is_today_a? :son_promoted_on;    end
+    def promote_fathers_today?; is_today_a? :father_promoted_on; end
+
+    # old ( offset_days % c[:son_promoted_on] ) == 0 ? true : false
+    # old ( offset_days % (c[:son_promoted_on] * c[:father_promoted_on]) ) == 0 ? true : false
+
+    private
+      def is_today_a?(symbol)
+        t = $test_time || Date.today
+        day = DateParser.date_from( c[symbol] )
+        day.include?( t )
+      end
+
+      def verify_local_backup_directory_exists(dir)
+        path = c[:backup_path]
+        full = path + "/" + dir
+        unless File.exists?(full) 
+          mkdir_p full  
+        end
+      end
+
+      #def offset_days
+      #  t = $test_time || Time.now # todo, write a test to use this
+      #  num_from_day = Time.num_from_day( c[:promote_on] )
+      #  offset = ( t.days_since_epoch + num_from_day + 0)
+      #  offset
+      #end
+
+      def cleanup_via_mv(where, num_keep)
+
+        files = Dir[where + "/*"].sort
+        diff  = files.size - num_keep
+
+        1.upto( diff ) do 
+          extra = files.shift
+          rm extra
+        end
+      end
+
+      def hierarchy 
+        %w{grandfathers fathers sons}
+      end
+
+      # figure out which generation today's backup is
+      public
+      def todays_generation
+        goes_in = promote_fathers_today? ? "grandfathers" :         \
+                  promote_sons_today?    ? "fathers"      : "sons"
+      end
+
+      def self.timestamped_prefix(name,time="%Y-%m-%d-%H-%M-%S")
+        time ||= "%Y-%m-%d-%H-%M-%S"  # there has to be a better way to do this
+                                      # but it works.
+        newname = Time.now.strftime(time) + "_" + File.basename(name)
+      end
+      
+      # Given +name+ returns a timestamped version of name. 
+      def timestamped_prefix(name)
+        Backup::Rotator.timestamped_prefix(name,c[:timestamp])
+      end
+
+      private
+
+      # Decide where to place the next backup. g,f,or s. See the standard
+      # config for a description of the difference between numeric and temporal
+      # rotation mode
+      def place_in
+        if :numeric == c[:rotation_mode]
+          goes_in = find_goes_in_by_state       
+        else
+          goes_in = todays_generation
+        end
+
+        place_in = c[:backup_path] + "/" + goes_in
+      end
+
+      def find_goes_in_by_state
+        previous_sons     = $state.system.sons_since_last_promotion    || 0
+        previous_fathers  = $state.system.fathers_since_last_promotion || 0
+
+        #puts "in find goes in"
+        #puts "sons since last: " + $state.system.sons_since_last_promotion.to_s
+        #puts "fathers since last: " + $state.system.fathers_since_last_promotion.to_s
+
+        if previous_sons >= c[:sons_promoted_after] 
+          if previous_fathers >= c[:fathers_promoted_after]
+            return "grandfathers"
+          end
+          return "fathers"
+        end
+        "sons"
+      end
+ 
+      # Returns the number of sons to keep. Looks for config values +:sons_to_keep+, 
+      # +:son_promoted_on+. Default +14+.
+      def sons_to_keep
+        c[:sons_to_keep]    || c[:son_promoted_on]    || 14
+      end
+      
+      # Returns the number of fathers to keep. Looks for config values +:fathers_to_keep+, 
+      # +:fathers_promoted_on+. Default +6+.
+      def fathers_to_keep
+        c[:fathers_to_keep] || c[:father_promoted_on] || 6
+      end
+
+      # Returns the number of grandfathers to keep. Looks for config values
+      # +:grandfathers_to_keep+. Default +6+.
+      def gfathers_to_keep
+        c[:grandfathers_to_keep] || 6
+      end
+
+      # This method returns the number of how many to keep in whatever today
+      # goes in. Example: if today is a day to create a +son+ then this
+      # function returns the value of +sons_to_keep+. If today is a +father+
+      # then +fathers_to_keep+ etc.
+      def how_many_to_keep_today
+        goes_in = todays_generation
+        keep = goes_in =~ /^sons$/         ? sons_to_keep     :
+               goes_in =~ /^fathers$/      ? fathers_to_keep  :
+               goes_in =~ /^grandfathers$/ ? gfathers_to_keep : 14
+      end
+
+   end
+end

data/lib/backup/s3_helpers.rb

@@ -0,0 +1,140 @@
+require 'yaml'
+
+module Backup
+  class S3Actor
+    
+    attr_accessor :rotation
+    
+    attr_reader :config
+    alias_method :c, :config
+
+    def initialize(config)
+      @config       = config
+      @rotation_key = c[:rotation_object_key] ||= 'backup_rotation_index.yml'
+      @access_key   = c[:aws_access] ||= ENV['AMAZON_ACCESS_KEY_ID']
+      @secret_key   = c[:aws_secret] ||= ENV['AMAZON_SECRET_ACCESS_KEY']
+      @bucket_key   = "#{@access_key}.#{c[:backup_path]}"
+      @s3 = RightAws::S3.new(@access_key, @secret_key)
+      @bucket = @s3.bucket(@bucket_key, true)
+    end
+
+    def rotation
+      key = @bucket.key(@rotation_key)
+      YAML::load(key.data) if key
+    end
+
+    def rotation=(index)
+      @bucket.put(@rotation_key, index.to_yaml)
+      index
+    end
+
+    # Send a file to s3
+    def put(last_result)
+      object_key = Rotator.timestamped_prefix(last_result)
+      puts "put: #{object_key}"
+      @bucket.put(object_key, open(last_result))
+      object_key
+    end
+
+    # Remove a file from s3
+    def delete(object_key)
+      puts "delete: #{object_key}"
+      @bucket.key(object_key).delete
+    end
+
+    # Make sure our rotation index exists and contains the hierarchy we're using.
+    # Create it if it does not exist
+    def verify_rotation_hierarchy_exists(hierarchy)
+      index = self.rotation
+      if index
+        verified_index = index.merge(init_rotation_index(hierarchy)) { |m,x,y| x ||= y }
+        unless (verified_index == index)
+          self.rotation = verified_index
+        end
+      else
+        self.rotation = init_rotation_index(hierarchy)
+      end
+    end
+
+    # Expire old objects
+    def cleanup(generation, keep)
+      puts "Cleaning up #{generation} #{keep}"
+      
+      new_rotation = self.rotation
+      keys = new_rotation[generation]
+            
+      diff = keys.size - keep
+      
+      1.upto( diff ) do
+        extra_key = keys.shift
+        delete extra_key
+      end
+            
+      # store updated index
+      self.rotation = new_rotation
+    end
+
+    private
+    
+      # Create a new index representing our backup hierarchy
+      def init_rotation_index(hierarchy)
+        hash = {}
+        hierarchy.each do |m|
+          hash[m] = Array.new
+        end
+        hash
+      end
+    
+  end
+
+  class ChunkingS3Actor < S3Actor
+    DEFAULT_MAX_OBJECT_SIZE = 5368709120 # 5 * 2^30 = 5GB
+    DEFAULT_CHUNK_SIZE = 4294967296 # 4 * 2^30 = 4GB
+
+    def initialize(config)
+      super
+      @max_object_size = c[:max_object_size] ||= DEFAULT_MAX_OBJECT_SIZE
+      @chunk_size = c[:chunk_size] ||= DEFAULT_CHUNK_SIZE
+    end
+
+    # Send a file to s3
+    def put(last_result)
+      object_key = Rotator.timestamped_prefix(last_result)
+      puts "put: #{object_key}"
+      # determine if the file is too large
+      if File.stat(last_result).size > @max_object_size
+        # if so, split
+        split_command = "cd #{File.dirname(last_result)} && split -d -b #{@chunk_size} #{File.basename(last_result)} #{File.basename(last_result)}."
+        puts "split: #{split_command}"
+        system split_command
+        chunks = Dir.glob("#{last_result}.*")
+        # put each file in the split
+        chunks.each do |chunk|
+          chunk_index = chunk.sub(last_result,"")
+          chunk_key = "#{object_key}#{chunk_index}"
+          puts "  #{chunk_key}"
+          @bucket.put(chunk_key, open(chunk))
+        end
+      else
+        @bucket.put(object_key, open(last_result))
+      end
+      object_key
+    end
+
+    # Remove a file from s3
+    def delete(object_key)
+      puts "delete: #{object_key}"
+      # determine if there are multiple objects with this key prefix
+      chunks = @bucket.keys(:prefix => object_key)
+      if chunks.size > 1
+        # delete them all
+        chunks.each do |chunk|
+          puts "  #{chunk.name}"
+          chunk.delete
+        end
+      else
+        chunks.first.delete
+      end
+    end
+  end
+end

data/lib/backup/ssh_helpers.rb

@@ -0,0 +1,135 @@
+# large portions borrowed from capistrano
+require 'rubygems'
+require 'net/ssh'
+
+# TODO - add in a way to extend the belay script to work with this. thats a
+# good idea for the belay script over all. write the kernal extensions, and the
+# rake extensions. form there write an example third party extension.
+module Backup
+  class Command
+    attr_reader   :command, :options
+    attr_reader   :actor
+
+    def initialize(server_name, command, callback, options, actor) #:nodoc:
+      @command  = command.strip.gsub(/\r?\n/, "\\\n")
+      @callback = callback
+      @options  = options
+      @actor    = actor
+      @channels = open_channels
+    end
+
+    def process!
+      since = Time.now
+      loop do
+        active = 0
+        @channels.each do |ch|
+          next if ch[:closed]
+          active += 1
+          ch.connection.process(0)
+        end
+
+        break if active == 0
+        if Time.now - since >= 1
+          since = Time.now
+          @channels.each { |ch| ch.connection.ping! }
+        end
+        sleep 0.01 # a brief respite, to keep the CPU from going crazy
+      end
+
+      #logger.trace "command finished"
+
+      if failed = @channels.detect { |ch| ch[:status] != 0 }
+        raise "command #{@command.inspect} failed"
+      end
+
+      self
+    end
+     
+    def open_channels
+      channel = actor.session.open_channel do |channel|
+           channel.request_pty do |ch, success|
+             if success
+               ch.exec command
+               ch.send_data options[:data] if options[:data]
+             else
+               ch.close
+             end
+           end
+
+           channel[:actor] = @actor
+
+           channel.on_data do |ch, data|
+             puts data
+             @callback[ch, :out, data] if @callback
+           end
+
+           channel.on_request "exit-status" do |ch, data|
+             ch[:status] = data.read_long
+           end
+
+           channel.on_close do |ch|
+             ch[:closed] = true
+           end
+      end
+      [channel]
+    end
+  end  # end Class Command
+
+  class SshActor
+     
+    #def self.new_for_ssh(server_name)
+    #  a = new(server_name)
+    #end 
+
+    attr_reader :session
+    attr_reader :config
+    alias_method :c, :config
+
+    def initialize(config)
+      @config = config
+    end
+
+    def connect
+      c[:servers].each do |server| # todo, make this actually work
+      @session = Net::SSH.start(
+           server,
+           c[:ssh_user],
+           :port         => c[:port],
+           :host_key     => "ssh-rsa",
+           :keys         => [ c[:identity_key] ],
+           :auth_methods => %w{ publickey } )
+      end
+    end
+
+    def run(cmd, options={}, &block)
+      #logger.debug "executing #{cmd.strip.inspect}"
+      puts "executing #{cmd.strip.inspect}"
+      command = Command.new(@server_name, cmd, block, options, self)
+      command.process! # raises an exception if command fails on any server
+    end
+
+    def on_remote(&block)
+      connect
+      self.instance_eval(&block)
+      close
+    end
+
+    def close
+      @session.close
+    end
+
+    def verify_directory_exists(dir)
+      run "if [ -d '#{dir}' ]; then true; else mkdir -p '#{dir}'; fi"
+    end
+
+    def cleanup_directory(dir, keep)
+      puts "Cleaning up" 
+      cleanup = <<-END
+      LOOK_IN="#{dir}"; COUNT=`ls -1 $LOOK_IN | wc -l`; MAX=#{keep}; if (( $COUNT > $MAX )); then let "OFFSET=$COUNT-$MAX"; i=1; for f in `ls -1 $LOOK_IN | sort`; do if (( $i <= $OFFSET )); then CMD="rm $LOOK_IN/$f"; echo $CMD; $CMD; fi; let "i=$i + 1"; done; else true; fi
+      END
+      run cleanup # todo make it so that even this can be overridden
+    end
+
+   end # end class SshActor
+
+end # end Module Backup