backupgem 0.0.9 → 0.0.11

data/lib/backup/rotator.rb

@@ -0,0 +1,219 @@
+require "rake"
+
+module Backup
+  class Rotator
+    include FileUtils
+
+    attr_reader :actor
+    attr_reader :configuration
+    alias_method :c, :configuration
+
+    def initialize(actor)
+      @actor = actor
+      @configuration = actor.configuration
+    end
+
+    # Take the last result and rotate it via mv on the local machine
+    def rotate_via_mv(last_result)
+      # verify that each of the directories exist, grandfathers, fathers, sons
+      hierarchy.each { |m| verify_local_backup_directory_exists(m) }
+
+      where = place_in
+
+      # place todays backup into the specified directory with a timestamp. 
+      newname = timestamped_prefix(last_result)
+      mv last_result, "#{where}/#{newname}"
+
+      cleanup_via_mv(where, how_many_to_keep_today)
+      update_state
+    end
+
+    def update_state
+      return unless :numeric == c[:rotation_mode] 
+      where = find_goes_in_by_state
+
+      previous_sons     = $state.system.sons_since_last_promotion    || 0
+      previous_fathers  = $state.system.fathers_since_last_promotion || 0
+
+      case where
+        when "sons"
+         $state.system.sons_since_last_promotion = previous_sons + 1
+        when "fathers"
+         $state.system.sons_since_last_promotion = 0
+         $state.system.fathers_since_last_promotion = previous_fathers + 1
+        when "grandfathers"
+         $state.system.sons_since_last_promotion = 0
+         $state.system.fathers_since_last_promotion = 0
+      end
+    end
+    
+    def rotate_via_ssh(last_result)
+      ssh = Backup::SshActor.new(c) 
+      ssh.connect 
+      ssh.run "echo \"#{last_result}\""
+
+      hierarchy.each do |m| 
+        dir = c[:backup_path] + "/" + m
+        ssh.verify_directory_exists(dir) 
+      end  
+
+      where = place_in
+
+      newname = timestamped_prefix(last_result)
+      ssh.run "mv #{last_result} #{where}/#{newname}"
+
+      ssh.cleanup_directory(where, how_many_to_keep_today)
+      ssh.close
+    end
+
+    # TODO
+    def rotate_via_ftp(last_result)
+#      ftp = Backup::FtpActor.new(c) 
+#      ftp.connect 
+#
+#      hierarchy.each do |m| 
+#        dir = c[:backup_path] + "/" + m
+#        ftp.verify_directory_exists(dir) 
+#      end  
+#
+#      newname = timestamped_prefix(last_result)
+#      ftp.run "mv #{last_result} #{place_in}/#{newname}"
+#
+#      ftp.cleanup_directory(place_in, how_many_to_keep_today)
+#      ftp.close
+    end
+
+    def rotate_via_s3(last_result)
+      s3 = Backup::S3Actor.new(c)
+      s3.verify_rotation_hierarchy_exists(hierarchy)
+      index = s3.rotation
+      index[todays_generation] << last_result
+      s3.rotation = index
+      s3.cleanup(todays_generation, how_many_to_keep_today)
+    end
+
+    def create_sons_today?;     is_today_a? :son_created_on;     end
+    def promote_sons_today?;    is_today_a? :son_promoted_on;    end
+    def promote_fathers_today?; is_today_a? :father_promoted_on; end
+
+    # old ( offset_days % c[:son_promoted_on] ) == 0 ? true : false
+    # old ( offset_days % (c[:son_promoted_on] * c[:father_promoted_on]) ) == 0 ? true : false
+
+    private
+      def is_today_a?(symbol)
+        t = $test_time || Date.today
+        day = DateParser.date_from( c[symbol] )
+        day.include?( t )
+      end
+
+      def verify_local_backup_directory_exists(dir)
+        path = c[:backup_path]
+        full = path + "/" + dir
+        unless File.exists?(full) 
+          mkdir_p full  
+        end
+      end
+
+      #def offset_days
+      #  t = $test_time || Time.now # todo, write a test to use this
+      #  num_from_day = Time.num_from_day( c[:promote_on] )
+      #  offset = ( t.days_since_epoch + num_from_day + 0)
+      #  offset
+      #end
+
+      def cleanup_via_mv(where, num_keep)
+
+        files = Dir[where + "/*"].sort
+        diff  = files.size - num_keep
+
+        1.upto( diff ) do 
+          extra = files.shift
+          rm extra
+        end
+      end
+
+      def hierarchy 
+        %w{grandfathers fathers sons}
+      end
+
+      # figure out which generation today's backup is
+      public
+      def todays_generation
+        goes_in = promote_fathers_today? ? "grandfathers" :         \
+                  promote_sons_today?    ? "fathers"      : "sons"
+      end
+
+      def self.timestamped_prefix(name,time="%Y-%m-%d-%H-%M-%S")
+        time ||= "%Y-%m-%d-%H-%M-%S"  # there has to be a better way to do this
+                                      # but it works.
+        newname = Time.now.strftime(time) + "_" + File.basename(name)
+      end
+      
+      # Given +name+ returns a timestamped version of name. 
+      def timestamped_prefix(name)
+        Backup::Rotator.timestamped_prefix(name,c[:timestamp])
+      end
+
+      private
+
+      # Decide where to place the next backup. g,f,or s. See the standard
+      # config for a description of the difference between numeric and temporal
+      # rotation mode
+      def place_in
+        if :numeric == c[:rotation_mode]
+          goes_in = find_goes_in_by_state       
+        else
+          goes_in = todays_generation
+        end
+
+        place_in = c[:backup_path] + "/" + goes_in
+      end
+
+      def find_goes_in_by_state
+        previous_sons     = $state.system.sons_since_last_promotion    || 0
+        previous_fathers  = $state.system.fathers_since_last_promotion || 0
+
+        #puts "in find goes in"
+        #puts "sons since last: " + $state.system.sons_since_last_promotion.to_s
+        #puts "fathers since last: " + $state.system.fathers_since_last_promotion.to_s
+
+        if previous_sons >= c[:sons_promoted_after] 
+          if previous_fathers >= c[:fathers_promoted_after]
+            return "grandfathers"
+          end
+          return "fathers"
+        end
+        "sons"
+      end
+ 
+      # Returns the number of sons to keep. Looks for config values +:sons_to_keep+, 
+      # +:son_promoted_on+. Default +14+.
+      def sons_to_keep
+        c[:sons_to_keep]    || c[:son_promoted_on]    || 14
+      end
+      
+      # Returns the number of fathers to keep. Looks for config values +:fathers_to_keep+, 
+      # +:fathers_promoted_on+. Default +6+.
+      def fathers_to_keep
+        c[:fathers_to_keep] || c[:father_promoted_on] || 6
+      end
+
+      # Returns the number of grandfathers to keep. Looks for config values
+      # +:grandfathers_to_keep+. Default +6+.
+      def gfathers_to_keep
+        c[:grandfathers_to_keep] || 6
+      end
+
+      # This method returns the number of how many to keep in whatever today
+      # goes in. Example: if today is a day to create a +son+ then this
+      # function returns the value of +sons_to_keep+. If today is a +father+
+      # then +fathers_to_keep+ etc.
+      def how_many_to_keep_today
+        goes_in = todays_generation
+        keep = goes_in =~ /^sons$/         ? sons_to_keep     :
+               goes_in =~ /^fathers$/      ? fathers_to_keep  :
+               goes_in =~ /^grandfathers$/ ? gfathers_to_keep : 14
+      end
+
+   end
+end

data/lib/backup/s3_helpers.rb

@@ -0,0 +1,97 @@
+require 'yaml'
+
+module Backup
+  class S3Actor
+    include AWS::S3
+    
+    attr_accessor :rotation
+    
+    attr_reader :config
+    alias_method :c, :config
+
+    def initialize(config)
+      @config       = config
+      @rotation_key = c[:rotation_object_key] ||= 'backup_rotation_index.yml'
+      @access_key   = c[:aws_access] ||= ENV['AMAZON_ACCESS_KEY_ID']
+      @secret_key   = c[:aws_secret] ||= ENV['AMAZON_SECRET_ACCESS_KEY']
+      @bucket_key   = "#{@access_key}.#{c[:backup_path]}"
+      Base.establish_connection!(
+        :access_key_id     => @access_key,
+        :secret_access_key => @secret_key
+      )
+      begin
+        # Look for our bucket, if it's not there, try to create it.
+        @bucket = Bucket.find @bucket_key
+      rescue NoSuchBucket
+        @bucket = Bucket.create @bucket_key
+        @bucket = Bucket.find @bucket_key
+      end   
+    end
+
+    def rotation
+      object = S3Object.find(@rotation_key, @bucket.name)
+      index  = YAML::load(object.value)
+    end
+
+    def rotation=(index)
+      object = S3Object.store(@rotation_key, index.to_yaml, @bucket.name)
+      index
+    end
+
+    # Send a file to s3
+    def put(last_result)
+      puts last_result
+      object_key = Rotator.timestamped_prefix(last_result)
+      S3Object.store object_key,
+                     open(last_result),
+                     @bucket.name
+      object_key
+    end
+
+    # Remove a file from s3
+    def delete(object_key)
+      S3Object.delete object_key, @bucket.name
+    end
+
+    # Make sure our rotation index exists and contains the hierarchy we're using.
+    # Create it if it does not exist
+    def verify_rotation_hierarchy_exists(hierarchy)
+      begin
+        index = self.rotation
+        verified_index = index.merge(init_rotation_index(hierarchy)) { |m,x,y| x ||= y }
+        unless (verified_index == index)
+          self.rotation = verified_index
+        end
+      rescue NoSuchKey
+        self.rotation = init_rotation_index(hierarchy)
+      end
+    end
+
+    # Expire old objects
+    def cleanup(generation, keep)
+      puts "Cleaning up"
+      
+      keys = self.rotation[generation]
+      diff = keys.size - keep
+      
+      1.upto( diff ) do
+        extra_key = keys.shift
+        delete extra_key
+      end
+      # store updated index
+      self.rotation = keys
+    end
+
+    private
+    
+      # Create a new index representing our backup hierarchy
+      def init_rotation_index(hierarchy)
+        hash = {}
+        hierarchy.each do |m|
+          hash[m] = Array.new
+        end
+        hash
+      end
+    
+  end
+end

data/lib/backup/ssh_helpers.rb

@@ -0,0 +1,139 @@
+# large portions borrowed from capistrano
+require 'rubygems'
+require 'net/ssh'
+
+# TODO - add in a way to extend the belay script to work with this. thats a
+# good idea for the belay script over all. write the kernal extensions, and the
+# rake extensions. form there write an example third party extension.
+module Backup
+  class Command
+    attr_reader   :command, :options
+    attr_reader   :actor
+
+    def initialize(server_name, command, callback, options, actor) #:nodoc:
+      @command  = command.strip.gsub(/\r?\n/, "\\\n")
+      @callback = callback
+      @options  = options
+      @actor    = actor
+      @channels = open_channels
+    end
+
+    def process!
+      since = Time.now
+      loop do
+        active = 0
+        @channels.each do |ch|
+          next if ch[:closed]
+          active += 1
+          ch.connection.process(true)
+        end
+
+        break if active == 0
+        if Time.now - since >= 1
+          since = Time.now
+          @channels.each { |ch| ch.connection.ping! }
+        end
+        sleep 0.01 # a brief respite, to keep the CPU from going crazy
+      end
+
+      #logger.trace "command finished"
+
+      if failed = @channels.detect { |ch| ch[:status] != 0 }
+        raise "command #{@command.inspect} failed"
+      end
+
+      self
+    end
+     
+    def open_channels
+      channel = actor.session.open_channel do |channel|
+           channel.request_pty( :want_reply => true )
+           channel[:actor] = @actor
+
+           channel.on_success do |ch|
+             #logger.trace "executing command", ch[:host]
+             ch.exec command
+             ch.send_data options[:data] if options[:data]
+           end
+
+           channel.on_data do |ch, data|
+             puts data
+             @callback[ch, :out, data] if @callback
+           end
+
+           channel.on_failure do |ch|
+             #logger.important "could not open channel", ch[:host]
+             # puts "we got a faulure"
+             ch.close
+           end
+
+           channel.on_request do |ch, request, reply, data|
+             ch[:status] = data.read_long if request == "exit-status"
+           end
+
+           channel.on_close do |ch|
+             ch[:closed] = true
+           end
+      end
+      [channel]
+    end
+  end  # end Class Command
+
+  class SshActor
+     
+    #def self.new_for_ssh(server_name)
+    #  a = new(server_name)
+    #end 
+
+    attr_reader :session
+    attr_reader :config
+    alias_method :c, :config
+
+    def initialize(config)
+      @config = config
+    end
+
+    def connect
+      c[:servers].each do |server| # todo, make this actually work
+      @session = Net::SSH.start(
+           server,
+           :port         => c[:port],
+           :username     => c[:ssh_user],
+           :host_key     => "ssh-rsa",
+           :keys         => [ c[:identity_key] ],
+           :auth_methods => %w{ publickey } )
+      end
+    end
+
+    def run(cmd, options={}, &block)
+      #logger.debug "executing #{cmd.strip.inspect}"
+      puts "executing #{cmd.strip.inspect}"
+      command = Command.new(@server_name, cmd, block, options, self)
+      command.process! # raises an exception if command fails on any server
+    end
+
+    def on_remote(&block)
+      connect
+      self.instance_eval(&block)
+      close
+    end
+
+    def close
+      @session.close
+    end
+
+    def verify_directory_exists(dir)
+      run "if [ -d '#{dir}' ]; then true; else mkdir -p '#{dir}'; fi"
+    end
+
+    def cleanup_directory(dir, keep)
+      puts "Cleaning up" 
+      cleanup = <<-END
+      LOOK_IN="#{dir}"; COUNT=`ls -1 $LOOK_IN | wc -l`; MAX=#{keep}; if (( $COUNT > $MAX )); then let "OFFSET=$COUNT-$MAX"; i=1; for f in `ls -1 $LOOK_IN | sort`; do if (( $i <= $OFFSET )); then CMD="rm $LOOK_IN/$f"; echo $CMD; $CMD; fi; let "i=$i + 1"; done; else true; fi
+      END
+      run cleanup # todo make it so that even this can be overridden
+    end
+
+   end # end class SshActor
+
+end # end Module Backup