backupgem 0.0.2

data/lib/backup/recipes/standard.rb

@@ -0,0 +1,89 @@
+#------------------------------------------------------------------------------
+# Backup Global Settings
+# @author: Nate Murray <nate@natemurray.com>
+#   @date: Mon Aug 28 07:28:22 PDT 2006
+# 
+# The settings contained in this file will be global for all tasks and can be
+# overridden locally.
+#------------------------------------------------------------------------------
+
+# Sepcify sever settings
+set :servers,           %w{ localhost }
+set :action_order,      %w{ content compress encrypt deliver rotate cleanup }
+
+# Name of the SSH user
+set :ssh_user,          ENV['USER']
+
+# Path to your SSH key
+set :identity_key,      ENV['HOME'] + "/.ssh/id_rsa"
+
+# Set global actions
+action :compress, :method => :tar_bz2 
+action :deliver,  :method => :mv      # action :deliver,  :method => :scp    
+action :rotate,   :method => :via_mv  # action :rotate,   :method => :via_ssh
+# action :encrypt,  :method => :gpg
+
+# Specify a directory that backup can use as a temporary directory
+set :tmp_dir, "/tmp"
+
+# Options to be passed to gpg when encrypting
+set :encrypt, false
+set :gpg_encrypt_options, ""
+
+# These settings specify the rotation variables
+# Rotation method. Currently the only method is gfs, grandfather-father-son. 
+# Read more about that below
+set :rotation_method,  :gfs
+
+# :mon-sun
+# :last_day_of_the_month # whatever son_promoted on son was, but the last of the month
+# everything else you can define with a Runt object
+# set :son_created_on,     :every_day - if you dont want a son created dont run the program
+# a backup is created every time the program is run
+
+set :son_promoted_on,    :fri
+set :father_promoted_on, :last_fri_of_the_month
+
+# more complex
+# mon_wed_fri = Runt::DIWeek.new(Runt::Mon) | 
+#               Runt::DIWeek.new(Runt::Wed) | 
+#               Runt::DIWeek.new(Runt::Fri)
+# set :son_promoted_on, mon_wed_fri
+
+set :sons_to_keep,         14
+set :fathers_to_keep,       6
+set :grandfathers_to_keep,  6   # 6 months, by default
+
+
+# -------------------------
+# Standard Actions
+# -------------------------
+action(:tar_bz2) do
+  name = c[:tmp_dir] + "/" + File.basename(last_result) + ".tar.bz2"
+  sh "tar -cvjf #{name} #{last_result}"
+  name
+end
+
+action(:scp) do
+  # what should the default scp task be?
+  # scp the local file to the foreign directory. same name.
+  c[:servers].each do |server|
+    host = server =~ /localhost/ ? "" : "#{server}:"
+    sh "scp #{last_result} #{host}#{c[:backup_path]}/"
+  end
+  c[:backup_path] + "/" + File.basename(last_result)  
+end
+
+action(:mv) do
+  sh "mv #{last_result} #{c[:backup_path]}/"
+  c[:backup_path] + "/" + File.basename(last_result)
+end
+
+action(:encrypt) do
+  result = last_result
+  if c[:encrypt]
+    sh "gpg #{c[:gpg_encrypt_options]} --encrypt #{last_result}"
+    result = last_result + ".gpg" # ?
+  end
+  result
+end

data/lib/backup/rotator.rb

@@ -0,0 +1,155 @@
+require "rake"
+
+module Backup
+  class Rotator
+    include FileUtils
+
+    attr_reader :actor
+    attr_reader :configuration
+    alias_method :c, :configuration
+
+    def initialize(actor)
+      @actor = actor
+      @configuration = actor.configuration
+    end
+
+    # Take the last result and rotate it via mv on the local machine
+    def rotate_via_mv(last_result)
+      # verify that each of the directories exist, grandfathers, fathers, sons
+      hierarchy.each { |m| verify_local_backup_directory_exists(m) }
+
+      # place todays backup into the specified directory with a timestamp. 
+      newname = timestamped_prefix(last_result)
+      sh "mv #{last_result} #{place_in}/#{newname}"
+
+      cleanup_via_mv(place_in, how_many_to_keep_today)
+    end
+    
+    def rotate_via_ssh(last_result)
+      ssh = Backup::SshActor.new(c) 
+      ssh.connect 
+      ssh.run "echo \"#{last_result}\""
+
+      hierarchy.each do |m| 
+        dir = c[:backup_path] + "/" + m
+        ssh.verify_directory_exists(dir) 
+      end  
+
+      newname = timestamped_prefix(last_result)
+      ssh.run "mv #{last_result} #{place_in}/#{newname}"
+
+      ssh.cleanup_directory(place_in, how_many_to_keep_today)
+      ssh.close
+    end
+
+    # TODO
+    def rotate_via_ftp(last_result)
+#      ftp = Backup::FtpActor.new(c) 
+#      ftp.connect 
+#
+#      hierarchy.each do |m| 
+#        dir = c[:backup_path] + "/" + m
+#        ftp.verify_directory_exists(dir) 
+#      end  
+#
+#      newname = timestamped_prefix(last_result)
+#      ftp.run "mv #{last_result} #{place_in}/#{newname}"
+#
+#      ftp.cleanup_directory(place_in, how_many_to_keep_today)
+#      ftp.close
+    end
+
+    def create_sons_today?;     is_today_a? :son_created_on;     end
+    def promote_sons_today?;    is_today_a? :son_promoted_on;    end
+    def promote_fathers_today?; is_today_a? :father_promoted_on; end
+
+    # old ( offset_days % c[:son_promoted_on] ) == 0 ? true : false
+    # old ( offset_days % (c[:son_promoted_on] * c[:father_promoted_on]) ) == 0 ? true : false
+
+    private
+      def is_today_a?(symbol)
+        t = $test_time || Date.today
+        day = DateParser.date_from( c[symbol] )
+        day.include?( t )
+      end
+
+      def verify_local_backup_directory_exists(dir)
+        path = c[:backup_path]
+        full = path + "/" + dir
+        unless File.exists?(full) 
+          sh "mkdir -p #{full}"  
+        end
+      end
+
+      #def offset_days
+      #  t = $test_time || Time.now # todo, write a test to use this
+      #  num_from_day = Time.num_from_day( c[:promote_on] )
+      #  offset = ( t.days_since_epoch + num_from_day + 0)
+      #  offset
+      #end
+
+      def cleanup_via_mv(where, num_keep)
+
+        files = Dir[where + "/*"].sort
+        diff  = files.size - num_keep
+
+        1.upto( diff ) do 
+          extra = files.shift
+          sh "rm #{extra}"
+        end
+      end
+
+      def hierarchy 
+        %w{grandfathers fathers sons}
+      end
+
+      # figure out which generation today's backup is
+      public
+      def todays_generation
+        goes_in = promote_fathers_today? ? "grandfathers" :         \
+                  promote_sons_today?    ? "fathers"      : "sons"
+      end
+
+      private
+      def place_in
+        goes_in = todays_generation
+        place_in = c[:backup_path] + "/" + goes_in
+      end
+ 
+
+      # Given +name+ returns a timestamped version of name. 
+      def timestamped_prefix(name)
+        newname = Time.now.strftime("%Y-%m-%d-%H-%M-%S_") + File.basename(name)
+      end
+
+      # Returns the number of sons to keep. Looks for config values +:sons_to_keep+, 
+      # +:son_promoted_on+. Default +14+.
+      def sons_to_keep
+        c[:sons_to_keep]    || c[:son_promoted_on]    || 14
+      end
+      
+      # Returns the number of fathers to keep. Looks for config values +:fathers_to_keep+, 
+      # +:fathers_promoted_on+. Default +6+.
+      def fathers_to_keep
+        c[:fathers_to_keep] || c[:father_promoted_on] || 6
+      end
+
+      # Returns the number of grandfathers to keep. Looks for config values
+      # +:grandfathers_to_keep+. Default +6+.
+      def gfathers_to_keep
+        c[:grandfathers_to_keep] || 6
+      end
+
+      # This method returns the number of how many to keep in whatever today
+      # goes in. Example: if today is a day to create a +son+ then this
+      # function returns the value of +sons_to_keep+. If today is a +father+
+      # then +fathers_to_keep+ etc.
+      def how_many_to_keep_today
+        goes_in = todays_generation
+        keep = goes_in =~ /^sons$/         ? sons_to_keep     :
+               goes_in =~ /^fathers$/      ? fathers_to_keep  :
+               goes_in =~ /^grandfathers$/ ? gfathers_to_keep : 14
+      end
+
+   end
+end

data/lib/backup/ssh_helpers.rb

@@ -0,0 +1,138 @@
+# large portions borrowed from capistrano
+require 'rubygems'
+require 'net/ssh'
+
+# TODO - add in a way to extend the belay script to work with this. thats a
+# good idea for the belay script over all. write the kernal extensions, and the
+# rake extensions. form there write an example third party extension.
+module Backup
+  class Command
+    attr_reader   :command, :options
+    attr_reader   :actor
+
+    def initialize(server_name, command, callback, options, actor) #:nodoc:
+      @command  = command.strip.gsub(/\r?\n/, "\\\n")
+      @callback = callback
+      @options  = options
+      @actor    = actor
+      @channels = open_channels
+    end
+
+    def process!
+      since = Time.now
+      loop do
+        active = 0
+        @channels.each do |ch|
+          next if ch[:closed]
+          active += 1
+          ch.connection.process(true)
+        end
+
+        break if active == 0
+        if Time.now - since >= 1
+          since = Time.now
+          @channels.each { |ch| ch.connection.ping! }
+        end
+        sleep 0.01 # a brief respite, to keep the CPU from going crazy
+      end
+
+      #logger.trace "command finished"
+
+      if failed = @channels.detect { |ch| ch[:status] != 0 }
+        raise "command #{@command.inspect} failed"
+      end
+
+      self
+    end
+     
+    def open_channels
+      channel = actor.session.open_channel do |channel|
+           channel.request_pty( :want_reply => true )
+           channel[:actor] = @actor
+
+           channel.on_success do |ch|
+             #logger.trace "executing command", ch[:host]
+             ch.exec command
+             ch.send_data options[:data] if options[:data]
+           end
+
+           channel.on_data do |ch, data|
+             puts data
+             @callback[ch, :out, data] if @callback
+           end
+
+           channel.on_failure do |ch|
+             #logger.important "could not open channel", ch[:host]
+             # puts "we got a faulure"
+             ch.close
+           end
+
+           channel.on_request do |ch, request, reply, data|
+             ch[:status] = data.read_long if request == "exit-status"
+           end
+
+           channel.on_close do |ch|
+             ch[:closed] = true
+           end
+      end
+      [channel]
+    end
+  end  # end Class Command
+
+  class SshActor
+     
+    #def self.new_for_ssh(server_name)
+    #  a = new(server_name)
+    #end 
+
+    attr_reader :session
+    attr_reader :config
+    alias_method :c, :config
+
+    def initialize(config)
+      @config = config
+    end
+
+    def connect
+      c[:servers].each do |server| # todo, make this actually work
+      @session = Net::SSH.start(
+           server, 
+           c[:ssh_user],
+           :host_key     => "ssh-rsa",
+           :keys         => [ c[:identity_key] ],
+           :auth_methods => %w{ publickey } )
+      end
+    end
+
+    def run(cmd, options={}, &block)
+      #logger.debug "executing #{cmd.strip.inspect}"
+      puts "executing #{cmd.strip.inspect}"
+      command = Command.new(@server_name, cmd, block, options, self)
+      command.process! # raises an exception if command fails on any server
+    end
+
+    def on_remote(&block)
+      connect
+      self.instance_eval(&block)
+      close
+    end
+
+    def close
+      @session.close
+    end
+
+    def verify_directory_exists(dir)
+      run "if [ -d '#{dir}' ]; then true; else mkdir -p '#{dir}'; fi"
+    end
+
+    def cleanup_directory(dir, keep)
+      puts "Cleaning up" 
+      cleanup = <<-END
+      LOOK_IN="#{dir}"; COUNT=`ls -1 $LOOK_IN | wc -l`; MAX=#{keep}; if (( $COUNT > $MAX )); then let "OFFSET=$COUNT-$MAX"; i=1; for f in `ls -1 $LOOK_IN | sort`; do if (( $i <= $OFFSET )); then CMD="rm $LOOK_IN/$f"; echo $CMD; $CMD; fi; let "i=$i + 1"; done; else true; fi
+      END
+      run cleanup # todo make it so that even this can be overridden
+    end
+
+   end # end class SshActor
+
+end # end Module Backup

data/lib/backup.rb

@@ -0,0 +1,7 @@
+require 'rubygems'
+require 'runt'
+require 'date'
+require 'backup/configuration'
+require 'backup/extensions'
+require 'backup/ssh_helpers'
+require 'backup/date_parser'

data/tests/actor_test.rb

@@ -0,0 +1,70 @@
+#!/usr/bin/ruby
+require File.dirname(__FILE__) + "/tests_helper"
+
+class ActorTest < Test::Unit::TestCase
+  def setup
+    @config = Backup::Configuration.new
+    @config.load "standard"
+    @actor = @config.actor
+    setup_tmp_backup_dir
+  end
+
+  def dont_test_exists
+     assert @actor
+  end
+
+  def dont_test_is_file
+    dir = create_tmp_files
+    config = <<-END
+      action :content, :is_file => "#{dir}/1"
+    END
+    @config.load :string => config
+    assert result = @actor.content
+    assert File.exists?(result)
+    puts "content result is: #{result}"
+    @actor.start_process! 
+  end
+
+  def dont_test_is_folder
+    dir = create_tmp_files
+    config = <<-END
+      action :content, :is_folder => "#{dir}"
+    END
+    @config.load :string => config
+    assert result = @actor.content
+    assert File.exists?(result)
+    assert File.directory?(result)
+    puts "content result is: #{result}"
+    @actor.start_process! 
+  end
+
+  def test_is_contents_of
+    dir = create_tmp_files
+    config = <<-END
+      action :content, :is_contents_of => "#{dir}", :copy => true
+    END
+    @config.load :string => config
+    #@actor.content
+    #@actor.cleanup
+    @actor.start_process! 
+  end
+
+  private
+    def setup_tmp_backup_dir
+      newtmp = @config[:tmp_dir] + "/backup_#{rand}_" + Time.now.strftime("%Y%m%d%H%M%S")
+      sh "mkdir #{newtmp}"
+      config = <<-END
+        set :backup_path, "#{newtmp}" 
+      END
+      @config.load :string => config
+    end
+
+    def create_tmp_files
+      newtmp = @config[:tmp_dir] + "/test_#{rand}_" + Time.now.strftime("%Y%m%d%H%M%S")
+      sh "mkdir #{newtmp}"
+      0.upto(5) { |i| sh "touch #{newtmp}/#{i}" }
+      newtmp
+    end
+
+end
+

data/tests/cleanup.sh

@@ -0,0 +1,2 @@
+#!/usr/bin/bash
+LOOK_IN="/var/local/backups/mediawiki/sons"; COUNT=`ls -1 $LOOK_IN | wc -l`; MAX=14; if (( $COUNT > $MAX )); then let "OFFSET=$COUNT-$MAX"; i=1; for f in `ls -1 $LOOK_IN | sort`; do if (( $i <= $OFFSET )); then CMD="rm $LOOK_IN/$f"; echo $CMD; $CMD; fi; echo "$i $f"; let "i=$i + 1"; done; else true; fi

data/tests/rotation_test.rb

@@ -0,0 +1,32 @@
+#!/usr/bin/ruby
+require File.dirname(__FILE__) + "/tests_helper"
+require 'runt'
+require 'date'
+
+class RotationTest < Test::Unit::TestCase
+  def setup
+    # setup our config to test the objects
+    @config = Backup::Configuration.new
+    @config.load "standard"
+    @rotator = Backup::Rotator.new(@config.actor) 
+  end
+
+  def test_rotator
+    t = Date.today
+    r = @rotator
+    0.upto(14) do |i|
+      $test_time = t
+      print t.to_s + t.strftime(" #{i} %a ").to_s 
+      puts r.todays_generation 
+      t += 1
+    end
+  end
+
+  def test_date_parsing
+    dp = Backup::DateParser.new
+    assert fri   = dp.date_from(:fri)
+    assert daily = dp.date_from(:daily)
+    assert last  = dp.date_from(:last_mon_of_the_month)
+    assert_raise(RuntimeError) { dp.date_from(:asdfasdf) }
+  end
+end

data/tests/ssh_test.rb

@@ -0,0 +1,21 @@
+require File.dirname(__FILE__) + "/tests_helper"
+
+class SSHTest < Test::Unit::TestCase
+  def setup
+    @config = Backup::Configuration.new
+    @config.load "standard"
+    @config.action :deliver, :method => :via_ssh 
+    @actor = Backup::SshActor.new(@config) 
+  end
+
+  def test_exists
+    assert @config
+    assert @actor
+  end
+
+  def test_on_remote
+    @actor.on_remote do
+      run "echo \"hello $HOSTNAME\""
+    end
+  end
+end

data/tests/tests_helper.rb

@@ -0,0 +1,5 @@
+$:.unshift File.dirname(__FILE__) + "/../lib"
+
+require 'test/unit'
+require 'backup'
+require 'backup/extensions'

metadata

@@ -0,0 +1,99 @@
+--- !ruby/object:Gem::Specification 
+rubygems_version: 0.9.0
+specification_version: 1
+name: backupgem
+version: !ruby/object:Gem::Version 
+  version: 0.0.2
+date: 2006-10-06 00:00:00 -07:00
+summary: Beginning-to-end solution for backups and rotation.
+require_paths: 
+- lib
+email: nate@natemurray.com
+homepage: http://tech.natemurray.com/backup
+rubyforge_project: 
+description: 
+autorequire: backupgem
+default_executable: 
+bindir: bin
+has_rdoc: true
+required_ruby_version: !ruby/object:Gem::Version::Requirement 
+  requirements: 
+  - - ">"
+    - !ruby/object:Gem::Version 
+      version: 0.0.0
+  version: 
+platform: ruby
+signing_key: 
+cert_chain: 
+post_install_message: 
+authors: 
+- Nate Murray
+files: 
+- bin/backup
+- bin/commands.sh
+- lib/backup
+- lib/backup.rb
+- lib/backup/actor.rb
+- lib/backup/cli.rb
+- lib/backup/configuration.rb
+- lib/backup/date_parser.rb
+- lib/backup/extensions.rb
+- lib/backup/recipes
+- lib/backup/rotator.rb
+- lib/backup/ssh_helpers.rb
+- lib/backup/recipes/standard.rb
+- tests/actor_test.rb
+- tests/cleanup.sh
+- tests/rotation_test.rb
+- tests/ssh_test.rb
+- tests/tests_helper.rb
+- examples/global.rb
+- examples/mediawiki.rb
+- doc/index.html
+- doc/LICENSE-GPL
+- doc/styles.css
+- README
+- CHANGELOG
+test_files: 
+- tests/actor_test.rb
+- tests/rotation_test.rb
+- tests/ssh_test.rb
+rdoc_options: []
+
+extra_rdoc_files: 
+- README
+- CHANGELOG
+executables: []
+
+extensions: []
+
+requirements: []
+
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: rake
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Version::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 0.7.1
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: runt
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Version::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 0.3.0
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: net-ssh
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Version::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 1.0.9
+    version: