jmstacey-cfbackup 0.5.0 → 0.6.0

data/CHANGELOG.markdown

@@ -1,6 +1,19 @@
 CFBackup ChangeLog
 ==================
 
+0.6.0 2009-05-03
+-----------------
+* Added example_scripts to RDoc.
+* gem creates cfconfig.yml in /etc for reference
+* CFBackup looks in multiple places for config file
+  * Hidden directory in $HOME ($HOME/.cfconfig.yml)
+  * Non-hidden in current directory (./cfconfig.yml)
+  * In /etc/cfconfig.yml
+* Refactoring and new usage with --action push|pull|delete
+* Pulling files implemented
+* Deleting files implemented
+* Initial unit tests completed
+
 0.5.0 2009-04-18
 -----------------
 

data/README.markdown

@@ -1,9 +1,7 @@
 CFBackup
 =========
 
-CFBackup is a small ruby program that transfers files or directories from the
-local machine to a Cloud Files container. It is meant to serve as a useful tool
-for automated backups.
+CFBackup is a small ruby program that transfers files or directories from the local machine to a Cloud Files container. It is meant to serve as a useful tool for automated backups.
 
 Features
 -----------
@@ -11,21 +9,50 @@ Features
 * Backup a single file or directory (recursion uses pseudo directories)
 * Pipe data straight to container
 * Free transfers over local Rackspace network for Slicehost/Cloud Server 
-	customers in DFW1 datacenter
+  customers in DFW1 datacenter
 	
 Requirements
 --------------
 
-TODO: Complete this area
 * ruby-cloudfiles
 
+Notes: 
+* If you install CFBackup as a gem, all of the dependencies _should_ automatically be installed for you.
+* Ubuntu Users: The Ubuntu rubygems package will installs executables outside your normal PATH. You will
+  need to update it or create a symlink to access cfbackup from anywhere. See the wiki for more information.reating a symlink.
+
 Install
 -----------
 
 * gem sources -a http://gems.github.com
 * sudo gem install jmstacey-cfbackup
 
-Depending on what Operating System you're using, you may be required to install libs that are outside of the gem world.
+Configuration
+-----------
+
+CFBackup will look in the following places (in order) for the configuration file named cfconfig.yml
+
+* Hidden in home directory (~/.cfbackup.yml)
+* Non-hidden in present working directory (./cfbackup.yml)
+* In etc (/etc/cfbackup.yml)
+
+The configuration file can be overridden at any time with the --config_file option
+
+Configuration
+-----------
+
+    Usage: cfbackup.rb --action push|pull|delete options --container CONTAINER
+            --action ACTION              Action to perform: push, pull, or delete.
+            --pipe_data                  Pipe data from another application and stream it to Cloud Files
+        -r, --recursive                  Traverse local path recursivley
+        -v, --verbose                    Run verbosely
+            --local_path LOCAL_PATH      Local path or file
+            --container CONTAINER        Cloud Files container name
+            --version                    Show current version
+            --config_file PATH           Use specified config file, rather than the default
+            --local_net                  Use unmetered connection in DFW1 (only applicable to Slicehost or Mosso Cloud Server customers)
+            
+The wiki has usage examples and some sample automation scripts can be found in the example_scripts directory.
 
 Copyright
 ------------

data/Rakefile

@@ -10,7 +10,7 @@ begin
     gem.homepage = "http://github.com/jmstacey/cfbackup"
     gem.authors = ["Jon Stacey"]
     
-    # dependencies
+    # Dependencies
     gem.add_dependency('jmstacey-ruby-cloudfiles', '>=1.3.3')
 
     # gem is a Gem::Specification... see http://www.rubygems.org/read/chapter/20 for additional settings
@@ -39,7 +39,6 @@ rescue LoadError
   end
 end
 
-
 task :default => :test
 
 require 'rake/rdoctask'
@@ -54,6 +53,7 @@ Rake::RDocTask.new do |rdoc|
   rdoc.rdoc_dir = 'rdoc'
   rdoc.title = "cfbackup #{version}"
   rdoc.rdoc_files.include('README*')
+  rdoc.rdoc_files.include('example_scripts/**')
   rdoc.rdoc_files.include('lib/**/*.rb')
 end
 

data/VERSION.yml

@@ -1,4 +1,4 @@
 --- 
 :major: 0
-:minor: 5
-:patch: 0
+:minor: 6
+:patch: 0

data/bin/cfbackup

@@ -1,6 +1,6 @@
 #!/usr/bin/env ruby
 require 'rubygems'
-require 'CFBackup'
+require 'cfbackup'
 
 backup = CFBackup.new(ARGV)
 backup::run

data/lib/OptCFBackup.rb

@@ -12,14 +12,14 @@ class OptCFBackup
   # Initializes object with command line arguments passed
   def initialize(args)
     
-    @banner = "Usage: cfbackup.rb [options] --pipe_data|--local_path PATH --container CONTAINER"
+    @banner = "Usage: cfbackup.rb --action push|pull|delete options --container CONTAINER"
     
     @options = OpenStruct.new
-    self.options.config       = ['~/.cfconfig.yml', './cfconfig.yml', '/etc/cfconfig.yml']
+    self.options.config       = ["#{ENV['HOME']}/.cfconfig.yml", './cfconfig.yml', '/etc/cfconfig.yml']
+    self.options.action       = ''
     self.options.pipe_data    = false
     self.options.show_ver     = false
     self.options.recursive    = false
-    self.options.restore      = false
     self.options.local_net    = false
     self.options.container    = ''
     self.options.local_path   = ''
@@ -29,6 +29,10 @@ class OptCFBackup
     opts = OptionParser.new do |opts|
       opts.banner = self.banner
       
+      opts.on("--action ACTION", "Action to perform: push, pull, or delete.") do |action|
+        self.options.action = action
+      end
+      
       opts.on("--pipe_data", "Pipe data from another application and stream it to Cloud Files") do |pipe|
         self.options.pipe_data = pipe
       end
@@ -50,15 +54,12 @@ class OptCFBackup
         clean_remote_path unless (self.options.remote_path.nil?) 
       end
       
-      opts.on("--restore", "Restore files to local path") do |restore|
-        self.options.restore = restore
-      end
-      
       opts.on("--version", "Show current version") do |version|
         self.options.show_ver = version
       end
       
-      opts.on("--config_file PATH", "Use specified config file, rather than the default") do |config|      
+      opts.on("--config_file PATH", "Use specified config file, rather than the default") do |config|
+        self.options.config = Array.new()
         self.options.config << config
       end
       
@@ -70,7 +71,7 @@ class OptCFBackup
     
     opts.parse!(args)
     
-  end # parse()
+  end # initialize()
   
   private
   
@@ -78,7 +79,7 @@ class OptCFBackup
     if self.options.remote_path[0,1] == "/"
       self.options.remote_path.slice!(0)
     end
-    # Won't work for piped data. Might result in "text.txt/"
+    # Follwoig won't work for piped data. Might result in "text.txt/"
     # self.options.remote_path = self.options.remote_path + "/" unless (self.options.remote_path[-1,1] == "/")
   end
   

data/lib/cfbackup.rb

@@ -15,8 +15,9 @@
 #     along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 require 'rubygems'
+require 'ftools'
 require 'cloudfiles'
-require 'OptCFBackup'
+require 'optcfbackup'
 require 'yaml'
 
 class CFBackup
@@ -27,8 +28,13 @@ class CFBackup
     # Special case if the version is requested
     if @opts.options.show_ver
       version_file = File.join(File.dirname(__FILE__), '..', 'VERSION.yml')
-      config = YAML::load(File.open(version_file))
-      show_error("CFBackup v#{config[:major]}.#{config[:minor]}.#{config[:patch]}")
+      if File.exist?(version_file)
+        config = YAML.load(File.read(version_file))
+        version = "#{config[:major]}.#{config[:minor]}.#{config[:patch]}"
+      else
+        version = "unkown version"
+      end
+      show_error("CFBackup #{version}")
     end
     
     # Locate and load config file
@@ -40,23 +46,26 @@ class CFBackup
     end
     show_error('Error: Unable to locate config file.') unless (@conf != nil)
     
+    prep_connection
+    
   end # initialize()
   
   def run
     
     show_error() unless (@opts.options.container != "")
     
-    if @opts.options.pipe_data
-      prep_container
-      run_piped_data
-    elsif @opts.options.local_path != ""
-      prep_container
-      
-      if @opts.options.restore
-        run_restore
+    # Run appropriate action
+    case @opts.options.action
+    when 'push'
+      if @opts.options.pipe_data
+        push_piped_data
       else
-        run_backup
+        push_files
       end
+    when 'pull'
+      pull_files()
+    when 'delete'
+      delete_files
     else
       show_error()
     end
@@ -65,8 +74,7 @@ class CFBackup
   
   private
   
-  def prep_container
-    
+  def prep_connection
     # Establish connection
     show_verbose "Establishing connection...", false
     @cf = CloudFiles::Connection.new(@conf["username"], @conf["api_key"]);
@@ -76,30 +84,38 @@ class CFBackup
     if @opts.options.local_net
       @cf.storagehost = 'snet-storage.clouddrive.com'
     end
+  end # prep_connection()
     
-    # Check for the container. If it doesn't exist, create it.
-    unless @cf.container_exists?(@opts.options.container)
-      show_verbose "Conainer '#{@opts.options.container}' does not exist. Creating it...", false
-      @cf.create_container(@opts.options.container)
-      show_verbose " done."
+  def prep_container(create_container = true)
+    # Check for the container. If it doesn't exist, create it if allowed
+    if !@cf.container_exists?(@opts.options.container)
+      if create_container
+        show_verbose "Container '#{@opts.options.container}' does not exist. Creating it...", false
+        @cf.create_container(@opts.options.container)
+        show_verbose " done."
+      else
+        show_error("Error: Container '#{@opts.options.container}' does not exist.")
+      end
     end
     
     @container = @cf.container(@opts.options.container)
-    
-  end # prepConnection()
+  end # prep_cnnection()
   
-  def run_piped_data
-    puts "Warning: 5GB per stream cap"
+  def push_piped_data
+    prep_container
+    
+    puts "Warning: 5GB maximum filesize"
     object = @container.create_object(@opts.options.remote_path, true)
     object.write("STDIN")
-  end
+  end # push_piped_data()
   
-  def run_backup
+  def push_files
+    prep_container
     
     path = @opts.options.local_path
-  
+    pwd  = Dir.getwd # Save current directory so we can come back
+    
     if FileTest::file?(path)
-      Dir.chdir(File::dirname(path))
       glob_options = File.join(File::basename(path))
     elsif @opts.options.recursive
       Dir.chdir(path)
@@ -111,41 +127,173 @@ class CFBackup
     files = Dir.glob(glob_options)
     
     # Upload file(s)
-    files.each do |file|  
+    counter = 1
+    show_verbose "There are #{files.length} files to process."
+    files.each do |file|
       file = file.sub(/\.\//, '')
       if file == "" || file[0,1] == "." || FileTest.directory?(file)
+        show_verbose "(#{counter}/#{files.length}) Skipping #{file}"
+        counter += 1
         next 
       end
       
-      show_verbose "Uploading #{file}...", false
-      object = @container.create_object(file, true)
+      show_verbose "(#{counter}/#{files.length}) Pushing file #{file}...", false
+      
+      if @opts.options.remote_path.to_s == ''
+        remote_path = file
+      else
+        remote_path = File.join(@opts.options.remote_path, file)
+      end
+        
+      object = @container.create_object(remote_path, true)
       object.load_from_filename(file)
+      
       show_verbose " done."
+      counter += 1
     end # files.each
     
-  end # run_backup()
+    Dir.chdir(pwd) # Go back to original directory
+    
+  end # push_files()
+  
+  def pull_files
+    prep_container(false)
+    
+    file    = object_file?
+    objects = get_objects_array
+    
+    # Process objects
+    counter = 1
+    show_verbose "There are #{objects.length} objects to process."
+    objects.each do |object_name|
+      object = @container.object(object_name)
+      if object.content_type == "application/directory"
+        show_verbose "(#{counter}/#{objects.length}) Pseudo directory #{object.name}"
+        counter += 1
+        next
+      end
+      
+      path_info = File.split(@opts.options.local_path.to_s)
+      file_info = File.split(object.name.to_s)
+      
+      if file # Dealing with a single file pull
+        if @opts.options.local_path.to_s == ''
+          filepath = file_info[1].to_s # Use current directory and original name
+        else
+          if File.exist?(@opts.options.local_path.to_s)
+            # The file exists, so we will overwrite it
+            filepath = File.join(@opts.options.local_path.to_s)
+          else
+            # If the file doesn't exist, a new name may have been given.
+            # Test the path.
+            if File.exist?(path_info[0])
+              # A new name was given with a valid path
+              filepath = File.join(path_info[0], path_info[1])
+            else
+              # The given path is not valid
+              show_error("cfbackup: #{file_info[0]}: No such file or directory.")
+            end
+          end
+        end
+      else # Dealing with a multi-object pull
+        if @opts.options.local_path.to_s == ''
+          filepath = object.name.to_s # Use current directory with object name
+          dir_path = file_info[0]
+        else
+          if File.directory?(@opts.options.local_path.to_s)
+            filepath = File.join(@opts.options.local_path.to_s, object.name.to_s)
+            dir_path = File.join(@opts.options.local_path, file_info[0])
+          else
+            # We can't copy a directory to a file...
+            show_error("cfbackup: #{@container.name}:#{@opts.options.remote_path.to_s}/ is a directory (not copied).")
+          end
+        end
+        File.makedirs(dir_path) # Create subdirectories as needed
+      end
+      
+      show_verbose "(#{counter}/#{objects.length}) Pulling object #{object.name}...", false
+      object.save_to_filename(filepath)
+      show_verbose " done"
+      counter += 1
+    end
+  end # pull_files()
+  
+  def delete_files
+    prep_container(false)
+    
+    if object_file?
+      show_verbose "Deleting object #{@opts.options.remote_path.to_s}"
+      @container.delete_object(@opts.options.remote_path.to_s)
+    else
+      if !@opts.options.recursive
+        show_error("Error: #{@opts.options.remote_path} is a directory but the the recursive option was not specified. Doing nothing.")
+      else
+        objects = get_objects_array
+        
+        # Process objects
+        counter = 1
+        show_verbose "There are #{objects.length} objects to process."
+        objects.each do |object_name|
+          show_verbose "(#{counter}/#{objects.length}) Deleting object #{object_name}...", false
+          @container.delete_object(object_name)
+          show_verbose " done"
+          counter += 1
+        end
+      end
+    end
+    
+  end # delete_files()
+  
+  
+  # Helper methods below
+  
+  def object_file?
+    
+    file = false
+    unless @opts.options.remote_path.to_s == ''
+      if @container.object_exists?(@opts.options.remote_path)
+        if @container.object(@opts.options.remote_path).content_type != "application/directory"
+          file = true
+          if @opts.options.recursive
+            puts "Warning: This is a file so the recursive option is meaningless."
+          end
+        end
+      else
+        show_error("The object #{@opts.options.remote_path} does not exist")
+      end
+    end
+    
+    return file
+  end
   
-  def run_restore
+  def get_objects_array
     
-    # TODO: Implement run_restore
-    # We have to do a bit of fancy footwork to make directories work
-    puts "Oops! Restore hasn't been implemented yet. Help me out and submit a patch :-)"
+    file = object_file?
     
-  end # run_restore()
+    # Get array of objects to process
+    objects = Array.new
+    if file
+      objects << @opts.options.remote_path.to_s
+    elsif @opts.options.recursive
+      # Use prefix instead of path so that "subdirectories" are included
+      objects = @container.objects(:prefix => @opts.options.remote_path)
+    else
+      objects = @container.objects(:path => @opts.options.remote_path)
+    end
+    
+    return objects
+  end
   
   # Shows given message if verbose output is turned on
   def show_verbose(message, line_break = true)
-    
     unless !@opts.options.verbose
       if line_break
         puts message
       else
         print message
       end
-      
       $stdout.flush
     end
-    
   end # show_verbose()
   
   # Show error message, banner and exit
@@ -155,8 +303,4 @@ class CFBackup
     exit
   end # show_error()
   
-  def parse_container_path(container)
-    # Split based on :
-  end # parse_container_path()
-  
 end # class CFBackup

data/test/cfbackup_test.rb

@@ -1,7 +1,107 @@
 require 'test_helper'
 
 class CfbackupTest < Test::Unit::TestCase
-  should "probably rename this file and start testing for real" do
-    flunk "hey buddy, you should probably rename this file and start testing for real"
+  TEST_DIR = 'test/tmp'
+  
+  context "A backup" do
+    
+    context "with a single file push" do
+      setup do
+        mock_ARGV = ['--action', 'push', '--local_path', 'test/data/data.txt', '--container', 'test', '--config_file', 'test/cfconfig.yml', '-v']
+        @backup = CFBackup.new(mock_ARGV)
+      end
+      
+      should "return true when file sucessfully sent" do
+        assert @backup.run
+      end
+    end
+    
+    context "with a recursive directory push" do
+      setup do
+        mock_ARGV = ['--action', 'push', '-r', '--local_path', 'test/data', '--container', 'test', '--config_file', 'test/cfconfig.yml', '-v']
+        @backup = CFBackup.new(mock_ARGV)
+      end
+      
+      should "return true when all files/directories successfully sent" do
+        assert @backup.run
+      end
+    end
+
+  end 
+  
+  context "A restore" do
+    
+    context "with a single file pull" do
+      file     = 'data.txt'
+      filepath = File.join(TEST_DIR, file)
+  
+      setup do
+        mock_ARGV = ['--action', 'pull', '--container', "test:#{file}", '--local_path', "#{filepath}", '--config_file', 'test/cfconfig.yml', '-v']
+        @backup = CFBackup.new(mock_ARGV)
+      end
+  
+      should "return true when file succesfully pulled" do
+        assert @backup.run
+      end
+  
+      # I don't know why this doesn't work. It's like File is caching results
+      # and not updating until the applicaiton exists. Overcoming by
+      # asserting the file deletion during teardown at the loss of shoulda
+      #
+      # should "result in test/tmp/#{file} existing" do      
+      #   assert File.exist?(filepath)
+      # end
+  
+      teardown do
+        assert File.delete(filepath)
+      end
+    end
+    
+    context "with a recursive pull" do    
+      setup do
+        mock_ARGV = ['--action', 'pull', '--container', "test", '--local_path', "test/tmp", '--config_file', 'test/cfconfig.yml', '-v']
+        @backup = CFBackup.new(mock_ARGV)
+      end
+      
+      should "return true when all files pulled" do
+        assert @backup.run
+      end
+      
+      # Todo compare directories
+      
+      teardown do
+        system "rm -rf test/tmp/*"
+        # I didn't feel safe with TEST_DIR + '/*'
+        # Disaster waiting to happen. rm -rf is already bad enough.
+      end
+    end # context "A recursive pull"
+    
   end
+  
+  context "A deletion" do
+    
+    context "of a single file" do
+      setup do
+        mock_ARGV = ['--action', 'delete', '--container', "test:folder_1/file1.txt", '--config_file', 'test/cfconfig.yml', '-v']
+        @backup = CFBackup.new(mock_ARGV)
+      end
+      
+      should "return true when file deleted" do
+        assert @backup.run
+      end
+    end
+    
+    context "of a pseudo directory" do
+      setup do
+        mock_ARGV = ['--action', 'delete', '-r', '--container', "test:folder_2", '--config_file', 'test/cfconfig.yml', '-v']
+        @backup = CFBackup.new(mock_ARGV)
+      end
+      
+      should "return true when directory deleted" do
+        assert @backup.run
+      end
+    end
+    
+  end # context "A deletion"
+  
 end

data/test/cfconfig.yml

@@ -0,0 +1,2 @@
+username: xxxxxxxxxxxxx
+api_key: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx

data/test/data/folder_1/file1.txt

@@ -0,0 +1,2 @@
+Folder 1
+Test File 1

data/test/data/folder_1/file2.txt

@@ -0,0 +1,2 @@
+Folder 1
+Test File 1

data/test/data/folder_1/folder_3/file1.txt

@@ -0,0 +1,2 @@
+Folder 1/Folder 3
+Test File 1

data/test/data/folder_2/file1.txt

@@ -0,0 +1,2 @@
+Folder 2
+Test File 1

data/test/data/folder_2/file2.txt

@@ -0,0 +1,2 @@
+Folder 2
+Test File 1

data/test/test_helper.rb

@@ -6,5 +6,5 @@ $LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
 $LOAD_PATH.unshift(File.dirname(__FILE__))
 require 'cfbackup'
 
-class Test::Unit::TestCase
+class Test::Unit::TestCase  
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: jmstacey-cfbackup
 version: !ruby/object:Gem::Version 
-  version: 0.5.0
+  version: 0.6.0
 platform: ruby
 authors: 
 - Jon Stacey
@@ -9,7 +9,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2009-04-19 00:00:00 -07:00
+date: 2009-05-03 00:00:00 -07:00
 default_executable: cfbackup
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -41,7 +41,13 @@ files:
 - lib/OptCFBackup.rb
 - lib/cfbackup.rb
 - test/cfbackup_test.rb
-- test/data.txt
+- test/cfconfig.yml
+- test/data/data.txt
+- test/data/folder_1/file1.txt
+- test/data/folder_1/file2.txt
+- test/data/folder_1/folder_3/file1.txt
+- test/data/folder_2/file1.txt
+- test/data/folder_2/file2.txt
 - test/test_helper.rb
 has_rdoc: true
 homepage: http://github.com/jmstacey/cfbackup