paparazzi 0.1.0

data/.gitignore

@@ -0,0 +1,3 @@
+*.gem
+.loadpath
+.project

data/LICENSE

@@ -0,0 +1,22 @@
+Copyright (c) 2011 Jonathan S. Garvin (http://www.5valleys.com)
+
+Permission is hereby granted, free of charge, to any person
+obtaining a copy of this software and associated documentation
+files (the "Software"), to deal in the Software without
+restriction, including without limitation the rights to use,
+copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the
+Software is furnished to do so, subject to the following
+conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,50 @@
+Paparazzi
+=========
+
+Paparazzi is a Ruby gem for making incremental snapshot rsync backups of a directory, maintaining
+hourly, daily, weekly, monthy, and yearly snapshots of directory state and files without consuming
+much more drive space than a single copy would (depending on how frequently existing files change).
+
+Only changed or new files are copied to the new snapshot. Hard links are created in the new snapshot
+to previously existing unchanged files, allowing for frequent backups to be made of very large
+directories (see 'How it works' below). Older snapshots maintain versions of files as they existed
+at the time that snapshot was made.
+
+Paparazzi automatically purges out old hourly, daily, weekly, and monthly snapshots. Yearly
+snapshots are not automatically purged and need to be removed manually when necessary.
+
+Installation
+------------
+
+    gem install paparazzi
+    
+
+Usage
+-----
+
+Create a ruby script that you'll run hourly from a cron.
+ 
+    require 'rubygems' #unless you use another gem package manager
+    require 'paparazzi'
+    
+    settings = {
+      :source => '/full/path/to/source/directory/',           # note the trailing '/'
+      :destination => '/mnt/external_drive/backup_folder',
+      :rsync_flags => '-L'                                    # see 'man rsync' for available options.
+    }    
+
+    Paparazzi::Camera.trigger(settings)
+    
+
+How it works.
+-------------
+
+Paparazzi uses rsync's ability to make hard links to files that haven't changed from previous
+backups. So, even though multiple incremental versions of the entire directory are kept, only a single
+copy of unique files are kept, with multiple hard links in separate snapshots pointing to the same
+file.
+
+This gem owes it's existance to Mike Rubel's excellent write-up
+[Easy Automated Snapshot-Style Backups with Linux and Rsync](http://www.mikerubel.org/computers/rsync_snapshots/).
+If you're not sure what "hard links" are or are confused how multiple snapshot versions of a
+directory can be made without taking up much more space than a single copy, then read his post.

data/Rakefile

@@ -0,0 +1,10 @@
+require 'rake/testtask'
+
+task :default => :test
+
+Rake::TestTask.new(:test) do |t|
+  t.libs << 'lib'
+  t.libs << 'test'
+  t.pattern = 'test/**/*_test.rb'
+  t.verbose = true
+end

data/lib/paparazzi.rb

@@ -0,0 +1 @@
+require File.expand_path('../paparazzi/camera', __FILE__)

data/lib/paparazzi/camera.rb

@@ -0,0 +1,124 @@
+require 'admit_one'
+require 'yaml'
+require 'fileutils'
+
+module Paparazzi
+  class Camera
+    FREQUENCIES = [:hourly,:daily,:weekly,:monthly,:yearly]
+    REQUIRED_SETTINGS = [:source,:destination]
+      
+    class << self
+      attr_accessor :source, :destination, :rsync_flags
+      
+      def trigger(settings = {})
+        validate_and_cache_settings(settings)
+        AdmitOne::LockFile.new(:paparazzi) do
+          initialize
+          purge_old_snapshots
+          make_snapshots
+        end
+        
+      end
+      
+      #######
+      private
+      #######
+      
+      def validate_and_cache_settings(settings)
+        [:source,:destination,:rsync_flags].each do |setting_name|
+          if REQUIRED_SETTINGS.include?(setting_name) and settings[setting_name].nil?
+            raise MissingSettingError, "#{setting_name} is required"
+          else
+            self.send("#{setting_name}=",settings[setting_name])
+          end
+        end
+        
+        raise MissingFolderError, source unless File.exist?(source) and File.directory?(source)
+        raise MissingFolderError, destination unless File.exist?(destination) and File.directory?(destination)
+      end
+      
+      def initialize
+        @previous_snapshot_name = {}
+        FREQUENCIES.each do |frequency|
+          Dir.mkdir(destination(frequency)) unless File.exists?(destination(frequency)) && File.directory?(destination(frequency))
+        
+          full_path = Dir[destination(frequency)+'/*'].sort{|a,b| File.ctime(b) <=> File.ctime(a) }.first
+          @previous_snapshot_name[frequency] = full_path ? File.basename(full_path) : ''
+        end
+        
+        if @previous_snapshot_name[:hourly] != last_successful_hourly_snapshot and !last_successful_hourly_snapshot.nil? and File.exists?(destination(:hourly) + '/' + last_successful_hourly_snapshot)
+          File.rename(previous_snapshot(:hourly), current_snapshot(:hourly))
+          @previous_snapshot_name[:hourly] = last_successful_hourly_snapshot
+        end
+      end
+      
+      def destination(frequency = nil)
+        frequency.nil? ? @destination : "#{@destination}/#{frequency}"
+      end
+      
+      def last_successful_hourly_snapshot
+        @last_successful_hourly_snapshot ||= File.exists?("#{destination}/.paparazzi.yml") ? YAML.load_file("#{destination}/.paparazzi.yml")[:last_successful_snapshot] : nil
+      end
+      
+      def last_successful_hourly_snapshot=(string)
+        cached_data ||= File.exists?("#{destination}/.paparazzi.yml") ? YAML.load_file("#{destination}/.paparazzi.yml") : {}
+        cached_data[:last_successful_snapshot] = string
+        File.open("#{destination}/.paparazzi.yml", 'w') {|file| file.write(cached_data.to_yaml) }
+      end
+    
+      def purge_old_snapshots
+        keepers = {:hourly => 24, :daily => 7, :weekly => 5, :monthly => 12}
+        keepers.keys.each do |frequency|
+          while Dir[destination(frequency)+'/*'].size > keepers[frequency]-1
+            full_path = Dir[destination(frequency)+'/*'].sort{|a,b| File.ctime(a) <=> File.ctime(b) }.first
+            FileUtils.rm_rf(full_path)
+          end
+        end
+      end
+
+      def make_snapshots
+        FREQUENCIES.each do |frequency|
+          Dir.mkdir(current_snapshot(frequency)) unless File.exists?(current_snapshot(frequency))
+          if frequency == :hourly and previous_snapshot_name(frequency) == ''
+            system 'rsync', *(['-aq', '--delete'] + [rsync_flags] + [source, current_snapshot(frequency)])
+            self.last_successful_hourly_snapshot = current_snapshot_name(:hourly)
+          elsif previous_snapshot_name(frequency) != current_snapshot_name(frequency)
+            system 'rsync', *(['-aq', '--delete', "--link-dest=#{link_destination(frequency)}"] + [rsync_flags] + [source, current_snapshot(frequency)])
+            self.last_successful_hourly_snapshot = current_snapshot_name(:hourly)
+          end
+        end
+      end
+      
+      def current_snapshot(frequency)
+        destination(frequency) + '/' + current_snapshot_name(frequency)
+      end
+      
+      def current_snapshot_name(frequency)
+        @start_time ||= Time.now #lock in time so that all results stay consistent over long runs
+        case frequency
+          when :hourly  then @start_time.strftime('%Y-%m-%d.%H')
+          when :daily   then @start_time.strftime('%Y-%m-%d')
+          when :weekly  then sprintf("%04d-%02d-week-%02d", @start_time.year, @start_time.month, (@start_time.day/7))
+          when :monthly then @start_time.strftime("%Y-%m")
+          when :yearly  then @start_time.strftime("%Y")
+        end
+      end
+      
+      def previous_snapshot(frequency)
+        destination(frequency) + '/' + previous_snapshot_name(frequency)
+      end
+      
+      def previous_snapshot_name(frequency)
+        @previous_snapshot_name[frequency]
+      end
+      
+      def link_destination(frequency)
+        frequency == :hourly ? "../#{previous_snapshot_name(:hourly)}" : "../../hourly/#{current_snapshot_name(:hourly)}"
+      end
+      
+    end
+  end
+  
+  class MissingSettingError < StandardError; end
+  class MissingFolderError < StandardError; end
+end

data/lib/paparazzi/version.rb

@@ -0,0 +1,3 @@
+module Paparazzi
+  VERSION = '0.1.0'
+end

data/paparazzi.gemspec

@@ -0,0 +1,18 @@
+require File.expand_path('../lib/paparazzi/version', __FILE__)
+
+Gem::Specification.new do |s|
+  s.name = "paparazzi"
+  s.version = Paparazzi::VERSION
+  s.platform = Gem::Platform::RUBY
+  s.authors = ["Jonathan S. Garvin"]
+  s.email = ["jon@5valleys.com"]
+  s.homepage = "https://github.com/jsgarvin/paparazzi"
+  s.summary = %q{Rsync backup library with incremental snapshots.}
+  s.description = %q{Rsync backup library that takes incremental hourly, daily, etc., snapshots.}
+
+  s.add_dependency('admit_one', '>= 0.2.2')
+    
+  s.files = `git ls-files`.split("\n")
+  s.test_files = `git ls-files -- test/*`.split("\n")
+  s.require_paths = ["lib"]
+end

data/test/source/test.txt

@@ -0,0 +1 @@
+This is a test, this is only a test.

data/test/unit/camera_test.rb

@@ -0,0 +1,111 @@
+require File.expand_path('../../../lib/paparazzi',  __FILE__)
+require 'test/unit'
+require 'fileutils'
+
+class CameraTest < Test::Unit::TestCase
+  
+  def setup
+    Paparazzi::Camera::FREQUENCIES.each do |frequency|
+      FileUtils.rm_rf("#{destination}/#{frequency}")
+    end
+    FileUtils.rm_rf("#{destination}/.paparazzi.yml")
+  end
+  
+  def test_should_raise_exception_on_missing_required_settings
+    assert_raise(Paparazzi::MissingSettingError) { Paparazzi::Camera.trigger({}) }
+  end
+  
+  def test_should_store_valid_settings_in_attr_accessors
+    Paparazzi::Camera.trigger(default_test_settings)
+    [:source,:destination,:rsync_flags].each do |setting_name|
+      assert_equal(default_test_settings[setting_name],Paparazzi::Camera.send(setting_name),setting_name)
+    end
+  end
+  
+  def test_should_raise_exception_on_missing_source_or_destination_folder
+    my_test_settings = default_test_settings.merge(:source => File.expand_path('../../missing_folder',  __FILE__))
+    assert_raise(Paparazzi::MissingFolderError) { Paparazzi::Camera.trigger(my_test_settings) }
+    
+    my_test_settings = default_test_settings.merge(:destination => File.expand_path('../../missing_folder',  __FILE__))
+    assert_raise(Paparazzi::MissingFolderError) { Paparazzi::Camera.trigger(my_test_settings) }
+  end
+  
+  def test_should_create_frequency_folders_on_initialization
+    Paparazzi::Camera::FREQUENCIES.each do |frequency|
+      assert(!File.exists?("#{destination}/#{frequency}"))
+    end
+    Paparazzi::Camera.trigger(default_test_settings)
+    Paparazzi::Camera::FREQUENCIES.each do |frequency|
+      assert(File.exists?("#{destination}/#{frequency}"))
+    end
+  end
+  
+  def test_should_make_all_first_snapshots_of_source
+    Paparazzi::Camera.trigger(default_test_settings)
+    Paparazzi::Camera::FREQUENCIES.each do |frequency|
+      assert(Dir["#{destination}/#{frequency}/*"].size > 0,"#{destination}/#{frequency}/ is empty")
+      file = File.open(%Q{#{Dir["#{destination}/#{frequency}/*"].first}/test.txt})
+      assert_equal('This is a test, this is only a test.',file.gets)
+    end
+  end
+  
+  def test_should_write_last_successful_snapshot_to_cached_values_file
+    Paparazzi::Camera.trigger(default_test_settings)
+    assert(YAML.load_file("#{destination}/.paparazzi.yml")[:last_successful_snapshot].match(/\d{4}-\d{2}-\d{2}\.\d{2}/))
+  end
+  
+  def test_purges_out_expired_snapshots
+    Dir.mkdir("#{destination}/weekly")
+    (1..5).each do |x|
+      Dir.mkdir("#{destination}/weekly/#{x}")
+      sleep 1;
+    end
+    assert_equal(5,Dir["#{destination}/weekly/*"].size)
+    previous_folder_contents = Dir["#{destination}/weekly/*"]
+    Paparazzi::Camera.trigger(default_test_settings)
+    assert_equal(5,Dir["#{destination}/weekly/*"].size)
+    assert_not_equal(previous_folder_contents,Dir["#{destination}/weekly/*"])
+  end
+  
+  def test_gracefully_recovers_if_last_hourly_snapshot_ended_prematurely
+    Paparazzi::Camera.instance_variable_set('@start_time',Time.now - 7200)
+    Paparazzi::Camera.trigger(default_test_settings)
+    successful_snapshot_name = Paparazzi::Camera.send(:current_snapshot_name,:hourly)
+    
+    Paparazzi::Camera.instance_variable_set('@start_time',Time.now - 3600)
+    Paparazzi::Camera.trigger(default_test_settings)
+    failed_snapshot_name = Paparazzi::Camera.send(:current_snapshot_name,:hourly)
+    
+    Paparazzi::Camera.send(:last_successful_hourly_snapshot=,successful_snapshot_name)
+    assert_equal(2,Dir["#{destination}/hourly/*"].size)
+    assert_equal(successful_snapshot_name,YAML.load_file("#{destination}/.paparazzi.yml")[:last_successful_snapshot])
+    
+    Paparazzi::Camera.instance_variable_set('@start_time',Time.now)
+    Paparazzi::Camera.trigger(default_test_settings)
+    
+    assert_equal(2,Dir["#{destination}/hourly/*"].size)
+    assert(Dir["#{destination}/hourly/*"].include?("#{destination}/hourly/#{successful_snapshot_name}"))
+    assert(!Dir["#{destination}/hourly/*"].include?("#{destination}/hourly/#{failed_snapshot_name}"))
+  end
+  
+  def test_correct_hard_links_are_created_to_multiple_snapshots_of_same_file
+    Paparazzi::Camera.trigger(default_test_settings)
+    assert_equal(5,File.stat("#{destination}/hourly/#{Paparazzi::Camera.send(:current_snapshot_name,:hourly)}/test.txt").nlink)   
+  end
+  
+  #######
+  private
+  #######
+  
+  def destination
+    @destination ||= File.expand_path('../../destination',  __FILE__)
+  end
+  
+  def default_test_settings
+    @default_test_settings ||= {
+      :source => "#{File.expand_path('../../source',  __FILE__)}/",
+      :destination => destination,
+      :rsync_flags => '-L'
+    }
+  end
+end

metadata

@@ -0,0 +1,91 @@
+--- !ruby/object:Gem::Specification 
+name: paparazzi
+version: !ruby/object:Gem::Version 
+  prerelease: false
+  segments: 
+  - 0
+  - 1
+  - 0
+  version: 0.1.0
+platform: ruby
+authors: 
+- Jonathan S. Garvin
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2011-05-22 00:00:00 -06:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: admit_one
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 0
+        - 2
+        - 2
+        version: 0.2.2
+  type: :runtime
+  version_requirements: *id001
+description: Rsync backup library that takes incremental hourly, daily, etc., snapshots.
+email: 
+- jon@5valleys.com
+executables: []
+
+extensions: []
+
+extra_rdoc_files: []
+
+files: 
+- .gitignore
+- LICENSE
+- README.md
+- Rakefile
+- lib/paparazzi.rb
+- lib/paparazzi/camera.rb
+- lib/paparazzi/version.rb
+- paparazzi.gemspec
+- test/destination/.gitignore
+- test/source/test.txt
+- test/unit/camera_test.rb
+has_rdoc: true
+homepage: https://github.com/jsgarvin/paparazzi
+licenses: []
+
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.3.7
+signing_key: 
+specification_version: 3
+summary: Rsync backup library with incremental snapshots.
+test_files: 
+- test/destination/.gitignore
+- test/source/test.txt
+- test/unit/camera_test.rb