paparazzi 0.1.0
Sign up to get free protection for your applications and to get access to all the features.
- data/.gitignore +3 -0
- data/LICENSE +22 -0
- data/README.md +50 -0
- data/Rakefile +10 -0
- data/lib/paparazzi.rb +1 -0
- data/lib/paparazzi/camera.rb +124 -0
- data/lib/paparazzi/version.rb +3 -0
- data/paparazzi.gemspec +18 -0
- data/test/source/test.txt +1 -0
- data/test/unit/camera_test.rb +111 -0
- metadata +91 -0
data/.gitignore
ADDED
data/LICENSE
ADDED
@@ -0,0 +1,22 @@
|
|
1
|
+
Copyright (c) 2011 Jonathan S. Garvin (http://www.5valleys.com)
|
2
|
+
|
3
|
+
Permission is hereby granted, free of charge, to any person
|
4
|
+
obtaining a copy of this software and associated documentation
|
5
|
+
files (the "Software"), to deal in the Software without
|
6
|
+
restriction, including without limitation the rights to use,
|
7
|
+
copy, modify, merge, publish, distribute, sublicense, and/or sell
|
8
|
+
copies of the Software, and to permit persons to whom the
|
9
|
+
Software is furnished to do so, subject to the following
|
10
|
+
conditions:
|
11
|
+
|
12
|
+
The above copyright notice and this permission notice shall be
|
13
|
+
included in all copies or substantial portions of the Software.
|
14
|
+
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
16
|
+
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
|
17
|
+
OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
18
|
+
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
|
19
|
+
HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
|
20
|
+
WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
21
|
+
FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
|
22
|
+
OTHER DEALINGS IN THE SOFTWARE.
|
data/README.md
ADDED
@@ -0,0 +1,50 @@
|
|
1
|
+
Paparazzi
|
2
|
+
=========
|
3
|
+
|
4
|
+
Paparazzi is a Ruby gem for making incremental snapshot rsync backups of a directory, maintaining
|
5
|
+
hourly, daily, weekly, monthy, and yearly snapshots of directory state and files without consuming
|
6
|
+
much more drive space than a single copy would (depending on how frequently existing files change).
|
7
|
+
|
8
|
+
Only changed or new files are copied to the new snapshot. Hard links are created in the new snapshot
|
9
|
+
to previously existing unchanged files, allowing for frequent backups to be made of very large
|
10
|
+
directories (see 'How it works' below). Older snapshots maintain versions of files as they existed
|
11
|
+
at the time that snapshot was made.
|
12
|
+
|
13
|
+
Paparazzi automatically purges out old hourly, daily, weekly, and monthly snapshots. Yearly
|
14
|
+
snapshots are not automatically purged and need to be removed manually when necessary.
|
15
|
+
|
16
|
+
Installation
|
17
|
+
------------
|
18
|
+
|
19
|
+
gem install paparazzi
|
20
|
+
|
21
|
+
|
22
|
+
Usage
|
23
|
+
-----
|
24
|
+
|
25
|
+
Create a ruby script that you'll run hourly from a cron.
|
26
|
+
|
27
|
+
require 'rubygems' #unless you use another gem package manager
|
28
|
+
require 'paparazzi'
|
29
|
+
|
30
|
+
settings = {
|
31
|
+
:source => '/full/path/to/source/directory/', # note the trailing '/'
|
32
|
+
:destination => '/mnt/external_drive/backup_folder',
|
33
|
+
:rsync_flags => '-L' # see 'man rsync' for available options.
|
34
|
+
}
|
35
|
+
|
36
|
+
Paparazzi::Camera.trigger(settings)
|
37
|
+
|
38
|
+
|
39
|
+
How it works.
|
40
|
+
-------------
|
41
|
+
|
42
|
+
Paparazzi uses rsync's ability to make hard links to files that haven't changed from previous
|
43
|
+
backups. So, even though multiple incremental versions of the entire directory are kept, only a single
|
44
|
+
copy of unique files are kept, with multiple hard links in separate snapshots pointing to the same
|
45
|
+
file.
|
46
|
+
|
47
|
+
This gem owes it's existance to Mike Rubel's excellent write-up
|
48
|
+
[Easy Automated Snapshot-Style Backups with Linux and Rsync](http://www.mikerubel.org/computers/rsync_snapshots/).
|
49
|
+
If you're not sure what "hard links" are or are confused how multiple snapshot versions of a
|
50
|
+
directory can be made without taking up much more space than a single copy, then read his post.
|
data/Rakefile
ADDED
data/lib/paparazzi.rb
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
require File.expand_path('../paparazzi/camera', __FILE__)
|
@@ -0,0 +1,124 @@
|
|
1
|
+
require 'admit_one'
|
2
|
+
require 'yaml'
|
3
|
+
require 'fileutils'
|
4
|
+
|
5
|
+
module Paparazzi
|
6
|
+
class Camera
|
7
|
+
FREQUENCIES = [:hourly,:daily,:weekly,:monthly,:yearly]
|
8
|
+
REQUIRED_SETTINGS = [:source,:destination]
|
9
|
+
|
10
|
+
class << self
|
11
|
+
attr_accessor :source, :destination, :rsync_flags
|
12
|
+
|
13
|
+
def trigger(settings = {})
|
14
|
+
validate_and_cache_settings(settings)
|
15
|
+
AdmitOne::LockFile.new(:paparazzi) do
|
16
|
+
initialize
|
17
|
+
purge_old_snapshots
|
18
|
+
make_snapshots
|
19
|
+
end
|
20
|
+
|
21
|
+
end
|
22
|
+
|
23
|
+
#######
|
24
|
+
private
|
25
|
+
#######
|
26
|
+
|
27
|
+
def validate_and_cache_settings(settings)
|
28
|
+
[:source,:destination,:rsync_flags].each do |setting_name|
|
29
|
+
if REQUIRED_SETTINGS.include?(setting_name) and settings[setting_name].nil?
|
30
|
+
raise MissingSettingError, "#{setting_name} is required"
|
31
|
+
else
|
32
|
+
self.send("#{setting_name}=",settings[setting_name])
|
33
|
+
end
|
34
|
+
end
|
35
|
+
|
36
|
+
raise MissingFolderError, source unless File.exist?(source) and File.directory?(source)
|
37
|
+
raise MissingFolderError, destination unless File.exist?(destination) and File.directory?(destination)
|
38
|
+
end
|
39
|
+
|
40
|
+
def initialize
|
41
|
+
@previous_snapshot_name = {}
|
42
|
+
FREQUENCIES.each do |frequency|
|
43
|
+
Dir.mkdir(destination(frequency)) unless File.exists?(destination(frequency)) && File.directory?(destination(frequency))
|
44
|
+
|
45
|
+
full_path = Dir[destination(frequency)+'/*'].sort{|a,b| File.ctime(b) <=> File.ctime(a) }.first
|
46
|
+
@previous_snapshot_name[frequency] = full_path ? File.basename(full_path) : ''
|
47
|
+
end
|
48
|
+
|
49
|
+
if @previous_snapshot_name[:hourly] != last_successful_hourly_snapshot and !last_successful_hourly_snapshot.nil? and File.exists?(destination(:hourly) + '/' + last_successful_hourly_snapshot)
|
50
|
+
File.rename(previous_snapshot(:hourly), current_snapshot(:hourly))
|
51
|
+
@previous_snapshot_name[:hourly] = last_successful_hourly_snapshot
|
52
|
+
end
|
53
|
+
end
|
54
|
+
|
55
|
+
def destination(frequency = nil)
|
56
|
+
frequency.nil? ? @destination : "#{@destination}/#{frequency}"
|
57
|
+
end
|
58
|
+
|
59
|
+
def last_successful_hourly_snapshot
|
60
|
+
@last_successful_hourly_snapshot ||= File.exists?("#{destination}/.paparazzi.yml") ? YAML.load_file("#{destination}/.paparazzi.yml")[:last_successful_snapshot] : nil
|
61
|
+
end
|
62
|
+
|
63
|
+
def last_successful_hourly_snapshot=(string)
|
64
|
+
cached_data ||= File.exists?("#{destination}/.paparazzi.yml") ? YAML.load_file("#{destination}/.paparazzi.yml") : {}
|
65
|
+
cached_data[:last_successful_snapshot] = string
|
66
|
+
File.open("#{destination}/.paparazzi.yml", 'w') {|file| file.write(cached_data.to_yaml) }
|
67
|
+
end
|
68
|
+
|
69
|
+
def purge_old_snapshots
|
70
|
+
keepers = {:hourly => 24, :daily => 7, :weekly => 5, :monthly => 12}
|
71
|
+
keepers.keys.each do |frequency|
|
72
|
+
while Dir[destination(frequency)+'/*'].size > keepers[frequency]-1
|
73
|
+
full_path = Dir[destination(frequency)+'/*'].sort{|a,b| File.ctime(a) <=> File.ctime(b) }.first
|
74
|
+
FileUtils.rm_rf(full_path)
|
75
|
+
end
|
76
|
+
end
|
77
|
+
end
|
78
|
+
|
79
|
+
def make_snapshots
|
80
|
+
FREQUENCIES.each do |frequency|
|
81
|
+
Dir.mkdir(current_snapshot(frequency)) unless File.exists?(current_snapshot(frequency))
|
82
|
+
if frequency == :hourly and previous_snapshot_name(frequency) == ''
|
83
|
+
system 'rsync', *(['-aq', '--delete'] + [rsync_flags] + [source, current_snapshot(frequency)])
|
84
|
+
self.last_successful_hourly_snapshot = current_snapshot_name(:hourly)
|
85
|
+
elsif previous_snapshot_name(frequency) != current_snapshot_name(frequency)
|
86
|
+
system 'rsync', *(['-aq', '--delete', "--link-dest=#{link_destination(frequency)}"] + [rsync_flags] + [source, current_snapshot(frequency)])
|
87
|
+
self.last_successful_hourly_snapshot = current_snapshot_name(:hourly)
|
88
|
+
end
|
89
|
+
end
|
90
|
+
end
|
91
|
+
|
92
|
+
def current_snapshot(frequency)
|
93
|
+
destination(frequency) + '/' + current_snapshot_name(frequency)
|
94
|
+
end
|
95
|
+
|
96
|
+
def current_snapshot_name(frequency)
|
97
|
+
@start_time ||= Time.now #lock in time so that all results stay consistent over long runs
|
98
|
+
case frequency
|
99
|
+
when :hourly then @start_time.strftime('%Y-%m-%d.%H')
|
100
|
+
when :daily then @start_time.strftime('%Y-%m-%d')
|
101
|
+
when :weekly then sprintf("%04d-%02d-week-%02d", @start_time.year, @start_time.month, (@start_time.day/7))
|
102
|
+
when :monthly then @start_time.strftime("%Y-%m")
|
103
|
+
when :yearly then @start_time.strftime("%Y")
|
104
|
+
end
|
105
|
+
end
|
106
|
+
|
107
|
+
def previous_snapshot(frequency)
|
108
|
+
destination(frequency) + '/' + previous_snapshot_name(frequency)
|
109
|
+
end
|
110
|
+
|
111
|
+
def previous_snapshot_name(frequency)
|
112
|
+
@previous_snapshot_name[frequency]
|
113
|
+
end
|
114
|
+
|
115
|
+
def link_destination(frequency)
|
116
|
+
frequency == :hourly ? "../#{previous_snapshot_name(:hourly)}" : "../../hourly/#{current_snapshot_name(:hourly)}"
|
117
|
+
end
|
118
|
+
|
119
|
+
end
|
120
|
+
end
|
121
|
+
|
122
|
+
class MissingSettingError < StandardError; end
|
123
|
+
class MissingFolderError < StandardError; end
|
124
|
+
end
|
data/paparazzi.gemspec
ADDED
@@ -0,0 +1,18 @@
|
|
1
|
+
require File.expand_path('../lib/paparazzi/version', __FILE__)
|
2
|
+
|
3
|
+
Gem::Specification.new do |s|
|
4
|
+
s.name = "paparazzi"
|
5
|
+
s.version = Paparazzi::VERSION
|
6
|
+
s.platform = Gem::Platform::RUBY
|
7
|
+
s.authors = ["Jonathan S. Garvin"]
|
8
|
+
s.email = ["jon@5valleys.com"]
|
9
|
+
s.homepage = "https://github.com/jsgarvin/paparazzi"
|
10
|
+
s.summary = %q{Rsync backup library with incremental snapshots.}
|
11
|
+
s.description = %q{Rsync backup library that takes incremental hourly, daily, etc., snapshots.}
|
12
|
+
|
13
|
+
s.add_dependency('admit_one', '>= 0.2.2')
|
14
|
+
|
15
|
+
s.files = `git ls-files`.split("\n")
|
16
|
+
s.test_files = `git ls-files -- test/*`.split("\n")
|
17
|
+
s.require_paths = ["lib"]
|
18
|
+
end
|
@@ -0,0 +1 @@
|
|
1
|
+
This is a test, this is only a test.
|
@@ -0,0 +1,111 @@
|
|
1
|
+
require File.expand_path('../../../lib/paparazzi', __FILE__)
|
2
|
+
require 'test/unit'
|
3
|
+
require 'fileutils'
|
4
|
+
|
5
|
+
class CameraTest < Test::Unit::TestCase
|
6
|
+
|
7
|
+
def setup
|
8
|
+
Paparazzi::Camera::FREQUENCIES.each do |frequency|
|
9
|
+
FileUtils.rm_rf("#{destination}/#{frequency}")
|
10
|
+
end
|
11
|
+
FileUtils.rm_rf("#{destination}/.paparazzi.yml")
|
12
|
+
end
|
13
|
+
|
14
|
+
def test_should_raise_exception_on_missing_required_settings
|
15
|
+
assert_raise(Paparazzi::MissingSettingError) { Paparazzi::Camera.trigger({}) }
|
16
|
+
end
|
17
|
+
|
18
|
+
def test_should_store_valid_settings_in_attr_accessors
|
19
|
+
Paparazzi::Camera.trigger(default_test_settings)
|
20
|
+
[:source,:destination,:rsync_flags].each do |setting_name|
|
21
|
+
assert_equal(default_test_settings[setting_name],Paparazzi::Camera.send(setting_name),setting_name)
|
22
|
+
end
|
23
|
+
end
|
24
|
+
|
25
|
+
def test_should_raise_exception_on_missing_source_or_destination_folder
|
26
|
+
my_test_settings = default_test_settings.merge(:source => File.expand_path('../../missing_folder', __FILE__))
|
27
|
+
assert_raise(Paparazzi::MissingFolderError) { Paparazzi::Camera.trigger(my_test_settings) }
|
28
|
+
|
29
|
+
my_test_settings = default_test_settings.merge(:destination => File.expand_path('../../missing_folder', __FILE__))
|
30
|
+
assert_raise(Paparazzi::MissingFolderError) { Paparazzi::Camera.trigger(my_test_settings) }
|
31
|
+
end
|
32
|
+
|
33
|
+
def test_should_create_frequency_folders_on_initialization
|
34
|
+
Paparazzi::Camera::FREQUENCIES.each do |frequency|
|
35
|
+
assert(!File.exists?("#{destination}/#{frequency}"))
|
36
|
+
end
|
37
|
+
Paparazzi::Camera.trigger(default_test_settings)
|
38
|
+
Paparazzi::Camera::FREQUENCIES.each do |frequency|
|
39
|
+
assert(File.exists?("#{destination}/#{frequency}"))
|
40
|
+
end
|
41
|
+
end
|
42
|
+
|
43
|
+
def test_should_make_all_first_snapshots_of_source
|
44
|
+
Paparazzi::Camera.trigger(default_test_settings)
|
45
|
+
Paparazzi::Camera::FREQUENCIES.each do |frequency|
|
46
|
+
assert(Dir["#{destination}/#{frequency}/*"].size > 0,"#{destination}/#{frequency}/ is empty")
|
47
|
+
file = File.open(%Q{#{Dir["#{destination}/#{frequency}/*"].first}/test.txt})
|
48
|
+
assert_equal('This is a test, this is only a test.',file.gets)
|
49
|
+
end
|
50
|
+
end
|
51
|
+
|
52
|
+
def test_should_write_last_successful_snapshot_to_cached_values_file
|
53
|
+
Paparazzi::Camera.trigger(default_test_settings)
|
54
|
+
assert(YAML.load_file("#{destination}/.paparazzi.yml")[:last_successful_snapshot].match(/\d{4}-\d{2}-\d{2}\.\d{2}/))
|
55
|
+
end
|
56
|
+
|
57
|
+
def test_purges_out_expired_snapshots
|
58
|
+
Dir.mkdir("#{destination}/weekly")
|
59
|
+
(1..5).each do |x|
|
60
|
+
Dir.mkdir("#{destination}/weekly/#{x}")
|
61
|
+
sleep 1;
|
62
|
+
end
|
63
|
+
assert_equal(5,Dir["#{destination}/weekly/*"].size)
|
64
|
+
previous_folder_contents = Dir["#{destination}/weekly/*"]
|
65
|
+
Paparazzi::Camera.trigger(default_test_settings)
|
66
|
+
assert_equal(5,Dir["#{destination}/weekly/*"].size)
|
67
|
+
assert_not_equal(previous_folder_contents,Dir["#{destination}/weekly/*"])
|
68
|
+
end
|
69
|
+
|
70
|
+
def test_gracefully_recovers_if_last_hourly_snapshot_ended_prematurely
|
71
|
+
Paparazzi::Camera.instance_variable_set('@start_time',Time.now - 7200)
|
72
|
+
Paparazzi::Camera.trigger(default_test_settings)
|
73
|
+
successful_snapshot_name = Paparazzi::Camera.send(:current_snapshot_name,:hourly)
|
74
|
+
|
75
|
+
Paparazzi::Camera.instance_variable_set('@start_time',Time.now - 3600)
|
76
|
+
Paparazzi::Camera.trigger(default_test_settings)
|
77
|
+
failed_snapshot_name = Paparazzi::Camera.send(:current_snapshot_name,:hourly)
|
78
|
+
|
79
|
+
Paparazzi::Camera.send(:last_successful_hourly_snapshot=,successful_snapshot_name)
|
80
|
+
assert_equal(2,Dir["#{destination}/hourly/*"].size)
|
81
|
+
assert_equal(successful_snapshot_name,YAML.load_file("#{destination}/.paparazzi.yml")[:last_successful_snapshot])
|
82
|
+
|
83
|
+
Paparazzi::Camera.instance_variable_set('@start_time',Time.now)
|
84
|
+
Paparazzi::Camera.trigger(default_test_settings)
|
85
|
+
|
86
|
+
assert_equal(2,Dir["#{destination}/hourly/*"].size)
|
87
|
+
assert(Dir["#{destination}/hourly/*"].include?("#{destination}/hourly/#{successful_snapshot_name}"))
|
88
|
+
assert(!Dir["#{destination}/hourly/*"].include?("#{destination}/hourly/#{failed_snapshot_name}"))
|
89
|
+
end
|
90
|
+
|
91
|
+
def test_correct_hard_links_are_created_to_multiple_snapshots_of_same_file
|
92
|
+
Paparazzi::Camera.trigger(default_test_settings)
|
93
|
+
assert_equal(5,File.stat("#{destination}/hourly/#{Paparazzi::Camera.send(:current_snapshot_name,:hourly)}/test.txt").nlink)
|
94
|
+
end
|
95
|
+
|
96
|
+
#######
|
97
|
+
private
|
98
|
+
#######
|
99
|
+
|
100
|
+
def destination
|
101
|
+
@destination ||= File.expand_path('../../destination', __FILE__)
|
102
|
+
end
|
103
|
+
|
104
|
+
def default_test_settings
|
105
|
+
@default_test_settings ||= {
|
106
|
+
:source => "#{File.expand_path('../../source', __FILE__)}/",
|
107
|
+
:destination => destination,
|
108
|
+
:rsync_flags => '-L'
|
109
|
+
}
|
110
|
+
end
|
111
|
+
end
|
metadata
ADDED
@@ -0,0 +1,91 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: paparazzi
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
prerelease: false
|
5
|
+
segments:
|
6
|
+
- 0
|
7
|
+
- 1
|
8
|
+
- 0
|
9
|
+
version: 0.1.0
|
10
|
+
platform: ruby
|
11
|
+
authors:
|
12
|
+
- Jonathan S. Garvin
|
13
|
+
autorequire:
|
14
|
+
bindir: bin
|
15
|
+
cert_chain: []
|
16
|
+
|
17
|
+
date: 2011-05-22 00:00:00 -06:00
|
18
|
+
default_executable:
|
19
|
+
dependencies:
|
20
|
+
- !ruby/object:Gem::Dependency
|
21
|
+
name: admit_one
|
22
|
+
prerelease: false
|
23
|
+
requirement: &id001 !ruby/object:Gem::Requirement
|
24
|
+
none: false
|
25
|
+
requirements:
|
26
|
+
- - ">="
|
27
|
+
- !ruby/object:Gem::Version
|
28
|
+
segments:
|
29
|
+
- 0
|
30
|
+
- 2
|
31
|
+
- 2
|
32
|
+
version: 0.2.2
|
33
|
+
type: :runtime
|
34
|
+
version_requirements: *id001
|
35
|
+
description: Rsync backup library that takes incremental hourly, daily, etc., snapshots.
|
36
|
+
email:
|
37
|
+
- jon@5valleys.com
|
38
|
+
executables: []
|
39
|
+
|
40
|
+
extensions: []
|
41
|
+
|
42
|
+
extra_rdoc_files: []
|
43
|
+
|
44
|
+
files:
|
45
|
+
- .gitignore
|
46
|
+
- LICENSE
|
47
|
+
- README.md
|
48
|
+
- Rakefile
|
49
|
+
- lib/paparazzi.rb
|
50
|
+
- lib/paparazzi/camera.rb
|
51
|
+
- lib/paparazzi/version.rb
|
52
|
+
- paparazzi.gemspec
|
53
|
+
- test/destination/.gitignore
|
54
|
+
- test/source/test.txt
|
55
|
+
- test/unit/camera_test.rb
|
56
|
+
has_rdoc: true
|
57
|
+
homepage: https://github.com/jsgarvin/paparazzi
|
58
|
+
licenses: []
|
59
|
+
|
60
|
+
post_install_message:
|
61
|
+
rdoc_options: []
|
62
|
+
|
63
|
+
require_paths:
|
64
|
+
- lib
|
65
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
66
|
+
none: false
|
67
|
+
requirements:
|
68
|
+
- - ">="
|
69
|
+
- !ruby/object:Gem::Version
|
70
|
+
segments:
|
71
|
+
- 0
|
72
|
+
version: "0"
|
73
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
74
|
+
none: false
|
75
|
+
requirements:
|
76
|
+
- - ">="
|
77
|
+
- !ruby/object:Gem::Version
|
78
|
+
segments:
|
79
|
+
- 0
|
80
|
+
version: "0"
|
81
|
+
requirements: []
|
82
|
+
|
83
|
+
rubyforge_project:
|
84
|
+
rubygems_version: 1.3.7
|
85
|
+
signing_key:
|
86
|
+
specification_version: 3
|
87
|
+
summary: Rsync backup library with incremental snapshots.
|
88
|
+
test_files:
|
89
|
+
- test/destination/.gitignore
|
90
|
+
- test/source/test.txt
|
91
|
+
- test/unit/camera_test.rb
|