backup 3.0.20 → 3.0.21
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/Gemfile +1 -5
- data/Gemfile.lock +46 -50
- data/README.md +54 -27
- data/lib/backup.rb +16 -39
- data/lib/backup/archive.rb +42 -18
- data/lib/backup/cleaner.rb +110 -25
- data/lib/backup/cli/helpers.rb +17 -32
- data/lib/backup/cli/utility.rb +46 -107
- data/lib/backup/compressor/base.rb +14 -2
- data/lib/backup/compressor/bzip2.rb +10 -24
- data/lib/backup/compressor/gzip.rb +10 -24
- data/lib/backup/compressor/lzma.rb +10 -23
- data/lib/backup/compressor/pbzip2.rb +12 -32
- data/lib/backup/config.rb +171 -0
- data/lib/backup/configuration/compressor/base.rb +1 -2
- data/lib/backup/configuration/compressor/pbzip2.rb +4 -4
- data/lib/backup/configuration/database/base.rb +2 -1
- data/lib/backup/configuration/database/mongodb.rb +8 -0
- data/lib/backup/configuration/database/mysql.rb +4 -0
- data/lib/backup/configuration/database/postgresql.rb +4 -0
- data/lib/backup/configuration/database/redis.rb +4 -0
- data/lib/backup/configuration/database/riak.rb +5 -1
- data/lib/backup/configuration/encryptor/base.rb +1 -2
- data/lib/backup/configuration/encryptor/open_ssl.rb +1 -1
- data/lib/backup/configuration/helpers.rb +7 -2
- data/lib/backup/configuration/notifier/base.rb +4 -28
- data/lib/backup/configuration/storage/base.rb +1 -1
- data/lib/backup/configuration/storage/dropbox.rb +14 -4
- data/lib/backup/configuration/syncer/base.rb +10 -0
- data/lib/backup/configuration/syncer/rsync/base.rb +28 -0
- data/lib/backup/configuration/syncer/rsync/local.rb +11 -0
- data/lib/backup/configuration/syncer/rsync/pull.rb +11 -0
- data/lib/backup/configuration/syncer/rsync/push.rb +31 -0
- data/lib/backup/configuration/syncer/s3.rb +0 -4
- data/lib/backup/database/base.rb +25 -7
- data/lib/backup/database/mongodb.rb +112 -75
- data/lib/backup/database/mysql.rb +54 -29
- data/lib/backup/database/postgresql.rb +60 -42
- data/lib/backup/database/redis.rb +61 -39
- data/lib/backup/database/riak.rb +35 -11
- data/lib/backup/dependency.rb +4 -5
- data/lib/backup/encryptor/base.rb +13 -1
- data/lib/backup/encryptor/gpg.rb +39 -39
- data/lib/backup/encryptor/open_ssl.rb +28 -38
- data/lib/backup/logger.rb +20 -11
- data/lib/backup/model.rb +206 -163
- data/lib/backup/notifier/base.rb +27 -25
- data/lib/backup/notifier/campfire.rb +7 -13
- data/lib/backup/notifier/hipchat.rb +28 -28
- data/lib/backup/notifier/mail.rb +24 -26
- data/lib/backup/notifier/presently.rb +10 -18
- data/lib/backup/notifier/prowl.rb +9 -17
- data/lib/backup/notifier/twitter.rb +11 -18
- data/lib/backup/package.rb +47 -0
- data/lib/backup/packager.rb +81 -16
- data/lib/backup/splitter.rb +48 -35
- data/lib/backup/storage/base.rb +44 -172
- data/lib/backup/storage/cloudfiles.rb +31 -46
- data/lib/backup/storage/cycler.rb +117 -0
- data/lib/backup/storage/dropbox.rb +92 -76
- data/lib/backup/storage/ftp.rb +30 -40
- data/lib/backup/storage/local.rb +44 -45
- data/lib/backup/storage/ninefold.rb +55 -49
- data/lib/backup/storage/rsync.rb +49 -56
- data/lib/backup/storage/s3.rb +33 -44
- data/lib/backup/storage/scp.rb +21 -48
- data/lib/backup/storage/sftp.rb +26 -40
- data/lib/backup/syncer/base.rb +7 -0
- data/lib/backup/syncer/rsync/base.rb +78 -0
- data/lib/backup/syncer/rsync/local.rb +53 -0
- data/lib/backup/syncer/rsync/pull.rb +38 -0
- data/lib/backup/syncer/rsync/push.rb +113 -0
- data/lib/backup/syncer/s3.rb +42 -32
- data/lib/backup/version.rb +1 -1
- data/spec/archive_spec.rb +235 -69
- data/spec/cleaner_spec.rb +304 -0
- data/spec/cli/helpers_spec.rb +142 -1
- data/spec/cli/utility_spec.rb +338 -13
- data/spec/compressor/base_spec.rb +31 -0
- data/spec/compressor/bzip2_spec.rb +60 -35
- data/spec/compressor/gzip_spec.rb +60 -35
- data/spec/compressor/lzma_spec.rb +60 -35
- data/spec/compressor/pbzip2_spec.rb +98 -37
- data/spec/config_spec.rb +321 -0
- data/spec/configuration/base_spec.rb +4 -4
- data/spec/configuration/compressor/bzip2_spec.rb +1 -0
- data/spec/configuration/compressor/gzip_spec.rb +1 -0
- data/spec/configuration/compressor/lzma_spec.rb +1 -0
- data/spec/configuration/compressor/pbzip2_spec.rb +32 -0
- data/spec/configuration/database/base_spec.rb +2 -1
- data/spec/configuration/database/mongodb_spec.rb +26 -16
- data/spec/configuration/database/mysql_spec.rb +4 -0
- data/spec/configuration/database/postgresql_spec.rb +4 -0
- data/spec/configuration/database/redis_spec.rb +4 -0
- data/spec/configuration/database/riak_spec.rb +4 -0
- data/spec/configuration/encryptor/gpg_spec.rb +1 -0
- data/spec/configuration/encryptor/open_ssl_spec.rb +1 -0
- data/spec/configuration/notifier/base_spec.rb +32 -0
- data/spec/configuration/notifier/campfire_spec.rb +1 -0
- data/spec/configuration/notifier/hipchat_spec.rb +1 -0
- data/spec/configuration/notifier/mail_spec.rb +1 -0
- data/spec/configuration/notifier/presently_spec.rb +1 -0
- data/spec/configuration/notifier/prowl_spec.rb +1 -0
- data/spec/configuration/notifier/twitter_spec.rb +1 -0
- data/spec/configuration/storage/cloudfiles_spec.rb +1 -0
- data/spec/configuration/storage/dropbox_spec.rb +4 -3
- data/spec/configuration/storage/ftp_spec.rb +1 -0
- data/spec/configuration/storage/local_spec.rb +1 -0
- data/spec/configuration/storage/ninefold_spec.rb +1 -0
- data/spec/configuration/storage/rsync_spec.rb +3 -1
- data/spec/configuration/storage/s3_spec.rb +1 -0
- data/spec/configuration/storage/scp_spec.rb +1 -0
- data/spec/configuration/storage/sftp_spec.rb +1 -0
- data/spec/configuration/syncer/rsync/base_spec.rb +33 -0
- data/spec/configuration/syncer/rsync/local_spec.rb +10 -0
- data/spec/configuration/syncer/rsync/pull_spec.rb +10 -0
- data/spec/configuration/syncer/{rsync_spec.rb → rsync/push_spec.rb} +12 -15
- data/spec/configuration/syncer/s3_spec.rb +2 -3
- data/spec/database/base_spec.rb +35 -20
- data/spec/database/mongodb_spec.rb +298 -119
- data/spec/database/mysql_spec.rb +147 -72
- data/spec/database/postgresql_spec.rb +155 -100
- data/spec/database/redis_spec.rb +200 -97
- data/spec/database/riak_spec.rb +82 -24
- data/spec/dependency_spec.rb +49 -0
- data/spec/encryptor/base_spec.rb +30 -0
- data/spec/encryptor/gpg_spec.rb +105 -28
- data/spec/encryptor/open_ssl_spec.rb +85 -114
- data/spec/logger_spec.rb +74 -8
- data/spec/model_spec.rb +528 -220
- data/spec/notifier/base_spec.rb +89 -0
- data/spec/notifier/campfire_spec.rb +147 -119
- data/spec/notifier/hipchat_spec.rb +140 -145
- data/spec/notifier/mail_spec.rb +190 -248
- data/spec/notifier/presently_spec.rb +147 -282
- data/spec/notifier/prowl_spec.rb +79 -111
- data/spec/notifier/twitter_spec.rb +87 -106
- data/spec/package_spec.rb +61 -0
- data/spec/packager_spec.rb +154 -0
- data/spec/spec_helper.rb +36 -13
- data/spec/splitter_spec.rb +90 -41
- data/spec/storage/base_spec.rb +95 -239
- data/spec/storage/cloudfiles_spec.rb +185 -75
- data/spec/storage/cycler_spec.rb +239 -0
- data/spec/storage/dropbox_spec.rb +318 -87
- data/spec/storage/ftp_spec.rb +165 -152
- data/spec/storage/local_spec.rb +206 -54
- data/spec/storage/ninefold_spec.rb +264 -128
- data/spec/storage/rsync_spec.rb +244 -163
- data/spec/storage/s3_spec.rb +175 -64
- data/spec/storage/scp_spec.rb +156 -150
- data/spec/storage/sftp_spec.rb +153 -135
- data/spec/syncer/base_spec.rb +22 -0
- data/spec/syncer/rsync/base_spec.rb +118 -0
- data/spec/syncer/rsync/local_spec.rb +121 -0
- data/spec/syncer/rsync/pull_spec.rb +90 -0
- data/spec/syncer/rsync/push_spec.rb +327 -0
- data/spec/syncer/s3_spec.rb +180 -91
- data/templates/cli/utility/config +1 -1
- data/templates/cli/utility/database/mongodb +4 -0
- data/templates/cli/utility/database/mysql +3 -0
- data/templates/cli/utility/database/postgresql +3 -0
- data/templates/cli/utility/database/redis +3 -0
- data/templates/cli/utility/database/riak +3 -0
- data/templates/cli/utility/storage/dropbox +4 -1
- data/templates/cli/utility/syncer/rsync_local +12 -0
- data/templates/cli/utility/syncer/{rsync → rsync_pull} +2 -2
- data/templates/cli/utility/syncer/rsync_push +17 -0
- data/templates/storage/dropbox/authorization_url.erb +1 -1
- metadata +42 -17
- data/lib/backup/configuration/syncer/rsync.rb +0 -45
- data/lib/backup/finder.rb +0 -87
- data/lib/backup/storage/object.rb +0 -47
- data/lib/backup/syncer/rsync.rb +0 -152
- data/spec/backup_spec.rb +0 -11
- data/spec/finder_spec.rb +0 -91
- data/spec/storage/object_spec.rb +0 -74
- data/spec/syncer/rsync_spec.rb +0 -195
data/Gemfile
CHANGED
|
@@ -6,12 +6,8 @@ source 'http://rubygems.org'
|
|
|
6
6
|
# Include gem dependencies from the gemspec for development purposes
|
|
7
7
|
gemspec
|
|
8
8
|
|
|
9
|
-
# Load Backup::Dependency
|
|
10
|
-
["cli/helpers", "dependency"].each do |library|
|
|
11
|
-
require File.expand_path("../lib/backup/#{library}", __FILE__)
|
|
12
|
-
end
|
|
13
|
-
|
|
14
9
|
# Dynamically define the dependencies specified in Backup::Dependency.all
|
|
10
|
+
require File.expand_path("../lib/backup/dependency", __FILE__)
|
|
15
11
|
Backup::Dependency.all.each do |name, gemspec|
|
|
16
12
|
gem(name, gemspec[:version])
|
|
17
13
|
end
|
data/Gemfile.lock
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
PATH
|
|
2
2
|
remote: .
|
|
3
3
|
specs:
|
|
4
|
-
backup (3.0.
|
|
4
|
+
backup (3.0.21)
|
|
5
5
|
POpen4 (~> 0.1.4)
|
|
6
6
|
thor (~> 0.14.6)
|
|
7
7
|
|
|
@@ -12,82 +12,80 @@ GEM
|
|
|
12
12
|
Platform (>= 0.4.0)
|
|
13
13
|
open4
|
|
14
14
|
Platform (0.4.0)
|
|
15
|
+
activesupport (3.1.3)
|
|
16
|
+
multi_json (~> 1.0)
|
|
15
17
|
addressable (2.2.6)
|
|
16
18
|
builder (3.0.0)
|
|
17
19
|
crack (0.1.8)
|
|
18
|
-
diff-lcs (1.1.
|
|
19
|
-
dropbox (1.
|
|
20
|
-
json
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
excon (0.6.6)
|
|
24
|
-
faraday (0.7.4)
|
|
20
|
+
diff-lcs (1.1.3)
|
|
21
|
+
dropbox-sdk (1.1)
|
|
22
|
+
json
|
|
23
|
+
excon (0.9.4)
|
|
24
|
+
faraday (0.7.5)
|
|
25
25
|
addressable (~> 2.2.6)
|
|
26
|
-
multipart-post (~> 1.1.
|
|
26
|
+
multipart-post (~> 1.1.3)
|
|
27
27
|
rack (>= 1.1.0, < 2)
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
ffi (1.0.9)
|
|
31
|
-
fog (0.11.0)
|
|
28
|
+
ffi (1.0.11)
|
|
29
|
+
fog (1.1.2)
|
|
32
30
|
builder
|
|
33
|
-
excon (~> 0.
|
|
31
|
+
excon (~> 0.9.0)
|
|
34
32
|
formatador (~> 0.2.0)
|
|
35
33
|
mime-types
|
|
36
34
|
multi_json (~> 1.0.3)
|
|
37
35
|
net-scp (~> 1.0.4)
|
|
38
|
-
net-ssh (
|
|
36
|
+
net-ssh (>= 2.1.3)
|
|
39
37
|
nokogiri (~> 1.5.0)
|
|
40
38
|
ruby-hmac
|
|
41
39
|
formatador (0.2.1)
|
|
42
|
-
fuubar (0.0.
|
|
40
|
+
fuubar (0.0.6)
|
|
43
41
|
rspec (~> 2.0)
|
|
44
|
-
rspec-instafail (~> 0.1.
|
|
42
|
+
rspec-instafail (~> 0.1.8)
|
|
45
43
|
ruby-progressbar (~> 0.0.10)
|
|
46
44
|
growl (1.0.3)
|
|
47
|
-
guard (0.
|
|
45
|
+
guard (0.10.0)
|
|
46
|
+
ffi (>= 0.5.0)
|
|
48
47
|
thor (~> 0.14.6)
|
|
49
|
-
guard-rspec (0.
|
|
50
|
-
guard (>= 0.
|
|
51
|
-
hashie (1.1.0)
|
|
48
|
+
guard-rspec (0.6.0)
|
|
49
|
+
guard (>= 0.10.0)
|
|
52
50
|
hipchat (0.4.1)
|
|
53
51
|
httparty
|
|
54
|
-
httparty (0.7.
|
|
52
|
+
httparty (0.7.8)
|
|
55
53
|
crack (= 0.1.8)
|
|
56
54
|
i18n (0.6.0)
|
|
57
55
|
json (1.5.4)
|
|
58
|
-
libnotify (0.
|
|
56
|
+
libnotify (0.7.1)
|
|
59
57
|
mail (2.3.0)
|
|
60
58
|
i18n (>= 0.4.0)
|
|
61
59
|
mime-types (~> 1.16)
|
|
62
60
|
treetop (~> 1.4.8)
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
61
|
+
metaclass (0.0.1)
|
|
62
|
+
mime-types (1.17.2)
|
|
63
|
+
mocha (0.10.0)
|
|
64
|
+
metaclass (~> 0.0.1)
|
|
65
|
+
multi_json (1.0.4)
|
|
66
|
+
multipart-post (1.1.4)
|
|
68
67
|
net-scp (1.0.4)
|
|
69
68
|
net-ssh (>= 1.99.1)
|
|
70
69
|
net-sftp (2.0.5)
|
|
71
70
|
net-ssh (>= 2.0.9)
|
|
72
71
|
net-ssh (2.1.4)
|
|
73
72
|
nokogiri (1.5.0)
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
polyglot (0.3.2)
|
|
73
|
+
open4 (1.3.0)
|
|
74
|
+
polyglot (0.3.3)
|
|
77
75
|
prowler (1.3.1)
|
|
78
|
-
rack (1.
|
|
79
|
-
rb-fsevent (0.4.
|
|
80
|
-
rb-inotify (0.8.
|
|
76
|
+
rack (1.4.0)
|
|
77
|
+
rb-fsevent (0.4.3.1)
|
|
78
|
+
rb-inotify (0.8.8)
|
|
81
79
|
ffi (>= 0.5.0)
|
|
82
|
-
rspec (2.
|
|
83
|
-
rspec-core (~> 2.
|
|
84
|
-
rspec-expectations (~> 2.
|
|
85
|
-
rspec-mocks (~> 2.
|
|
86
|
-
rspec-core (2.
|
|
87
|
-
rspec-expectations (2.
|
|
80
|
+
rspec (2.8.0)
|
|
81
|
+
rspec-core (~> 2.8.0)
|
|
82
|
+
rspec-expectations (~> 2.8.0)
|
|
83
|
+
rspec-mocks (~> 2.8.0)
|
|
84
|
+
rspec-core (2.8.0)
|
|
85
|
+
rspec-expectations (2.8.0)
|
|
88
86
|
diff-lcs (~> 1.1.2)
|
|
89
|
-
rspec-instafail (0.1.
|
|
90
|
-
rspec-mocks (2.
|
|
87
|
+
rspec-instafail (0.1.9)
|
|
88
|
+
rspec-mocks (2.8.0)
|
|
91
89
|
ruby-hmac (0.4.0)
|
|
92
90
|
ruby-progressbar (0.0.10)
|
|
93
91
|
simple_oauth (0.1.5)
|
|
@@ -96,20 +94,18 @@ GEM
|
|
|
96
94
|
treetop (1.4.10)
|
|
97
95
|
polyglot
|
|
98
96
|
polyglot (>= 0.3.1)
|
|
99
|
-
twitter (
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
multi_xml (~> 0.3.0)
|
|
105
|
-
simple_oauth (~> 0.1.5)
|
|
97
|
+
twitter (2.0.2)
|
|
98
|
+
activesupport (>= 2.3.9, < 4)
|
|
99
|
+
faraday (~> 0.7)
|
|
100
|
+
multi_json (~> 1.0)
|
|
101
|
+
simple_oauth (~> 0.1)
|
|
106
102
|
|
|
107
103
|
PLATFORMS
|
|
108
104
|
ruby
|
|
109
105
|
|
|
110
106
|
DEPENDENCIES
|
|
111
107
|
backup!
|
|
112
|
-
dropbox (~> 1.
|
|
108
|
+
dropbox-sdk (~> 1.1.0)
|
|
113
109
|
fog (>= 0.11.0)
|
|
114
110
|
fuubar
|
|
115
111
|
growl
|
data/README.md
CHANGED
|
@@ -15,12 +15,6 @@ Author
|
|
|
15
15
|
Drop me a message for any questions, suggestions, requests, bugs or submit them to the [issue log](https://github.com/meskyanichi/backup/issues).
|
|
16
16
|
|
|
17
17
|
|
|
18
|
-
Core Contributor
|
|
19
|
-
----------------
|
|
20
|
-
|
|
21
|
-
**[Brian Burns](https://github.com/burns)**
|
|
22
|
-
|
|
23
|
-
|
|
24
18
|
Installation
|
|
25
19
|
------------
|
|
26
20
|
|
|
@@ -92,14 +86,14 @@ Storage Features
|
|
|
92
86
|
- **Incremental Backups, applies to:**
|
|
93
87
|
- Remote Servers *(Only Protocols: RSync)*
|
|
94
88
|
|
|
95
|
-
[
|
|
89
|
+
[Cycling Wiki Page](https://github.com/meskyanichi/backup/wiki/Cycling)
|
|
96
90
|
|
|
97
91
|
[Splitter Wiki Page](https://github.com/meskyanichi/backup/wiki/Splitter)
|
|
98
92
|
|
|
99
93
|
Syncers
|
|
100
94
|
-------
|
|
101
95
|
|
|
102
|
-
- RSync
|
|
96
|
+
- RSync (Push, Pull and Local)
|
|
103
97
|
- Amazon Simple Storage Service (S3)
|
|
104
98
|
|
|
105
99
|
[Syncer Wiki Page](https://github.com/meskyanichi/backup/wiki/Syncers)
|
|
@@ -240,21 +234,50 @@ end
|
|
|
240
234
|
|
|
241
235
|
### Brief explanation for the above example configuration
|
|
242
236
|
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
237
|
+
First, it will dump the two Databases (MySQL and MongoDB). The MySQL dump will be piped through the Gzip Compressor into
|
|
238
|
+
`sample_backup/databases/MySQL/my_sample_mysql_db.sql.gz`. The MongoDB dump will be dumped into
|
|
239
|
+
`sample_backup/databases/MongoDB/`, which will then be packaged into `sample_backup/databases/MongoDB-#####.tar.gz`
|
|
240
|
+
(`#####` will be a simple unique identifier, in case multiple dumps are performed.)
|
|
241
|
+
Next, it will create two _tar_ Archives (user_avatars and logs). Each will be piped through the Gzip Compressor into
|
|
242
|
+
`sample_backup/archives/` as `user_archives.tar.gz` and `logs.tar.gz`.
|
|
243
|
+
Finally, the `sample_backup` directory will be packaged into an uncompressed _tar_ archive, which will be piped through
|
|
244
|
+
the OpenSSL Encryptor to encrypt this final package into `YYYY-MM-DD-hh-mm-ss.sample_backup.tar.enc`. This final
|
|
245
|
+
encrypted archive will then be transfered to your Amazon S3 account. If all goes well, and no exceptions are raised,
|
|
246
|
+
you'll be notified via the Twitter notifier that the backup succeeded. If any warnings were issued or there was an
|
|
247
|
+
exception raised during the backup process, then you'd receive an email in your inbox containing detailed exception
|
|
248
|
+
information, as well as receive a simple Twitter message that something went wrong.
|
|
249
|
+
|
|
250
|
+
Aside of S3, we have also defined two `SFTP` storage methods, and given them two unique identifiers `Server A` and
|
|
251
|
+
`Server B` to distinguish between the two. With these in place, a copy of the backup will now also be stored on two
|
|
252
|
+
separate servers: `a.my-backup-server.com` and `b.my-backup-server.com`.
|
|
253
|
+
|
|
254
|
+
As you can see, you can freely mix and match **archives**, **databases**, **compressors**, **encryptors**, **storages**
|
|
255
|
+
and **notifiers** for your backups. You could even specify 4 storage locations if you wanted: Amazon S3, Rackspace Cloud
|
|
256
|
+
Files, Ninefold and Dropbox, it'd then store your packaged backup to 4 separate locations for high redundancy.
|
|
257
|
+
|
|
258
|
+
Also, notice the `split_into_chunks_of 4000` at the top of the configuration. This tells Backup to split any backups
|
|
259
|
+
that exceed in 4000 MEGABYTES of size in to multiple smaller chunks. Assuming your backup file is 12000 MEGABYTES (12GB)
|
|
260
|
+
in size, then Backup will take the output which was piped from _tar_ into the OpenSSL Compressor and additionally pipe
|
|
261
|
+
that output through the _split_ utility, which will result in 3 chunks of 4000 MEGABYTES with additional file extensions
|
|
262
|
+
of `-aa`, `-ab` and `-ac`. These files will then be individually transfered. This is useful for when you are using
|
|
263
|
+
Amazon S3, Rackspace Cloud Files, or other 3rd party storage services which limit you to "5GB per file" uploads. So with
|
|
264
|
+
this, the backup file size is no longer a constraint.
|
|
265
|
+
|
|
266
|
+
Additionally we have also defined a **S3 Syncer** ( `sync_with S3` ), which does not follow the above process of
|
|
267
|
+
archiving/compression/encryption, but instead will directly sync the whole `videos` and `music` folder structures from
|
|
268
|
+
your machine to your Amazon S3 account. (very efficient and cost-effective since it will only transfer files that were
|
|
269
|
+
added/changed. Additionally, since we flagged it to 'mirror', it'll also remove files from S3 that no longer exist). If
|
|
270
|
+
you simply wanted to sync to a separate backup server that you own, you could also use the RSync syncer for even more
|
|
271
|
+
efficient backups that only transfer the **bytes** of each file that changed.
|
|
272
|
+
|
|
273
|
+
There are more **archives**, **databases**, **compressors**, **encryptors**, **storages** and **notifiers** than
|
|
274
|
+
displayed in the example, all available components are listed at the top of this README, as well as in the
|
|
275
|
+
[Wiki](https://github.com/meskyanichi/backup/wiki) with more detailed information.
|
|
254
276
|
|
|
255
277
|
### Running the example
|
|
256
278
|
|
|
257
|
-
Notice the `Backup::Model.new(:sample_backup, 'A sample backup configuration') do` at the top of the above example. The
|
|
279
|
+
Notice the `Backup::Model.new(:sample_backup, 'A sample backup configuration') do` at the top of the above example. The
|
|
280
|
+
`:sample_backup` is called the **trigger**. This is used to identify the backup procedure/file and initialize it.
|
|
258
281
|
|
|
259
282
|
``` sh
|
|
260
283
|
backup perform -t [--trigger] sample_backup
|
|
@@ -264,7 +287,9 @@ Now it'll run the backup, it's as simple as that.
|
|
|
264
287
|
|
|
265
288
|
### Automatic backups
|
|
266
289
|
|
|
267
|
-
Since Backup is an easy-to-use command line utility, you should write a crontask to invoke it periodically. I recommend
|
|
290
|
+
Since Backup is an easy-to-use command line utility, you should write a crontask to invoke it periodically. I recommend
|
|
291
|
+
using [Whenever](https://github.com/javan/whenever) to manage your crontab. It'll allow you to write to the crontab
|
|
292
|
+
using pure Ruby, and it provides an elegant DSL to do so. Here's an example:
|
|
268
293
|
|
|
269
294
|
``` rb
|
|
270
295
|
every 6.hours do
|
|
@@ -272,12 +297,14 @@ every 6.hours do
|
|
|
272
297
|
end
|
|
273
298
|
```
|
|
274
299
|
|
|
275
|
-
With this in place, run `whenever --update-crontab backup` to write the equivalent of the above Ruby syntax to the
|
|
300
|
+
With this in place, run `whenever --update-crontab backup` to write the equivalent of the above Ruby syntax to the
|
|
301
|
+
crontab in cron-syntax. Cron will now invoke `backup perform --trigger sample_backup` every 6 hours. Check out the
|
|
302
|
+
Whenever project page for more information.
|
|
276
303
|
|
|
277
304
|
Documentation
|
|
278
305
|
-------------
|
|
279
306
|
|
|
280
|
-
See the [Wiki Pages](https://github.com/meskyanichi/backup/wiki).
|
|
307
|
+
See the [Wiki Pages](https://github.com/meskyanichi/backup/wiki).
|
|
281
308
|
|
|
282
309
|
|
|
283
310
|
Suggestions, Bugs, Requests, Questions
|
|
@@ -293,6 +320,10 @@ Contributors
|
|
|
293
320
|
<th>Contributor</th>
|
|
294
321
|
<th>Contribution</th>
|
|
295
322
|
</tr>
|
|
323
|
+
<tr>
|
|
324
|
+
<td><a href="https://github.com/burns" target="_blank"><b>Brian D. Burns ( burns )</b></a></td>
|
|
325
|
+
<td><b>Core Contributor</b></td>
|
|
326
|
+
</tr>
|
|
296
327
|
<tr>
|
|
297
328
|
<td><a href="https://github.com/asanghi" target="_blank">Aditya Sanghi ( asanghi )</a></td>
|
|
298
329
|
<td>Twitter Notifier, Dropbox Timeout Configuration</td>
|
|
@@ -409,10 +440,6 @@ Contributors
|
|
|
409
440
|
<td><a href="https://github.com/szymonpk" target="_blank">Szymon ( szymonpk )</a></td>
|
|
410
441
|
<td>Pbzip2 compressor</td>
|
|
411
442
|
</tr>
|
|
412
|
-
<tr>
|
|
413
|
-
<td><a href="https://github.com/burns" target="_blank">burns ( burns )</a></td>
|
|
414
|
-
<td>Improved Backup cycling implementation by refreshing all user configuration during the cycle procedure</td>
|
|
415
|
-
</tr>
|
|
416
443
|
</table>
|
|
417
444
|
|
|
418
445
|
|
data/lib/backup.rb
CHANGED
|
@@ -23,22 +23,6 @@ end
|
|
|
23
23
|
# The Backup Ruby Gem
|
|
24
24
|
module Backup
|
|
25
25
|
|
|
26
|
-
##
|
|
27
|
-
# List the available database, storage, compressor, encryptor and notifier constants.
|
|
28
|
-
# These are used to dynamically define these constants as classes inside Backup::Finder
|
|
29
|
-
# to provide a nicer configuration file DSL syntax to the users. Adding existing constants
|
|
30
|
-
# to the arrays below will enable the user to use a constant instead of a string.
|
|
31
|
-
# Example, instead of:
|
|
32
|
-
# database "MySQL" do |mysql|
|
|
33
|
-
# You can do:
|
|
34
|
-
# database MySQL do |mysql|
|
|
35
|
-
DATABASES = ['MySQL', 'PostgreSQL', 'MongoDB', 'Redis', 'Riak']
|
|
36
|
-
STORAGES = ['S3', 'CloudFiles', 'Ninefold', 'Dropbox', 'FTP', 'SFTP', 'SCP', 'RSync', 'Local']
|
|
37
|
-
COMPRESSORS = ['Gzip', 'Bzip2', 'Pbzip2', 'Lzma']
|
|
38
|
-
ENCRYPTORS = ['OpenSSL', 'GPG']
|
|
39
|
-
SYNCERS = ['RSync', 'S3']
|
|
40
|
-
NOTIFIERS = ['Mail', 'Twitter', 'Campfire', 'Presently', 'Prowl', 'Hipchat']
|
|
41
|
-
|
|
42
26
|
##
|
|
43
27
|
# Backup's internal paths
|
|
44
28
|
LIBRARY_PATH = File.join(File.dirname(__FILE__), 'backup')
|
|
@@ -52,25 +36,15 @@ module Backup
|
|
|
52
36
|
CONFIGURATION_PATH = File.join(LIBRARY_PATH, 'configuration')
|
|
53
37
|
TEMPLATE_PATH = File.expand_path('../../templates', __FILE__)
|
|
54
38
|
|
|
55
|
-
##
|
|
56
|
-
# Backup's Environment paths
|
|
57
|
-
USER = ENV['USER'] || Etc.getpwuid.name
|
|
58
|
-
HOME = File.expand_path(ENV['HOME'] || '')
|
|
59
|
-
PATH = File.join(HOME, 'Backup')
|
|
60
|
-
CONFIG_FILE = File.join(PATH, 'config.rb')
|
|
61
|
-
DATA_PATH = File.join(PATH, 'data')
|
|
62
|
-
LOG_PATH = File.join(PATH, 'log')
|
|
63
|
-
CACHE_PATH = File.join(PATH, '.cache')
|
|
64
|
-
TMP_PATH = File.join(PATH, '.tmp')
|
|
65
|
-
|
|
66
39
|
##
|
|
67
40
|
# Autoload Backup base files
|
|
68
41
|
autoload :Model, File.join(LIBRARY_PATH, 'model')
|
|
69
42
|
autoload :Archive, File.join(LIBRARY_PATH, 'archive')
|
|
70
43
|
autoload :Packager, File.join(LIBRARY_PATH, 'packager')
|
|
44
|
+
autoload :Package, File.join(LIBRARY_PATH, 'package')
|
|
71
45
|
autoload :Cleaner, File.join(LIBRARY_PATH, 'cleaner')
|
|
72
46
|
autoload :Splitter, File.join(LIBRARY_PATH, 'splitter')
|
|
73
|
-
autoload :
|
|
47
|
+
autoload :Config, File.join(LIBRARY_PATH, 'config')
|
|
74
48
|
autoload :Binder, File.join(LIBRARY_PATH, 'binder')
|
|
75
49
|
autoload :Template, File.join(LIBRARY_PATH, 'template')
|
|
76
50
|
autoload :Dependency, File.join(LIBRARY_PATH, 'dependency')
|
|
@@ -89,7 +63,7 @@ module Backup
|
|
|
89
63
|
# Autoload Backup storage files
|
|
90
64
|
module Storage
|
|
91
65
|
autoload :Base, File.join(STORAGE_PATH, 'base')
|
|
92
|
-
autoload :
|
|
66
|
+
autoload :Cycler, File.join(STORAGE_PATH, 'cycler')
|
|
93
67
|
autoload :S3, File.join(STORAGE_PATH, 's3')
|
|
94
68
|
autoload :CloudFiles, File.join(STORAGE_PATH, 'cloudfiles')
|
|
95
69
|
autoload :Ninefold, File.join(STORAGE_PATH, 'ninefold')
|
|
@@ -105,8 +79,13 @@ module Backup
|
|
|
105
79
|
# Autoload Backup syncer files
|
|
106
80
|
module Syncer
|
|
107
81
|
autoload :Base, File.join(SYNCER_PATH, 'base')
|
|
108
|
-
autoload :RSync, File.join(SYNCER_PATH, 'rsync')
|
|
109
82
|
autoload :S3, File.join(SYNCER_PATH, 's3')
|
|
83
|
+
module RSync
|
|
84
|
+
autoload :Base, File.join(SYNCER_PATH, 'rsync', 'base')
|
|
85
|
+
autoload :Local, File.join(SYNCER_PATH, 'rsync', 'local')
|
|
86
|
+
autoload :Push, File.join(SYNCER_PATH, 'rsync', 'push')
|
|
87
|
+
autoload :Pull, File.join(SYNCER_PATH, 'rsync', 'pull')
|
|
88
|
+
end
|
|
110
89
|
end
|
|
111
90
|
|
|
112
91
|
##
|
|
@@ -195,8 +174,14 @@ module Backup
|
|
|
195
174
|
end
|
|
196
175
|
|
|
197
176
|
module Syncer
|
|
198
|
-
autoload :
|
|
177
|
+
autoload :Base, File.join(CONFIGURATION_PATH, 'syncer', 'base')
|
|
199
178
|
autoload :S3, File.join(CONFIGURATION_PATH, 'syncer', 's3')
|
|
179
|
+
module RSync
|
|
180
|
+
autoload :Base, File.join(CONFIGURATION_PATH, 'syncer', 'rsync', 'base')
|
|
181
|
+
autoload :Local, File.join(CONFIGURATION_PATH, 'syncer', 'rsync', 'local')
|
|
182
|
+
autoload :Push, File.join(CONFIGURATION_PATH, 'syncer', 'rsync', 'push')
|
|
183
|
+
autoload :Pull, File.join(CONFIGURATION_PATH, 'syncer', 'rsync', 'pull')
|
|
184
|
+
end
|
|
200
185
|
end
|
|
201
186
|
|
|
202
187
|
module Database
|
|
@@ -209,12 +194,4 @@ module Backup
|
|
|
209
194
|
end
|
|
210
195
|
end
|
|
211
196
|
|
|
212
|
-
##
|
|
213
|
-
# Dynamically defines all the available database, storage, compressor, encryptor and notifier
|
|
214
|
-
# classes inside Backup::Finder to improve the DSL for the configuration file
|
|
215
|
-
(DATABASES + STORAGES + COMPRESSORS + ENCRYPTORS + NOTIFIERS + SYNCERS).each do |constant|
|
|
216
|
-
unless Backup::Finder.const_defined?(constant)
|
|
217
|
-
Backup::Finder.const_set(constant, Class.new)
|
|
218
|
-
end
|
|
219
|
-
end
|
|
220
197
|
end
|
data/lib/backup/archive.rb
CHANGED
|
@@ -17,24 +17,34 @@ module Backup
|
|
|
17
17
|
attr_accessor :excludes
|
|
18
18
|
|
|
19
19
|
##
|
|
20
|
-
#
|
|
21
|
-
attr_accessor :
|
|
20
|
+
# String of additional arguments for the `tar` command
|
|
21
|
+
attr_accessor :tar_args
|
|
22
22
|
|
|
23
23
|
##
|
|
24
24
|
# Takes the name of the archive and the configuration block
|
|
25
|
-
def initialize(name, &block)
|
|
26
|
-
@
|
|
25
|
+
def initialize(model, name, &block)
|
|
26
|
+
@model = model
|
|
27
|
+
@name = name.to_s
|
|
27
28
|
@paths = Array.new
|
|
28
29
|
@excludes = Array.new
|
|
29
|
-
@
|
|
30
|
+
@tar_args = ''
|
|
30
31
|
|
|
31
|
-
instance_eval(&block)
|
|
32
|
+
instance_eval(&block) if block_given?
|
|
32
33
|
end
|
|
33
34
|
|
|
34
35
|
##
|
|
35
36
|
# Adds new paths to the @paths instance variable array
|
|
36
37
|
def add(path)
|
|
37
|
-
|
|
38
|
+
path = File.expand_path(path)
|
|
39
|
+
if File.exist?(path)
|
|
40
|
+
@paths << path
|
|
41
|
+
else
|
|
42
|
+
Logger.warn Errors::Archive::NotFoundError.new(<<-EOS)
|
|
43
|
+
The following path was not found:
|
|
44
|
+
#{ path }
|
|
45
|
+
This path will be omitted from the '#{ name }' Archive.
|
|
46
|
+
EOS
|
|
47
|
+
end
|
|
38
48
|
end
|
|
39
49
|
|
|
40
50
|
##
|
|
@@ -47,39 +57,53 @@ module Backup
|
|
|
47
57
|
# Adds the given String of +options+ to the `tar` command.
|
|
48
58
|
# e.g. '-h --xattrs'
|
|
49
59
|
def tar_options(options)
|
|
50
|
-
@
|
|
60
|
+
@tar_args = options
|
|
51
61
|
end
|
|
52
62
|
|
|
53
63
|
##
|
|
54
64
|
# Archives all the provided paths in to a single .tar file
|
|
55
65
|
# and places that .tar file in the folder which later will be packaged
|
|
66
|
+
# If the model is configured with a Compressor, the tar command output
|
|
67
|
+
# will be piped through the Compressor command and the file extension
|
|
68
|
+
# will be adjusted to indicate the type of compression used.
|
|
56
69
|
def perform!
|
|
57
|
-
@archive_path = File.join(TMP_PATH, TRIGGER, 'archive')
|
|
58
|
-
mkdir(archive_path)
|
|
59
|
-
|
|
60
70
|
Logger.message "#{ self.class } started packaging and archiving:\n" +
|
|
61
71
|
paths.map {|path| " #{path}" }.join("\n")
|
|
62
72
|
|
|
63
|
-
|
|
64
|
-
|
|
73
|
+
archive_path = File.join(Config.tmp_path, @model.trigger, 'archives')
|
|
74
|
+
FileUtils.mkdir_p(archive_path)
|
|
75
|
+
|
|
76
|
+
archive_ext = 'tar'
|
|
77
|
+
archive_cmd = "#{ utility(:tar) } #{ tar_args } -cf - " +
|
|
78
|
+
"#{ paths_to_exclude } #{ paths_to_package }"
|
|
79
|
+
|
|
80
|
+
if @model.compressor
|
|
81
|
+
@model.compressor.compress_with do |command, ext|
|
|
82
|
+
archive_cmd << " | #{command}"
|
|
83
|
+
archive_ext << ext
|
|
84
|
+
end
|
|
85
|
+
end
|
|
86
|
+
|
|
87
|
+
archive_cmd << " > '#{ File.join(archive_path, "#{name}.#{archive_ext}") }'"
|
|
88
|
+
|
|
89
|
+
run(archive_cmd)
|
|
65
90
|
end
|
|
66
91
|
|
|
67
|
-
|
|
92
|
+
private
|
|
68
93
|
|
|
69
94
|
##
|
|
70
95
|
# Returns a "tar-ready" string of all the specified paths combined
|
|
71
96
|
def paths_to_package
|
|
72
|
-
paths.map
|
|
73
|
-
"'#{path}'"
|
|
74
|
-
end.join("\s")
|
|
97
|
+
paths.map {|path| "'#{path}'" }.join(' ')
|
|
75
98
|
end
|
|
76
99
|
|
|
77
100
|
##
|
|
78
101
|
# Returns a "tar-ready" string of all the specified excludes combined
|
|
79
102
|
def paths_to_exclude
|
|
80
103
|
if excludes.any?
|
|
81
|
-
excludes.map{
|
|
104
|
+
excludes.map {|path| "--exclude='#{path}'" }.join(' ')
|
|
82
105
|
end
|
|
83
106
|
end
|
|
107
|
+
|
|
84
108
|
end
|
|
85
109
|
end
|