backup 3.0.23 → 3.0.24
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/Gemfile.lock +42 -45
- data/Guardfile +7 -4
- data/README.md +10 -7
- data/backup.gemspec +2 -2
- data/lib/backup.rb +27 -97
- data/lib/backup/archive.rb +14 -6
- data/lib/backup/cli/helpers.rb +52 -49
- data/lib/backup/cli/utility.rb +9 -1
- data/lib/backup/compressor/base.rb +10 -4
- data/lib/backup/compressor/bzip2.rb +22 -26
- data/lib/backup/compressor/custom.rb +53 -0
- data/lib/backup/compressor/gzip.rb +22 -23
- data/lib/backup/compressor/lzma.rb +15 -13
- data/lib/backup/compressor/pbzip2.rb +20 -17
- data/lib/backup/config.rb +6 -3
- data/lib/backup/configuration.rb +33 -0
- data/lib/backup/configuration/helpers.rb +114 -28
- data/lib/backup/configuration/store.rb +24 -0
- data/lib/backup/database/base.rb +0 -6
- data/lib/backup/database/mongodb.rb +27 -11
- data/lib/backup/database/mysql.rb +19 -14
- data/lib/backup/database/postgresql.rb +16 -11
- data/lib/backup/database/redis.rb +7 -11
- data/lib/backup/database/riak.rb +3 -6
- data/lib/backup/dependency.rb +5 -11
- data/lib/backup/model.rb +14 -5
- data/lib/backup/notifier/campfire.rb +3 -16
- data/lib/backup/notifier/hipchat.rb +1 -7
- data/lib/backup/notifier/mail.rb +1 -1
- data/lib/backup/packager.rb +29 -19
- data/lib/backup/pipeline.rb +110 -0
- data/lib/backup/storage/dropbox.rb +4 -7
- data/lib/backup/syncer/base.rb +8 -4
- data/lib/backup/syncer/cloud/base.rb +247 -0
- data/lib/backup/syncer/cloud/cloud_files.rb +78 -0
- data/lib/backup/syncer/cloud/s3.rb +68 -0
- data/lib/backup/syncer/rsync/base.rb +1 -4
- data/lib/backup/syncer/rsync/local.rb +9 -5
- data/lib/backup/syncer/rsync/pull.rb +1 -1
- data/lib/backup/syncer/rsync/push.rb +10 -5
- data/lib/backup/version.rb +1 -1
- data/spec-live/.gitignore +6 -0
- data/spec-live/README +7 -0
- data/spec-live/backups/config.rb +153 -0
- data/spec-live/backups/config.yml.template +43 -0
- data/spec-live/compressor/custom_spec.rb +30 -0
- data/spec-live/compressor/gzip_spec.rb +30 -0
- data/spec-live/notifier/mail_spec.rb +85 -0
- data/spec-live/spec_helper.rb +85 -0
- data/spec-live/storage/dropbox_spec.rb +151 -0
- data/spec-live/storage/local_spec.rb +83 -0
- data/spec-live/storage/scp_spec.rb +193 -0
- data/spec-live/syncer/cloud/s3_spec.rb +124 -0
- data/spec/archive_spec.rb +86 -31
- data/spec/cleaner_spec.rb +8 -0
- data/spec/cli/helpers_spec.rb +200 -75
- data/spec/cli/utility_spec.rb +11 -3
- data/spec/compressor/base_spec.rb +31 -10
- data/spec/compressor/bzip2_spec.rb +212 -57
- data/spec/compressor/custom_spec.rb +106 -0
- data/spec/compressor/gzip_spec.rb +212 -57
- data/spec/compressor/lzma_spec.rb +75 -35
- data/spec/compressor/pbzip2_spec.rb +93 -52
- data/spec/configuration/helpers_spec.rb +406 -0
- data/spec/configuration/store_spec.rb +39 -0
- data/spec/configuration_spec.rb +62 -0
- data/spec/database/base_spec.rb +19 -10
- data/spec/database/mongodb_spec.rb +195 -70
- data/spec/database/mysql_spec.rb +183 -64
- data/spec/database/postgresql_spec.rb +167 -53
- data/spec/database/redis_spec.rb +121 -46
- data/spec/database/riak_spec.rb +96 -27
- data/spec/dependency_spec.rb +2 -0
- data/spec/encryptor/base_spec.rb +10 -0
- data/spec/encryptor/gpg_spec.rb +29 -13
- data/spec/encryptor/open_ssl_spec.rb +40 -21
- data/spec/logger_spec.rb +4 -0
- data/spec/model_spec.rb +19 -2
- data/spec/notifier/base_spec.rb +32 -17
- data/spec/notifier/campfire_spec.rb +63 -45
- data/spec/notifier/hipchat_spec.rb +79 -56
- data/spec/notifier/mail_spec.rb +82 -46
- data/spec/notifier/prowl_spec.rb +53 -32
- data/spec/notifier/twitter_spec.rb +62 -41
- data/spec/packager_spec.rb +95 -36
- data/spec/pipeline_spec.rb +259 -0
- data/spec/spec_helper.rb +6 -5
- data/spec/storage/base_spec.rb +61 -41
- data/spec/storage/cloudfiles_spec.rb +69 -45
- data/spec/storage/dropbox_spec.rb +158 -36
- data/spec/storage/ftp_spec.rb +69 -45
- data/spec/storage/local_spec.rb +47 -23
- data/spec/storage/ninefold_spec.rb +55 -31
- data/spec/storage/rsync_spec.rb +67 -50
- data/spec/storage/s3_spec.rb +65 -41
- data/spec/storage/scp_spec.rb +65 -41
- data/spec/storage/sftp_spec.rb +65 -41
- data/spec/syncer/base_spec.rb +91 -4
- data/spec/syncer/cloud/base_spec.rb +511 -0
- data/spec/syncer/cloud/cloud_files_spec.rb +181 -0
- data/spec/syncer/cloud/s3_spec.rb +174 -0
- data/spec/syncer/rsync/base_spec.rb +46 -66
- data/spec/syncer/rsync/local_spec.rb +55 -26
- data/spec/syncer/rsync/pull_spec.rb +15 -4
- data/spec/syncer/rsync/push_spec.rb +59 -52
- data/templates/cli/utility/compressor/bzip2 +1 -4
- data/templates/cli/utility/compressor/custom +11 -0
- data/templates/cli/utility/compressor/gzip +1 -4
- data/templates/cli/utility/compressor/lzma +3 -0
- data/templates/cli/utility/compressor/pbzip2 +3 -0
- data/templates/cli/utility/database/mysql +4 -1
- data/templates/cli/utility/syncer/cloud_files +17 -19
- data/templates/cli/utility/syncer/s3 +18 -20
- metadata +38 -92
- data/lib/backup/configuration/base.rb +0 -15
- data/lib/backup/configuration/compressor/base.rb +0 -9
- data/lib/backup/configuration/compressor/bzip2.rb +0 -23
- data/lib/backup/configuration/compressor/gzip.rb +0 -23
- data/lib/backup/configuration/compressor/lzma.rb +0 -23
- data/lib/backup/configuration/compressor/pbzip2.rb +0 -28
- data/lib/backup/configuration/database/base.rb +0 -19
- data/lib/backup/configuration/database/mongodb.rb +0 -49
- data/lib/backup/configuration/database/mysql.rb +0 -42
- data/lib/backup/configuration/database/postgresql.rb +0 -41
- data/lib/backup/configuration/database/redis.rb +0 -39
- data/lib/backup/configuration/database/riak.rb +0 -29
- data/lib/backup/configuration/encryptor/base.rb +0 -9
- data/lib/backup/configuration/encryptor/gpg.rb +0 -17
- data/lib/backup/configuration/encryptor/open_ssl.rb +0 -32
- data/lib/backup/configuration/notifier/base.rb +0 -28
- data/lib/backup/configuration/notifier/campfire.rb +0 -25
- data/lib/backup/configuration/notifier/hipchat.rb +0 -41
- data/lib/backup/configuration/notifier/mail.rb +0 -112
- data/lib/backup/configuration/notifier/presently.rb +0 -25
- data/lib/backup/configuration/notifier/prowl.rb +0 -23
- data/lib/backup/configuration/notifier/twitter.rb +0 -21
- data/lib/backup/configuration/storage/base.rb +0 -18
- data/lib/backup/configuration/storage/cloudfiles.rb +0 -25
- data/lib/backup/configuration/storage/dropbox.rb +0 -58
- data/lib/backup/configuration/storage/ftp.rb +0 -29
- data/lib/backup/configuration/storage/local.rb +0 -17
- data/lib/backup/configuration/storage/ninefold.rb +0 -20
- data/lib/backup/configuration/storage/rsync.rb +0 -29
- data/lib/backup/configuration/storage/s3.rb +0 -25
- data/lib/backup/configuration/storage/scp.rb +0 -25
- data/lib/backup/configuration/storage/sftp.rb +0 -25
- data/lib/backup/configuration/syncer/base.rb +0 -10
- data/lib/backup/configuration/syncer/cloud.rb +0 -23
- data/lib/backup/configuration/syncer/cloud_files.rb +0 -30
- data/lib/backup/configuration/syncer/rsync/base.rb +0 -28
- data/lib/backup/configuration/syncer/rsync/local.rb +0 -11
- data/lib/backup/configuration/syncer/rsync/pull.rb +0 -11
- data/lib/backup/configuration/syncer/rsync/push.rb +0 -31
- data/lib/backup/configuration/syncer/s3.rb +0 -23
- data/lib/backup/notifier/presently.rb +0 -88
- data/lib/backup/syncer/cloud.rb +0 -187
- data/lib/backup/syncer/cloud_files.rb +0 -56
- data/lib/backup/syncer/s3.rb +0 -47
- data/spec/configuration/base_spec.rb +0 -35
- data/spec/configuration/compressor/bzip2_spec.rb +0 -29
- data/spec/configuration/compressor/gzip_spec.rb +0 -29
- data/spec/configuration/compressor/lzma_spec.rb +0 -29
- data/spec/configuration/compressor/pbzip2_spec.rb +0 -32
- data/spec/configuration/database/base_spec.rb +0 -17
- data/spec/configuration/database/mongodb_spec.rb +0 -56
- data/spec/configuration/database/mysql_spec.rb +0 -53
- data/spec/configuration/database/postgresql_spec.rb +0 -53
- data/spec/configuration/database/redis_spec.rb +0 -50
- data/spec/configuration/database/riak_spec.rb +0 -35
- data/spec/configuration/encryptor/gpg_spec.rb +0 -26
- data/spec/configuration/encryptor/open_ssl_spec.rb +0 -35
- data/spec/configuration/notifier/base_spec.rb +0 -32
- data/spec/configuration/notifier/campfire_spec.rb +0 -32
- data/spec/configuration/notifier/hipchat_spec.rb +0 -44
- data/spec/configuration/notifier/mail_spec.rb +0 -71
- data/spec/configuration/notifier/presently_spec.rb +0 -35
- data/spec/configuration/notifier/prowl_spec.rb +0 -29
- data/spec/configuration/notifier/twitter_spec.rb +0 -35
- data/spec/configuration/storage/cloudfiles_spec.rb +0 -41
- data/spec/configuration/storage/dropbox_spec.rb +0 -38
- data/spec/configuration/storage/ftp_spec.rb +0 -44
- data/spec/configuration/storage/local_spec.rb +0 -29
- data/spec/configuration/storage/ninefold_spec.rb +0 -32
- data/spec/configuration/storage/rsync_spec.rb +0 -41
- data/spec/configuration/storage/s3_spec.rb +0 -38
- data/spec/configuration/storage/scp_spec.rb +0 -41
- data/spec/configuration/storage/sftp_spec.rb +0 -41
- data/spec/configuration/syncer/cloud_files_spec.rb +0 -44
- data/spec/configuration/syncer/rsync/base_spec.rb +0 -33
- data/spec/configuration/syncer/rsync/local_spec.rb +0 -10
- data/spec/configuration/syncer/rsync/pull_spec.rb +0 -10
- data/spec/configuration/syncer/rsync/push_spec.rb +0 -43
- data/spec/configuration/syncer/s3_spec.rb +0 -38
- data/spec/notifier/presently_spec.rb +0 -181
- data/spec/syncer/cloud_files_spec.rb +0 -192
- data/spec/syncer/s3_spec.rb +0 -192
- data/templates/cli/utility/notifier/presently +0 -13
data/Gemfile.lock
CHANGED
|
@@ -2,29 +2,25 @@ PATH
|
|
|
2
2
|
remote: .
|
|
3
3
|
specs:
|
|
4
4
|
backup (3.0.23)
|
|
5
|
-
|
|
5
|
+
open4 (~> 1.3.0)
|
|
6
6
|
thor (~> 0.14.6)
|
|
7
7
|
|
|
8
8
|
GEM
|
|
9
9
|
remote: http://rubygems.org/
|
|
10
10
|
specs:
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
open4
|
|
14
|
-
Platform (0.4.0)
|
|
15
|
-
activesupport (3.1.3)
|
|
11
|
+
activesupport (3.2.2)
|
|
12
|
+
i18n (~> 0.6)
|
|
16
13
|
multi_json (~> 1.0)
|
|
17
|
-
addressable (2.2.
|
|
14
|
+
addressable (2.2.7)
|
|
18
15
|
builder (3.0.0)
|
|
19
|
-
crack (0.1.8)
|
|
20
16
|
diff-lcs (1.1.3)
|
|
21
|
-
dropbox-sdk (1.
|
|
17
|
+
dropbox-sdk (1.2)
|
|
22
18
|
json
|
|
23
|
-
excon (0.9.
|
|
24
|
-
faraday (0.7.
|
|
25
|
-
addressable (~> 2.2
|
|
26
|
-
multipart-post (~> 1.1
|
|
27
|
-
rack (
|
|
19
|
+
excon (0.9.6)
|
|
20
|
+
faraday (0.7.6)
|
|
21
|
+
addressable (~> 2.2)
|
|
22
|
+
multipart-post (~> 1.1)
|
|
23
|
+
rack (~> 1.1)
|
|
28
24
|
ffi (1.0.11)
|
|
29
25
|
fog (1.1.2)
|
|
30
26
|
builder
|
|
@@ -37,56 +33,58 @@ GEM
|
|
|
37
33
|
nokogiri (~> 1.5.0)
|
|
38
34
|
ruby-hmac
|
|
39
35
|
formatador (0.2.1)
|
|
40
|
-
fuubar (0.0
|
|
36
|
+
fuubar (1.0.0)
|
|
41
37
|
rspec (~> 2.0)
|
|
42
|
-
rspec-instafail (~> 0.
|
|
38
|
+
rspec-instafail (~> 0.2.0)
|
|
43
39
|
ruby-progressbar (~> 0.0.10)
|
|
44
40
|
growl (1.0.3)
|
|
45
|
-
guard (0.
|
|
41
|
+
guard (1.0.1)
|
|
46
42
|
ffi (>= 0.5.0)
|
|
47
43
|
thor (~> 0.14.6)
|
|
48
44
|
guard-rspec (0.6.0)
|
|
49
45
|
guard (>= 0.10.0)
|
|
50
46
|
hipchat (0.4.1)
|
|
51
47
|
httparty
|
|
52
|
-
httparty (0.
|
|
53
|
-
|
|
48
|
+
httparty (0.8.1)
|
|
49
|
+
multi_json
|
|
50
|
+
multi_xml
|
|
54
51
|
i18n (0.6.0)
|
|
55
|
-
json (1.5
|
|
56
|
-
libnotify (0.7.
|
|
57
|
-
mail (2.4.
|
|
52
|
+
json (1.6.5)
|
|
53
|
+
libnotify (0.7.2)
|
|
54
|
+
mail (2.4.4)
|
|
58
55
|
i18n (>= 0.4.0)
|
|
59
56
|
mime-types (~> 1.16)
|
|
60
57
|
treetop (~> 1.4.8)
|
|
61
58
|
metaclass (0.0.1)
|
|
62
59
|
mime-types (1.17.2)
|
|
63
|
-
mocha (0.10.
|
|
60
|
+
mocha (0.10.5)
|
|
64
61
|
metaclass (~> 0.0.1)
|
|
65
62
|
multi_json (1.0.4)
|
|
66
|
-
|
|
63
|
+
multi_xml (0.4.2)
|
|
64
|
+
multipart-post (1.1.5)
|
|
67
65
|
net-scp (1.0.4)
|
|
68
66
|
net-ssh (>= 1.99.1)
|
|
69
67
|
net-sftp (2.0.5)
|
|
70
68
|
net-ssh (>= 2.0.9)
|
|
71
|
-
net-ssh (2.
|
|
72
|
-
nokogiri (1.5.
|
|
69
|
+
net-ssh (2.3.0)
|
|
70
|
+
nokogiri (1.5.2)
|
|
73
71
|
open4 (1.3.0)
|
|
74
|
-
parallel (0.5.
|
|
72
|
+
parallel (0.5.16)
|
|
75
73
|
polyglot (0.3.3)
|
|
76
74
|
prowler (1.3.1)
|
|
77
|
-
rack (1.4.
|
|
78
|
-
rb-fsevent (0.
|
|
75
|
+
rack (1.4.1)
|
|
76
|
+
rb-fsevent (0.9.0)
|
|
79
77
|
rb-inotify (0.8.8)
|
|
80
78
|
ffi (>= 0.5.0)
|
|
81
|
-
rspec (2.
|
|
82
|
-
rspec-core (~> 2.
|
|
83
|
-
rspec-expectations (~> 2.
|
|
84
|
-
rspec-mocks (~> 2.
|
|
85
|
-
rspec-core (2.
|
|
86
|
-
rspec-expectations (2.
|
|
87
|
-
diff-lcs (~> 1.1.
|
|
88
|
-
rspec-instafail (0.
|
|
89
|
-
rspec-mocks (2.
|
|
79
|
+
rspec (2.9.0)
|
|
80
|
+
rspec-core (~> 2.9.0)
|
|
81
|
+
rspec-expectations (~> 2.9.0)
|
|
82
|
+
rspec-mocks (~> 2.9.0)
|
|
83
|
+
rspec-core (2.9.0)
|
|
84
|
+
rspec-expectations (2.9.0)
|
|
85
|
+
diff-lcs (~> 1.1.3)
|
|
86
|
+
rspec-instafail (0.2.2)
|
|
87
|
+
rspec-mocks (2.9.0)
|
|
90
88
|
ruby-hmac (0.4.0)
|
|
91
89
|
ruby-progressbar (0.0.10)
|
|
92
90
|
simple_oauth (0.1.5)
|
|
@@ -95,7 +93,7 @@ GEM
|
|
|
95
93
|
treetop (1.4.10)
|
|
96
94
|
polyglot
|
|
97
95
|
polyglot (>= 0.3.1)
|
|
98
|
-
twitter (2.
|
|
96
|
+
twitter (2.1.1)
|
|
99
97
|
activesupport (>= 2.3.9, < 4)
|
|
100
98
|
faraday (~> 0.7)
|
|
101
99
|
multi_json (~> 1.0)
|
|
@@ -106,21 +104,20 @@ PLATFORMS
|
|
|
106
104
|
|
|
107
105
|
DEPENDENCIES
|
|
108
106
|
backup!
|
|
109
|
-
dropbox-sdk (~> 1.
|
|
110
|
-
fog (
|
|
107
|
+
dropbox-sdk (~> 1.2.0)
|
|
108
|
+
fog (~> 1.1.0)
|
|
111
109
|
fuubar
|
|
112
110
|
growl
|
|
113
111
|
guard
|
|
114
112
|
guard-rspec
|
|
115
113
|
hipchat (~> 0.4.1)
|
|
116
|
-
httparty (~> 0.
|
|
117
|
-
json (~> 1.5.1)
|
|
114
|
+
httparty (~> 0.8.1)
|
|
118
115
|
libnotify
|
|
119
|
-
mail (
|
|
116
|
+
mail (~> 2.4.0)
|
|
120
117
|
mocha
|
|
121
118
|
net-scp (~> 1.0.4)
|
|
122
119
|
net-sftp (~> 2.0.5)
|
|
123
|
-
net-ssh (~> 2.
|
|
120
|
+
net-ssh (~> 2.3.0)
|
|
124
121
|
parallel (~> 0.5.12)
|
|
125
122
|
prowler (>= 1.3.1)
|
|
126
123
|
rb-fsevent
|
data/Guardfile
CHANGED
|
@@ -12,10 +12,13 @@
|
|
|
12
12
|
guard "rspec",
|
|
13
13
|
:version => 2,
|
|
14
14
|
:rvm => ["1.9.3", "1.9.2", "1.8.7"],
|
|
15
|
-
:
|
|
16
|
-
:
|
|
15
|
+
:cli => "--color --format Fuubar",
|
|
16
|
+
:notification => false,
|
|
17
|
+
:all_after_pass => false,
|
|
18
|
+
:all_on_start => false do
|
|
17
19
|
|
|
20
|
+
watch("lib/backup.rb") { "spec" }
|
|
21
|
+
watch("spec/spec_helper.rb") { "spec" }
|
|
22
|
+
watch(%r{^lib/backup/(.+)\.rb}) {|m| "spec/#{ m[1] }_spec.rb" }
|
|
18
23
|
watch(%r{^spec/.+_spec\.rb})
|
|
19
|
-
watch(%r{^lib/(.+)\.rb}) { "spec" }
|
|
20
|
-
watch("spec/spec_helper.rb") { "spec" }
|
|
21
24
|
end
|
data/README.md
CHANGED
|
@@ -69,6 +69,9 @@ Below you find a list of components that Backup currently supports. If you'd lik
|
|
|
69
69
|
- Dropbox Web Service
|
|
70
70
|
- Remote Servers *(Only Protocols: FTP, SFTP, SCP)*
|
|
71
71
|
- Local Storage
|
|
72
|
+
|
|
73
|
+
[Cycling Wiki Page](https://github.com/meskyanichi/backup/wiki/Cycling)
|
|
74
|
+
|
|
72
75
|
- **Backup Splitting, applies to:**
|
|
73
76
|
- Amazon Simple Storage Service (S3)
|
|
74
77
|
- Rackspace Cloud Files (Mosso)
|
|
@@ -76,13 +79,12 @@ Below you find a list of components that Backup currently supports. If you'd lik
|
|
|
76
79
|
- Dropbox Web Service
|
|
77
80
|
- Remote Servers *(Only Protocols: FTP, SFTP, SCP)*
|
|
78
81
|
- Local Storage
|
|
79
|
-
- **Incremental Backups, applies to:**
|
|
80
|
-
- Remote Servers *(Only Protocols: RSync)*
|
|
81
|
-
|
|
82
|
-
[Cycling Wiki Page](https://github.com/meskyanichi/backup/wiki/Cycling)
|
|
83
82
|
|
|
84
83
|
[Splitter Wiki Page](https://github.com/meskyanichi/backup/wiki/Splitter)
|
|
85
84
|
|
|
85
|
+
- **Incremental Backups, applies to:**
|
|
86
|
+
- Remote Servers *(Only Protocols: RSync)*
|
|
87
|
+
|
|
86
88
|
### Syncers
|
|
87
89
|
|
|
88
90
|
- RSync (Push, Pull and Local)
|
|
@@ -128,7 +130,8 @@ Below you find a list of components that Backup currently supports. If you'd lik
|
|
|
128
130
|
A sample Backup configuration file
|
|
129
131
|
----------------------------------
|
|
130
132
|
|
|
131
|
-
This is a Backup configuration file. Check it out and read the explanation below.
|
|
133
|
+
This is a Backup configuration file. Check it out and read the explanation below.
|
|
134
|
+
Backup has a [great wiki](https://github.com/meskyanichi/backup/wiki) which explains each component of Backup in detail.
|
|
132
135
|
|
|
133
136
|
``` rb
|
|
134
137
|
Backup::Model.new(:sample_backup, 'A sample backup configuration') do
|
|
@@ -193,7 +196,7 @@ Backup::Model.new(:sample_backup, 'A sample backup configuration') do
|
|
|
193
196
|
s3.keep = 20
|
|
194
197
|
end
|
|
195
198
|
|
|
196
|
-
sync_with S3 do |s3|
|
|
199
|
+
sync_with Cloud::S3 do |s3|
|
|
197
200
|
s3.access_key_id = "my_access_key_id"
|
|
198
201
|
s3.secret_access_key = "my_secret_access_key"
|
|
199
202
|
s3.bucket = "my-bucket"
|
|
@@ -252,7 +255,7 @@ of `-aa`, `-ab` and `-ac`. These files will then be individually transfered. Thi
|
|
|
252
255
|
Amazon S3, Rackspace Cloud Files, or other 3rd party storage services which limit you to "5GB per file" uploads. So with
|
|
253
256
|
this, the backup file size is no longer a constraint.
|
|
254
257
|
|
|
255
|
-
Additionally we have also defined a **S3 Syncer** ( `sync_with S3` ), which does not follow the above process of
|
|
258
|
+
Additionally we have also defined a **S3 Syncer** ( `sync_with Cloud::S3` ), which does not follow the above process of
|
|
256
259
|
archiving/compression/encryption, but instead will directly sync the whole `videos` and `music` folder structures from
|
|
257
260
|
your machine to your Amazon S3 account. (very efficient and cost-effective since it will only transfer files that were
|
|
258
261
|
added/changed. Additionally, since we flagged it to 'mirror', it'll also remove files from S3 that no longer exist). If
|
data/backup.gemspec
CHANGED
data/lib/backup.rb
CHANGED
|
@@ -7,17 +7,8 @@ require 'tempfile'
|
|
|
7
7
|
require 'yaml'
|
|
8
8
|
require 'etc'
|
|
9
9
|
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
gem 'POpen4', '~> 0.1.4'
|
|
13
|
-
gem 'thor', '~> 0.14.6'
|
|
14
|
-
require 'popen4'
|
|
15
|
-
require 'thor'
|
|
16
|
-
rescue LoadError
|
|
17
|
-
puts "\nBackup requires Thor to load the CLI Utility (Command Line Interface Utility) and POpen4 to determine the status of the unix processes."
|
|
18
|
-
puts "Please install both the Thor and POpen4 libraries first:\n\ngem install thor -v '~> 0.14.6'\ngem install POpen4 -v '~> 0.1.4'"
|
|
19
|
-
exit 1
|
|
20
|
-
end
|
|
10
|
+
require 'open4'
|
|
11
|
+
require 'thor'
|
|
21
12
|
|
|
22
13
|
##
|
|
23
14
|
# The Backup Ruby Gem
|
|
@@ -33,25 +24,8 @@ module Backup
|
|
|
33
24
|
ENCRYPTOR_PATH = File.join(LIBRARY_PATH, 'encryptor')
|
|
34
25
|
NOTIFIER_PATH = File.join(LIBRARY_PATH, 'notifier')
|
|
35
26
|
SYNCER_PATH = File.join(LIBRARY_PATH, 'syncer')
|
|
36
|
-
CONFIGURATION_PATH = File.join(LIBRARY_PATH, 'configuration')
|
|
37
27
|
TEMPLATE_PATH = File.expand_path('../../templates', __FILE__)
|
|
38
28
|
|
|
39
|
-
##
|
|
40
|
-
# Autoload Backup base files
|
|
41
|
-
autoload :Model, File.join(LIBRARY_PATH, 'model')
|
|
42
|
-
autoload :Archive, File.join(LIBRARY_PATH, 'archive')
|
|
43
|
-
autoload :Packager, File.join(LIBRARY_PATH, 'packager')
|
|
44
|
-
autoload :Package, File.join(LIBRARY_PATH, 'package')
|
|
45
|
-
autoload :Cleaner, File.join(LIBRARY_PATH, 'cleaner')
|
|
46
|
-
autoload :Splitter, File.join(LIBRARY_PATH, 'splitter')
|
|
47
|
-
autoload :Config, File.join(LIBRARY_PATH, 'config')
|
|
48
|
-
autoload :Binder, File.join(LIBRARY_PATH, 'binder')
|
|
49
|
-
autoload :Template, File.join(LIBRARY_PATH, 'template')
|
|
50
|
-
autoload :Dependency, File.join(LIBRARY_PATH, 'dependency')
|
|
51
|
-
autoload :Logger, File.join(LIBRARY_PATH, 'logger')
|
|
52
|
-
autoload :Version, File.join(LIBRARY_PATH, 'version')
|
|
53
|
-
autoload :Errors, File.join(LIBRARY_PATH, 'errors')
|
|
54
|
-
|
|
55
29
|
##
|
|
56
30
|
# Autoload Backup CLI files
|
|
57
31
|
module CLI
|
|
@@ -78,10 +52,12 @@ module Backup
|
|
|
78
52
|
##
|
|
79
53
|
# Autoload Backup syncer files
|
|
80
54
|
module Syncer
|
|
81
|
-
autoload :Base,
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
55
|
+
autoload :Base, File.join(SYNCER_PATH, 'base')
|
|
56
|
+
module Cloud
|
|
57
|
+
autoload :Base, File.join(SYNCER_PATH, 'cloud', 'base')
|
|
58
|
+
autoload :CloudFiles, File.join(SYNCER_PATH, 'cloud', 'cloud_files')
|
|
59
|
+
autoload :S3, File.join(SYNCER_PATH, 'cloud', 's3')
|
|
60
|
+
end
|
|
85
61
|
module RSync
|
|
86
62
|
autoload :Base, File.join(SYNCER_PATH, 'rsync', 'base')
|
|
87
63
|
autoload :Local, File.join(SYNCER_PATH, 'rsync', 'local')
|
|
@@ -107,6 +83,7 @@ module Backup
|
|
|
107
83
|
autoload :Base, File.join(COMPRESSOR_PATH, 'base')
|
|
108
84
|
autoload :Gzip, File.join(COMPRESSOR_PATH, 'gzip')
|
|
109
85
|
autoload :Bzip2, File.join(COMPRESSOR_PATH, 'bzip2')
|
|
86
|
+
autoload :Custom, File.join(COMPRESSOR_PATH, 'custom')
|
|
110
87
|
autoload :Pbzip2, File.join(COMPRESSOR_PATH, 'pbzip2')
|
|
111
88
|
autoload :Lzma, File.join(COMPRESSOR_PATH, 'lzma')
|
|
112
89
|
end
|
|
@@ -127,75 +104,28 @@ module Backup
|
|
|
127
104
|
autoload :Mail, File.join(NOTIFIER_PATH, 'mail')
|
|
128
105
|
autoload :Twitter, File.join(NOTIFIER_PATH, 'twitter')
|
|
129
106
|
autoload :Campfire, File.join(NOTIFIER_PATH, 'campfire')
|
|
130
|
-
autoload :Presently, File.join(NOTIFIER_PATH, 'presently')
|
|
131
107
|
autoload :Prowl, File.join(NOTIFIER_PATH, 'prowl')
|
|
132
108
|
autoload :Hipchat, File.join(NOTIFIER_PATH, 'hipchat')
|
|
133
109
|
end
|
|
134
110
|
|
|
135
111
|
##
|
|
136
|
-
#
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
autoload :GPG, File.join(CONFIGURATION_PATH, 'encryptor', 'gpg')
|
|
155
|
-
end
|
|
156
|
-
|
|
157
|
-
module Compressor
|
|
158
|
-
autoload :Base, File.join(CONFIGURATION_PATH, 'compressor', 'base')
|
|
159
|
-
autoload :Gzip, File.join(CONFIGURATION_PATH, 'compressor', 'gzip')
|
|
160
|
-
autoload :Bzip2, File.join(CONFIGURATION_PATH, 'compressor', 'bzip2')
|
|
161
|
-
autoload :Pbzip2, File.join(CONFIGURATION_PATH, 'compressor', 'pbzip2')
|
|
162
|
-
autoload :Lzma, File.join(CONFIGURATION_PATH, 'compressor', 'lzma')
|
|
163
|
-
end
|
|
164
|
-
|
|
165
|
-
module Storage
|
|
166
|
-
autoload :Base, File.join(CONFIGURATION_PATH, 'storage', 'base')
|
|
167
|
-
autoload :S3, File.join(CONFIGURATION_PATH, 'storage', 's3')
|
|
168
|
-
autoload :CloudFiles, File.join(CONFIGURATION_PATH, 'storage', 'cloudfiles')
|
|
169
|
-
autoload :Ninefold, File.join(CONFIGURATION_PATH, 'storage', 'ninefold')
|
|
170
|
-
autoload :Dropbox, File.join(CONFIGURATION_PATH, 'storage', 'dropbox')
|
|
171
|
-
autoload :FTP, File.join(CONFIGURATION_PATH, 'storage', 'ftp')
|
|
172
|
-
autoload :SFTP, File.join(CONFIGURATION_PATH, 'storage', 'sftp')
|
|
173
|
-
autoload :SCP, File.join(CONFIGURATION_PATH, 'storage', 'scp')
|
|
174
|
-
autoload :RSync, File.join(CONFIGURATION_PATH, 'storage', 'rsync')
|
|
175
|
-
autoload :Local, File.join(CONFIGURATION_PATH, 'storage', 'local')
|
|
176
|
-
end
|
|
177
|
-
|
|
178
|
-
module Syncer
|
|
179
|
-
autoload :Base, File.join(CONFIGURATION_PATH, 'syncer', 'base')
|
|
180
|
-
autoload :Cloud, File.join(CONFIGURATION_PATH, 'syncer', 'cloud')
|
|
181
|
-
autoload :CloudFiles, File.join(CONFIGURATION_PATH, 'syncer', 'cloud_files')
|
|
182
|
-
autoload :S3, File.join(CONFIGURATION_PATH, 'syncer', 's3')
|
|
183
|
-
module RSync
|
|
184
|
-
autoload :Base, File.join(CONFIGURATION_PATH, 'syncer', 'rsync', 'base')
|
|
185
|
-
autoload :Local, File.join(CONFIGURATION_PATH, 'syncer', 'rsync', 'local')
|
|
186
|
-
autoload :Push, File.join(CONFIGURATION_PATH, 'syncer', 'rsync', 'push')
|
|
187
|
-
autoload :Pull, File.join(CONFIGURATION_PATH, 'syncer', 'rsync', 'pull')
|
|
188
|
-
end
|
|
189
|
-
end
|
|
190
|
-
|
|
191
|
-
module Database
|
|
192
|
-
autoload :Base, File.join(CONFIGURATION_PATH, 'database', 'base')
|
|
193
|
-
autoload :MySQL, File.join(CONFIGURATION_PATH, 'database', 'mysql')
|
|
194
|
-
autoload :PostgreSQL, File.join(CONFIGURATION_PATH, 'database', 'postgresql')
|
|
195
|
-
autoload :MongoDB, File.join(CONFIGURATION_PATH, 'database', 'mongodb')
|
|
196
|
-
autoload :Redis, File.join(CONFIGURATION_PATH, 'database', 'redis')
|
|
197
|
-
autoload :Riak, File.join(CONFIGURATION_PATH, 'database', 'riak')
|
|
198
|
-
end
|
|
199
|
-
end
|
|
112
|
+
# Require Backup base files
|
|
113
|
+
%w{
|
|
114
|
+
archive
|
|
115
|
+
binder
|
|
116
|
+
cleaner
|
|
117
|
+
config
|
|
118
|
+
configuration
|
|
119
|
+
dependency
|
|
120
|
+
errors
|
|
121
|
+
logger
|
|
122
|
+
model
|
|
123
|
+
package
|
|
124
|
+
packager
|
|
125
|
+
pipeline
|
|
126
|
+
splitter
|
|
127
|
+
template
|
|
128
|
+
version
|
|
129
|
+
}.each {|lib| require File.join(LIBRARY_PATH, lib) }
|
|
200
130
|
|
|
201
131
|
end
|
data/lib/backup/archive.rb
CHANGED
|
@@ -67,26 +67,34 @@ module Backup
|
|
|
67
67
|
# will be piped through the Compressor command and the file extension
|
|
68
68
|
# will be adjusted to indicate the type of compression used.
|
|
69
69
|
def perform!
|
|
70
|
-
Logger.message "#{ self.class } started
|
|
70
|
+
Logger.message "#{ self.class } has started archiving:\n" +
|
|
71
71
|
paths.map {|path| " #{path}" }.join("\n")
|
|
72
72
|
|
|
73
73
|
archive_path = File.join(Config.tmp_path, @model.trigger, 'archives')
|
|
74
74
|
FileUtils.mkdir_p(archive_path)
|
|
75
75
|
|
|
76
76
|
archive_ext = 'tar'
|
|
77
|
-
|
|
77
|
+
pipeline = Pipeline.new
|
|
78
|
+
|
|
79
|
+
pipeline << "#{ utility(:tar) } #{ tar_args } -cPf - " +
|
|
78
80
|
"#{ paths_to_exclude } #{ paths_to_package }"
|
|
79
81
|
|
|
80
82
|
if @model.compressor
|
|
81
83
|
@model.compressor.compress_with do |command, ext|
|
|
82
|
-
|
|
84
|
+
pipeline << command
|
|
83
85
|
archive_ext << ext
|
|
84
86
|
end
|
|
85
87
|
end
|
|
86
88
|
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
89
|
+
pipeline << "cat > '#{ File.join(archive_path, "#{name}.#{archive_ext}") }'"
|
|
90
|
+
pipeline.run
|
|
91
|
+
if pipeline.success?
|
|
92
|
+
Logger.message "#{ self.class } Complete!"
|
|
93
|
+
else
|
|
94
|
+
raise Errors::Archive::PipelineError,
|
|
95
|
+
"Failed to Create Backup Archive\n" +
|
|
96
|
+
pipeline.error_messages
|
|
97
|
+
end
|
|
90
98
|
end
|
|
91
99
|
|
|
92
100
|
private
|
data/lib/backup/cli/helpers.rb
CHANGED
|
@@ -6,45 +6,73 @@ module Backup
|
|
|
6
6
|
UTILITY = {}
|
|
7
7
|
|
|
8
8
|
##
|
|
9
|
-
# Runs a
|
|
10
|
-
# The STDOUT, STDERR and the returned exit code of the utility will be stored in the process_data Hash.
|
|
9
|
+
# Runs a system command
|
|
11
10
|
#
|
|
12
|
-
#
|
|
13
|
-
#
|
|
14
|
-
# you can pass in an array of exit codes to ignore (whitelist), for example:
|
|
11
|
+
# All messages generated by the command will be logged.
|
|
12
|
+
# Messages on STDERR will be logged as warnings.
|
|
15
13
|
#
|
|
16
|
-
#
|
|
14
|
+
# If the command fails to execute, or returns a non-zero exit status
|
|
15
|
+
# an Error will be raised.
|
|
17
16
|
#
|
|
18
|
-
#
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
17
|
+
# Returns nil
|
|
18
|
+
def run(command)
|
|
19
|
+
name = command_name(command)
|
|
20
|
+
Logger.message "Running system utility '#{ name }'..."
|
|
21
|
+
|
|
22
|
+
begin
|
|
23
|
+
out, err = '', ''
|
|
24
|
+
ps = Open4.popen4(command) do |pid, stdin, stdout, stderr|
|
|
25
|
+
stdin.close
|
|
26
|
+
out, err = stdout.read.strip, stderr.read.strip
|
|
27
|
+
end
|
|
28
|
+
rescue Exception => e
|
|
29
|
+
raise Errors::CLI::SystemCallError.wrap(e, <<-EOS)
|
|
30
|
+
Failed to execute system command on #{ RUBY_PLATFORM }
|
|
31
|
+
Command was: #{ command }
|
|
32
|
+
EOS
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
if ps.success?
|
|
36
|
+
unless out.empty?
|
|
37
|
+
Logger.message(
|
|
38
|
+
out.lines.map {|line| "#{ name }:STDOUT: #{ line }" }.join
|
|
39
|
+
)
|
|
40
|
+
end
|
|
24
41
|
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
process_data[:ignore_exit_codes] = ((options[:ignore_exit_codes] || Array.new) << 0).uniq
|
|
42
|
+
unless err.empty?
|
|
43
|
+
Logger.warn(
|
|
44
|
+
err.lines.map {|line| "#{ name }:STDERR: #{ line }" }.join
|
|
45
|
+
)
|
|
46
|
+
end
|
|
31
47
|
|
|
32
|
-
|
|
33
|
-
|
|
48
|
+
return nil
|
|
49
|
+
else
|
|
50
|
+
raise Errors::CLI::SystemCallError, <<-EOS
|
|
51
|
+
'#{ name }' Failed on #{ RUBY_PLATFORM }
|
|
52
|
+
The following information should help to determine the problem:
|
|
53
|
+
Command was: #{ command }
|
|
54
|
+
Exit Status: #{ ps.exitstatus }
|
|
55
|
+
STDOUT Messages: #{ out.empty? ? 'None' : "\n#{ out }" }
|
|
56
|
+
STDERR Messages: #{ err.empty? ? 'None' : "\n#{ err }" }
|
|
57
|
+
EOS
|
|
58
|
+
end
|
|
34
59
|
end
|
|
35
60
|
|
|
61
|
+
|
|
36
62
|
##
|
|
37
63
|
# Returns the full path to the specified utility.
|
|
38
64
|
# Raises an error if utility can not be found in the system's $PATH
|
|
39
65
|
def utility(name)
|
|
40
|
-
|
|
66
|
+
name = name.to_s.strip
|
|
67
|
+
raise Errors::CLI::UtilityNotFoundError,
|
|
68
|
+
'Utility Name Empty' if name.empty?
|
|
69
|
+
|
|
70
|
+
path = UTILITY[name] || %x[which #{ name } 2>/dev/null].chomp
|
|
41
71
|
if path.empty?
|
|
42
72
|
raise Errors::CLI::UtilityNotFoundError, <<-EOS
|
|
43
|
-
|
|
73
|
+
Could not locate '#{ name }'.
|
|
44
74
|
Make sure the specified utility is installed
|
|
45
75
|
and available in your system's $PATH.
|
|
46
|
-
If this is a database utility, you may need to specify the full path
|
|
47
|
-
using the Database's '<utility_name>_utility' configuration setting.
|
|
48
76
|
EOS
|
|
49
77
|
end
|
|
50
78
|
UTILITY[name] = path
|
|
@@ -58,31 +86,6 @@ module Backup
|
|
|
58
86
|
command.split('/')[-1]
|
|
59
87
|
end
|
|
60
88
|
|
|
61
|
-
##
|
|
62
|
-
# Inspects the exit code returned from the POpen4 child process. If the exit code isn't listed
|
|
63
|
-
# in the process_data[:ignore_exit_codes] array, an exception will be raised, aborting the backup process.
|
|
64
|
-
#
|
|
65
|
-
# Information regarding the error ( EXIT CODE and STDERR ) will be returned to the shell so the user can
|
|
66
|
-
# investigate the issue.
|
|
67
|
-
#
|
|
68
|
-
# raises Backup::Errors::CLI::SystemCallError
|
|
69
|
-
def raise_if_command_failed!(utility, process_data)
|
|
70
|
-
unless process_data[:ignore_exit_codes].include?(process_data[:status].to_i)
|
|
71
|
-
|
|
72
|
-
stderr = process_data[:stderr].empty? ?
|
|
73
|
-
nil : "STDERR:\n#{process_data[:stderr]}\n"
|
|
74
|
-
stdout = process_data[:stdout].empty? ?
|
|
75
|
-
nil : "STDOUT:\n#{process_data[:stdout]}\n"
|
|
76
|
-
|
|
77
|
-
raise Errors::CLI::SystemCallError, <<-EOS
|
|
78
|
-
Failed to run #{utility} on #{RUBY_PLATFORM}
|
|
79
|
-
The following information should help to determine the problem:
|
|
80
|
-
Exit Code: #{process_data[:status]}
|
|
81
|
-
#{stderr}#{stdout}
|
|
82
|
-
EOS
|
|
83
|
-
end
|
|
84
|
-
end
|
|
85
|
-
|
|
86
89
|
end
|
|
87
90
|
end
|
|
88
91
|
end
|