backup 3.0.23 → 3.0.24
Sign up to get free protection for your applications and to get access to all the features.
- data/Gemfile.lock +42 -45
- data/Guardfile +7 -4
- data/README.md +10 -7
- data/backup.gemspec +2 -2
- data/lib/backup.rb +27 -97
- data/lib/backup/archive.rb +14 -6
- data/lib/backup/cli/helpers.rb +52 -49
- data/lib/backup/cli/utility.rb +9 -1
- data/lib/backup/compressor/base.rb +10 -4
- data/lib/backup/compressor/bzip2.rb +22 -26
- data/lib/backup/compressor/custom.rb +53 -0
- data/lib/backup/compressor/gzip.rb +22 -23
- data/lib/backup/compressor/lzma.rb +15 -13
- data/lib/backup/compressor/pbzip2.rb +20 -17
- data/lib/backup/config.rb +6 -3
- data/lib/backup/configuration.rb +33 -0
- data/lib/backup/configuration/helpers.rb +114 -28
- data/lib/backup/configuration/store.rb +24 -0
- data/lib/backup/database/base.rb +0 -6
- data/lib/backup/database/mongodb.rb +27 -11
- data/lib/backup/database/mysql.rb +19 -14
- data/lib/backup/database/postgresql.rb +16 -11
- data/lib/backup/database/redis.rb +7 -11
- data/lib/backup/database/riak.rb +3 -6
- data/lib/backup/dependency.rb +5 -11
- data/lib/backup/model.rb +14 -5
- data/lib/backup/notifier/campfire.rb +3 -16
- data/lib/backup/notifier/hipchat.rb +1 -7
- data/lib/backup/notifier/mail.rb +1 -1
- data/lib/backup/packager.rb +29 -19
- data/lib/backup/pipeline.rb +110 -0
- data/lib/backup/storage/dropbox.rb +4 -7
- data/lib/backup/syncer/base.rb +8 -4
- data/lib/backup/syncer/cloud/base.rb +247 -0
- data/lib/backup/syncer/cloud/cloud_files.rb +78 -0
- data/lib/backup/syncer/cloud/s3.rb +68 -0
- data/lib/backup/syncer/rsync/base.rb +1 -4
- data/lib/backup/syncer/rsync/local.rb +9 -5
- data/lib/backup/syncer/rsync/pull.rb +1 -1
- data/lib/backup/syncer/rsync/push.rb +10 -5
- data/lib/backup/version.rb +1 -1
- data/spec-live/.gitignore +6 -0
- data/spec-live/README +7 -0
- data/spec-live/backups/config.rb +153 -0
- data/spec-live/backups/config.yml.template +43 -0
- data/spec-live/compressor/custom_spec.rb +30 -0
- data/spec-live/compressor/gzip_spec.rb +30 -0
- data/spec-live/notifier/mail_spec.rb +85 -0
- data/spec-live/spec_helper.rb +85 -0
- data/spec-live/storage/dropbox_spec.rb +151 -0
- data/spec-live/storage/local_spec.rb +83 -0
- data/spec-live/storage/scp_spec.rb +193 -0
- data/spec-live/syncer/cloud/s3_spec.rb +124 -0
- data/spec/archive_spec.rb +86 -31
- data/spec/cleaner_spec.rb +8 -0
- data/spec/cli/helpers_spec.rb +200 -75
- data/spec/cli/utility_spec.rb +11 -3
- data/spec/compressor/base_spec.rb +31 -10
- data/spec/compressor/bzip2_spec.rb +212 -57
- data/spec/compressor/custom_spec.rb +106 -0
- data/spec/compressor/gzip_spec.rb +212 -57
- data/spec/compressor/lzma_spec.rb +75 -35
- data/spec/compressor/pbzip2_spec.rb +93 -52
- data/spec/configuration/helpers_spec.rb +406 -0
- data/spec/configuration/store_spec.rb +39 -0
- data/spec/configuration_spec.rb +62 -0
- data/spec/database/base_spec.rb +19 -10
- data/spec/database/mongodb_spec.rb +195 -70
- data/spec/database/mysql_spec.rb +183 -64
- data/spec/database/postgresql_spec.rb +167 -53
- data/spec/database/redis_spec.rb +121 -46
- data/spec/database/riak_spec.rb +96 -27
- data/spec/dependency_spec.rb +2 -0
- data/spec/encryptor/base_spec.rb +10 -0
- data/spec/encryptor/gpg_spec.rb +29 -13
- data/spec/encryptor/open_ssl_spec.rb +40 -21
- data/spec/logger_spec.rb +4 -0
- data/spec/model_spec.rb +19 -2
- data/spec/notifier/base_spec.rb +32 -17
- data/spec/notifier/campfire_spec.rb +63 -45
- data/spec/notifier/hipchat_spec.rb +79 -56
- data/spec/notifier/mail_spec.rb +82 -46
- data/spec/notifier/prowl_spec.rb +53 -32
- data/spec/notifier/twitter_spec.rb +62 -41
- data/spec/packager_spec.rb +95 -36
- data/spec/pipeline_spec.rb +259 -0
- data/spec/spec_helper.rb +6 -5
- data/spec/storage/base_spec.rb +61 -41
- data/spec/storage/cloudfiles_spec.rb +69 -45
- data/spec/storage/dropbox_spec.rb +158 -36
- data/spec/storage/ftp_spec.rb +69 -45
- data/spec/storage/local_spec.rb +47 -23
- data/spec/storage/ninefold_spec.rb +55 -31
- data/spec/storage/rsync_spec.rb +67 -50
- data/spec/storage/s3_spec.rb +65 -41
- data/spec/storage/scp_spec.rb +65 -41
- data/spec/storage/sftp_spec.rb +65 -41
- data/spec/syncer/base_spec.rb +91 -4
- data/spec/syncer/cloud/base_spec.rb +511 -0
- data/spec/syncer/cloud/cloud_files_spec.rb +181 -0
- data/spec/syncer/cloud/s3_spec.rb +174 -0
- data/spec/syncer/rsync/base_spec.rb +46 -66
- data/spec/syncer/rsync/local_spec.rb +55 -26
- data/spec/syncer/rsync/pull_spec.rb +15 -4
- data/spec/syncer/rsync/push_spec.rb +59 -52
- data/templates/cli/utility/compressor/bzip2 +1 -4
- data/templates/cli/utility/compressor/custom +11 -0
- data/templates/cli/utility/compressor/gzip +1 -4
- data/templates/cli/utility/compressor/lzma +3 -0
- data/templates/cli/utility/compressor/pbzip2 +3 -0
- data/templates/cli/utility/database/mysql +4 -1
- data/templates/cli/utility/syncer/cloud_files +17 -19
- data/templates/cli/utility/syncer/s3 +18 -20
- metadata +38 -92
- data/lib/backup/configuration/base.rb +0 -15
- data/lib/backup/configuration/compressor/base.rb +0 -9
- data/lib/backup/configuration/compressor/bzip2.rb +0 -23
- data/lib/backup/configuration/compressor/gzip.rb +0 -23
- data/lib/backup/configuration/compressor/lzma.rb +0 -23
- data/lib/backup/configuration/compressor/pbzip2.rb +0 -28
- data/lib/backup/configuration/database/base.rb +0 -19
- data/lib/backup/configuration/database/mongodb.rb +0 -49
- data/lib/backup/configuration/database/mysql.rb +0 -42
- data/lib/backup/configuration/database/postgresql.rb +0 -41
- data/lib/backup/configuration/database/redis.rb +0 -39
- data/lib/backup/configuration/database/riak.rb +0 -29
- data/lib/backup/configuration/encryptor/base.rb +0 -9
- data/lib/backup/configuration/encryptor/gpg.rb +0 -17
- data/lib/backup/configuration/encryptor/open_ssl.rb +0 -32
- data/lib/backup/configuration/notifier/base.rb +0 -28
- data/lib/backup/configuration/notifier/campfire.rb +0 -25
- data/lib/backup/configuration/notifier/hipchat.rb +0 -41
- data/lib/backup/configuration/notifier/mail.rb +0 -112
- data/lib/backup/configuration/notifier/presently.rb +0 -25
- data/lib/backup/configuration/notifier/prowl.rb +0 -23
- data/lib/backup/configuration/notifier/twitter.rb +0 -21
- data/lib/backup/configuration/storage/base.rb +0 -18
- data/lib/backup/configuration/storage/cloudfiles.rb +0 -25
- data/lib/backup/configuration/storage/dropbox.rb +0 -58
- data/lib/backup/configuration/storage/ftp.rb +0 -29
- data/lib/backup/configuration/storage/local.rb +0 -17
- data/lib/backup/configuration/storage/ninefold.rb +0 -20
- data/lib/backup/configuration/storage/rsync.rb +0 -29
- data/lib/backup/configuration/storage/s3.rb +0 -25
- data/lib/backup/configuration/storage/scp.rb +0 -25
- data/lib/backup/configuration/storage/sftp.rb +0 -25
- data/lib/backup/configuration/syncer/base.rb +0 -10
- data/lib/backup/configuration/syncer/cloud.rb +0 -23
- data/lib/backup/configuration/syncer/cloud_files.rb +0 -30
- data/lib/backup/configuration/syncer/rsync/base.rb +0 -28
- data/lib/backup/configuration/syncer/rsync/local.rb +0 -11
- data/lib/backup/configuration/syncer/rsync/pull.rb +0 -11
- data/lib/backup/configuration/syncer/rsync/push.rb +0 -31
- data/lib/backup/configuration/syncer/s3.rb +0 -23
- data/lib/backup/notifier/presently.rb +0 -88
- data/lib/backup/syncer/cloud.rb +0 -187
- data/lib/backup/syncer/cloud_files.rb +0 -56
- data/lib/backup/syncer/s3.rb +0 -47
- data/spec/configuration/base_spec.rb +0 -35
- data/spec/configuration/compressor/bzip2_spec.rb +0 -29
- data/spec/configuration/compressor/gzip_spec.rb +0 -29
- data/spec/configuration/compressor/lzma_spec.rb +0 -29
- data/spec/configuration/compressor/pbzip2_spec.rb +0 -32
- data/spec/configuration/database/base_spec.rb +0 -17
- data/spec/configuration/database/mongodb_spec.rb +0 -56
- data/spec/configuration/database/mysql_spec.rb +0 -53
- data/spec/configuration/database/postgresql_spec.rb +0 -53
- data/spec/configuration/database/redis_spec.rb +0 -50
- data/spec/configuration/database/riak_spec.rb +0 -35
- data/spec/configuration/encryptor/gpg_spec.rb +0 -26
- data/spec/configuration/encryptor/open_ssl_spec.rb +0 -35
- data/spec/configuration/notifier/base_spec.rb +0 -32
- data/spec/configuration/notifier/campfire_spec.rb +0 -32
- data/spec/configuration/notifier/hipchat_spec.rb +0 -44
- data/spec/configuration/notifier/mail_spec.rb +0 -71
- data/spec/configuration/notifier/presently_spec.rb +0 -35
- data/spec/configuration/notifier/prowl_spec.rb +0 -29
- data/spec/configuration/notifier/twitter_spec.rb +0 -35
- data/spec/configuration/storage/cloudfiles_spec.rb +0 -41
- data/spec/configuration/storage/dropbox_spec.rb +0 -38
- data/spec/configuration/storage/ftp_spec.rb +0 -44
- data/spec/configuration/storage/local_spec.rb +0 -29
- data/spec/configuration/storage/ninefold_spec.rb +0 -32
- data/spec/configuration/storage/rsync_spec.rb +0 -41
- data/spec/configuration/storage/s3_spec.rb +0 -38
- data/spec/configuration/storage/scp_spec.rb +0 -41
- data/spec/configuration/storage/sftp_spec.rb +0 -41
- data/spec/configuration/syncer/cloud_files_spec.rb +0 -44
- data/spec/configuration/syncer/rsync/base_spec.rb +0 -33
- data/spec/configuration/syncer/rsync/local_spec.rb +0 -10
- data/spec/configuration/syncer/rsync/pull_spec.rb +0 -10
- data/spec/configuration/syncer/rsync/push_spec.rb +0 -43
- data/spec/configuration/syncer/s3_spec.rb +0 -38
- data/spec/notifier/presently_spec.rb +0 -181
- data/spec/syncer/cloud_files_spec.rb +0 -192
- data/spec/syncer/s3_spec.rb +0 -192
- data/templates/cli/utility/notifier/presently +0 -13
data/Gemfile.lock
CHANGED
@@ -2,29 +2,25 @@ PATH
|
|
2
2
|
remote: .
|
3
3
|
specs:
|
4
4
|
backup (3.0.23)
|
5
|
-
|
5
|
+
open4 (~> 1.3.0)
|
6
6
|
thor (~> 0.14.6)
|
7
7
|
|
8
8
|
GEM
|
9
9
|
remote: http://rubygems.org/
|
10
10
|
specs:
|
11
|
-
|
12
|
-
|
13
|
-
open4
|
14
|
-
Platform (0.4.0)
|
15
|
-
activesupport (3.1.3)
|
11
|
+
activesupport (3.2.2)
|
12
|
+
i18n (~> 0.6)
|
16
13
|
multi_json (~> 1.0)
|
17
|
-
addressable (2.2.
|
14
|
+
addressable (2.2.7)
|
18
15
|
builder (3.0.0)
|
19
|
-
crack (0.1.8)
|
20
16
|
diff-lcs (1.1.3)
|
21
|
-
dropbox-sdk (1.
|
17
|
+
dropbox-sdk (1.2)
|
22
18
|
json
|
23
|
-
excon (0.9.
|
24
|
-
faraday (0.7.
|
25
|
-
addressable (~> 2.2
|
26
|
-
multipart-post (~> 1.1
|
27
|
-
rack (
|
19
|
+
excon (0.9.6)
|
20
|
+
faraday (0.7.6)
|
21
|
+
addressable (~> 2.2)
|
22
|
+
multipart-post (~> 1.1)
|
23
|
+
rack (~> 1.1)
|
28
24
|
ffi (1.0.11)
|
29
25
|
fog (1.1.2)
|
30
26
|
builder
|
@@ -37,56 +33,58 @@ GEM
|
|
37
33
|
nokogiri (~> 1.5.0)
|
38
34
|
ruby-hmac
|
39
35
|
formatador (0.2.1)
|
40
|
-
fuubar (0.0
|
36
|
+
fuubar (1.0.0)
|
41
37
|
rspec (~> 2.0)
|
42
|
-
rspec-instafail (~> 0.
|
38
|
+
rspec-instafail (~> 0.2.0)
|
43
39
|
ruby-progressbar (~> 0.0.10)
|
44
40
|
growl (1.0.3)
|
45
|
-
guard (0.
|
41
|
+
guard (1.0.1)
|
46
42
|
ffi (>= 0.5.0)
|
47
43
|
thor (~> 0.14.6)
|
48
44
|
guard-rspec (0.6.0)
|
49
45
|
guard (>= 0.10.0)
|
50
46
|
hipchat (0.4.1)
|
51
47
|
httparty
|
52
|
-
httparty (0.
|
53
|
-
|
48
|
+
httparty (0.8.1)
|
49
|
+
multi_json
|
50
|
+
multi_xml
|
54
51
|
i18n (0.6.0)
|
55
|
-
json (1.5
|
56
|
-
libnotify (0.7.
|
57
|
-
mail (2.4.
|
52
|
+
json (1.6.5)
|
53
|
+
libnotify (0.7.2)
|
54
|
+
mail (2.4.4)
|
58
55
|
i18n (>= 0.4.0)
|
59
56
|
mime-types (~> 1.16)
|
60
57
|
treetop (~> 1.4.8)
|
61
58
|
metaclass (0.0.1)
|
62
59
|
mime-types (1.17.2)
|
63
|
-
mocha (0.10.
|
60
|
+
mocha (0.10.5)
|
64
61
|
metaclass (~> 0.0.1)
|
65
62
|
multi_json (1.0.4)
|
66
|
-
|
63
|
+
multi_xml (0.4.2)
|
64
|
+
multipart-post (1.1.5)
|
67
65
|
net-scp (1.0.4)
|
68
66
|
net-ssh (>= 1.99.1)
|
69
67
|
net-sftp (2.0.5)
|
70
68
|
net-ssh (>= 2.0.9)
|
71
|
-
net-ssh (2.
|
72
|
-
nokogiri (1.5.
|
69
|
+
net-ssh (2.3.0)
|
70
|
+
nokogiri (1.5.2)
|
73
71
|
open4 (1.3.0)
|
74
|
-
parallel (0.5.
|
72
|
+
parallel (0.5.16)
|
75
73
|
polyglot (0.3.3)
|
76
74
|
prowler (1.3.1)
|
77
|
-
rack (1.4.
|
78
|
-
rb-fsevent (0.
|
75
|
+
rack (1.4.1)
|
76
|
+
rb-fsevent (0.9.0)
|
79
77
|
rb-inotify (0.8.8)
|
80
78
|
ffi (>= 0.5.0)
|
81
|
-
rspec (2.
|
82
|
-
rspec-core (~> 2.
|
83
|
-
rspec-expectations (~> 2.
|
84
|
-
rspec-mocks (~> 2.
|
85
|
-
rspec-core (2.
|
86
|
-
rspec-expectations (2.
|
87
|
-
diff-lcs (~> 1.1.
|
88
|
-
rspec-instafail (0.
|
89
|
-
rspec-mocks (2.
|
79
|
+
rspec (2.9.0)
|
80
|
+
rspec-core (~> 2.9.0)
|
81
|
+
rspec-expectations (~> 2.9.0)
|
82
|
+
rspec-mocks (~> 2.9.0)
|
83
|
+
rspec-core (2.9.0)
|
84
|
+
rspec-expectations (2.9.0)
|
85
|
+
diff-lcs (~> 1.1.3)
|
86
|
+
rspec-instafail (0.2.2)
|
87
|
+
rspec-mocks (2.9.0)
|
90
88
|
ruby-hmac (0.4.0)
|
91
89
|
ruby-progressbar (0.0.10)
|
92
90
|
simple_oauth (0.1.5)
|
@@ -95,7 +93,7 @@ GEM
|
|
95
93
|
treetop (1.4.10)
|
96
94
|
polyglot
|
97
95
|
polyglot (>= 0.3.1)
|
98
|
-
twitter (2.
|
96
|
+
twitter (2.1.1)
|
99
97
|
activesupport (>= 2.3.9, < 4)
|
100
98
|
faraday (~> 0.7)
|
101
99
|
multi_json (~> 1.0)
|
@@ -106,21 +104,20 @@ PLATFORMS
|
|
106
104
|
|
107
105
|
DEPENDENCIES
|
108
106
|
backup!
|
109
|
-
dropbox-sdk (~> 1.
|
110
|
-
fog (
|
107
|
+
dropbox-sdk (~> 1.2.0)
|
108
|
+
fog (~> 1.1.0)
|
111
109
|
fuubar
|
112
110
|
growl
|
113
111
|
guard
|
114
112
|
guard-rspec
|
115
113
|
hipchat (~> 0.4.1)
|
116
|
-
httparty (~> 0.
|
117
|
-
json (~> 1.5.1)
|
114
|
+
httparty (~> 0.8.1)
|
118
115
|
libnotify
|
119
|
-
mail (
|
116
|
+
mail (~> 2.4.0)
|
120
117
|
mocha
|
121
118
|
net-scp (~> 1.0.4)
|
122
119
|
net-sftp (~> 2.0.5)
|
123
|
-
net-ssh (~> 2.
|
120
|
+
net-ssh (~> 2.3.0)
|
124
121
|
parallel (~> 0.5.12)
|
125
122
|
prowler (>= 1.3.1)
|
126
123
|
rb-fsevent
|
data/Guardfile
CHANGED
@@ -12,10 +12,13 @@
|
|
12
12
|
guard "rspec",
|
13
13
|
:version => 2,
|
14
14
|
:rvm => ["1.9.3", "1.9.2", "1.8.7"],
|
15
|
-
:
|
16
|
-
:
|
15
|
+
:cli => "--color --format Fuubar",
|
16
|
+
:notification => false,
|
17
|
+
:all_after_pass => false,
|
18
|
+
:all_on_start => false do
|
17
19
|
|
20
|
+
watch("lib/backup.rb") { "spec" }
|
21
|
+
watch("spec/spec_helper.rb") { "spec" }
|
22
|
+
watch(%r{^lib/backup/(.+)\.rb}) {|m| "spec/#{ m[1] }_spec.rb" }
|
18
23
|
watch(%r{^spec/.+_spec\.rb})
|
19
|
-
watch(%r{^lib/(.+)\.rb}) { "spec" }
|
20
|
-
watch("spec/spec_helper.rb") { "spec" }
|
21
24
|
end
|
data/README.md
CHANGED
@@ -69,6 +69,9 @@ Below you find a list of components that Backup currently supports. If you'd lik
|
|
69
69
|
- Dropbox Web Service
|
70
70
|
- Remote Servers *(Only Protocols: FTP, SFTP, SCP)*
|
71
71
|
- Local Storage
|
72
|
+
|
73
|
+
[Cycling Wiki Page](https://github.com/meskyanichi/backup/wiki/Cycling)
|
74
|
+
|
72
75
|
- **Backup Splitting, applies to:**
|
73
76
|
- Amazon Simple Storage Service (S3)
|
74
77
|
- Rackspace Cloud Files (Mosso)
|
@@ -76,13 +79,12 @@ Below you find a list of components that Backup currently supports. If you'd lik
|
|
76
79
|
- Dropbox Web Service
|
77
80
|
- Remote Servers *(Only Protocols: FTP, SFTP, SCP)*
|
78
81
|
- Local Storage
|
79
|
-
- **Incremental Backups, applies to:**
|
80
|
-
- Remote Servers *(Only Protocols: RSync)*
|
81
|
-
|
82
|
-
[Cycling Wiki Page](https://github.com/meskyanichi/backup/wiki/Cycling)
|
83
82
|
|
84
83
|
[Splitter Wiki Page](https://github.com/meskyanichi/backup/wiki/Splitter)
|
85
84
|
|
85
|
+
- **Incremental Backups, applies to:**
|
86
|
+
- Remote Servers *(Only Protocols: RSync)*
|
87
|
+
|
86
88
|
### Syncers
|
87
89
|
|
88
90
|
- RSync (Push, Pull and Local)
|
@@ -128,7 +130,8 @@ Below you find a list of components that Backup currently supports. If you'd lik
|
|
128
130
|
A sample Backup configuration file
|
129
131
|
----------------------------------
|
130
132
|
|
131
|
-
This is a Backup configuration file. Check it out and read the explanation below.
|
133
|
+
This is a Backup configuration file. Check it out and read the explanation below.
|
134
|
+
Backup has a [great wiki](https://github.com/meskyanichi/backup/wiki) which explains each component of Backup in detail.
|
132
135
|
|
133
136
|
``` rb
|
134
137
|
Backup::Model.new(:sample_backup, 'A sample backup configuration') do
|
@@ -193,7 +196,7 @@ Backup::Model.new(:sample_backup, 'A sample backup configuration') do
|
|
193
196
|
s3.keep = 20
|
194
197
|
end
|
195
198
|
|
196
|
-
sync_with S3 do |s3|
|
199
|
+
sync_with Cloud::S3 do |s3|
|
197
200
|
s3.access_key_id = "my_access_key_id"
|
198
201
|
s3.secret_access_key = "my_secret_access_key"
|
199
202
|
s3.bucket = "my-bucket"
|
@@ -252,7 +255,7 @@ of `-aa`, `-ab` and `-ac`. These files will then be individually transfered. Thi
|
|
252
255
|
Amazon S3, Rackspace Cloud Files, or other 3rd party storage services which limit you to "5GB per file" uploads. So with
|
253
256
|
this, the backup file size is no longer a constraint.
|
254
257
|
|
255
|
-
Additionally we have also defined a **S3 Syncer** ( `sync_with S3` ), which does not follow the above process of
|
258
|
+
Additionally we have also defined a **S3 Syncer** ( `sync_with Cloud::S3` ), which does not follow the above process of
|
256
259
|
archiving/compression/encryption, but instead will directly sync the whole `videos` and `music` folder structures from
|
257
260
|
your machine to your Amazon S3 account. (very efficient and cost-effective since it will only transfer files that were
|
258
261
|
added/changed. Additionally, since we flagged it to 'mirror', it'll also remove files from S3 that no longer exist). If
|
data/backup.gemspec
CHANGED
data/lib/backup.rb
CHANGED
@@ -7,17 +7,8 @@ require 'tempfile'
|
|
7
7
|
require 'yaml'
|
8
8
|
require 'etc'
|
9
9
|
|
10
|
-
|
11
|
-
|
12
|
-
gem 'POpen4', '~> 0.1.4'
|
13
|
-
gem 'thor', '~> 0.14.6'
|
14
|
-
require 'popen4'
|
15
|
-
require 'thor'
|
16
|
-
rescue LoadError
|
17
|
-
puts "\nBackup requires Thor to load the CLI Utility (Command Line Interface Utility) and POpen4 to determine the status of the unix processes."
|
18
|
-
puts "Please install both the Thor and POpen4 libraries first:\n\ngem install thor -v '~> 0.14.6'\ngem install POpen4 -v '~> 0.1.4'"
|
19
|
-
exit 1
|
20
|
-
end
|
10
|
+
require 'open4'
|
11
|
+
require 'thor'
|
21
12
|
|
22
13
|
##
|
23
14
|
# The Backup Ruby Gem
|
@@ -33,25 +24,8 @@ module Backup
|
|
33
24
|
ENCRYPTOR_PATH = File.join(LIBRARY_PATH, 'encryptor')
|
34
25
|
NOTIFIER_PATH = File.join(LIBRARY_PATH, 'notifier')
|
35
26
|
SYNCER_PATH = File.join(LIBRARY_PATH, 'syncer')
|
36
|
-
CONFIGURATION_PATH = File.join(LIBRARY_PATH, 'configuration')
|
37
27
|
TEMPLATE_PATH = File.expand_path('../../templates', __FILE__)
|
38
28
|
|
39
|
-
##
|
40
|
-
# Autoload Backup base files
|
41
|
-
autoload :Model, File.join(LIBRARY_PATH, 'model')
|
42
|
-
autoload :Archive, File.join(LIBRARY_PATH, 'archive')
|
43
|
-
autoload :Packager, File.join(LIBRARY_PATH, 'packager')
|
44
|
-
autoload :Package, File.join(LIBRARY_PATH, 'package')
|
45
|
-
autoload :Cleaner, File.join(LIBRARY_PATH, 'cleaner')
|
46
|
-
autoload :Splitter, File.join(LIBRARY_PATH, 'splitter')
|
47
|
-
autoload :Config, File.join(LIBRARY_PATH, 'config')
|
48
|
-
autoload :Binder, File.join(LIBRARY_PATH, 'binder')
|
49
|
-
autoload :Template, File.join(LIBRARY_PATH, 'template')
|
50
|
-
autoload :Dependency, File.join(LIBRARY_PATH, 'dependency')
|
51
|
-
autoload :Logger, File.join(LIBRARY_PATH, 'logger')
|
52
|
-
autoload :Version, File.join(LIBRARY_PATH, 'version')
|
53
|
-
autoload :Errors, File.join(LIBRARY_PATH, 'errors')
|
54
|
-
|
55
29
|
##
|
56
30
|
# Autoload Backup CLI files
|
57
31
|
module CLI
|
@@ -78,10 +52,12 @@ module Backup
|
|
78
52
|
##
|
79
53
|
# Autoload Backup syncer files
|
80
54
|
module Syncer
|
81
|
-
autoload :Base,
|
82
|
-
|
83
|
-
|
84
|
-
|
55
|
+
autoload :Base, File.join(SYNCER_PATH, 'base')
|
56
|
+
module Cloud
|
57
|
+
autoload :Base, File.join(SYNCER_PATH, 'cloud', 'base')
|
58
|
+
autoload :CloudFiles, File.join(SYNCER_PATH, 'cloud', 'cloud_files')
|
59
|
+
autoload :S3, File.join(SYNCER_PATH, 'cloud', 's3')
|
60
|
+
end
|
85
61
|
module RSync
|
86
62
|
autoload :Base, File.join(SYNCER_PATH, 'rsync', 'base')
|
87
63
|
autoload :Local, File.join(SYNCER_PATH, 'rsync', 'local')
|
@@ -107,6 +83,7 @@ module Backup
|
|
107
83
|
autoload :Base, File.join(COMPRESSOR_PATH, 'base')
|
108
84
|
autoload :Gzip, File.join(COMPRESSOR_PATH, 'gzip')
|
109
85
|
autoload :Bzip2, File.join(COMPRESSOR_PATH, 'bzip2')
|
86
|
+
autoload :Custom, File.join(COMPRESSOR_PATH, 'custom')
|
110
87
|
autoload :Pbzip2, File.join(COMPRESSOR_PATH, 'pbzip2')
|
111
88
|
autoload :Lzma, File.join(COMPRESSOR_PATH, 'lzma')
|
112
89
|
end
|
@@ -127,75 +104,28 @@ module Backup
|
|
127
104
|
autoload :Mail, File.join(NOTIFIER_PATH, 'mail')
|
128
105
|
autoload :Twitter, File.join(NOTIFIER_PATH, 'twitter')
|
129
106
|
autoload :Campfire, File.join(NOTIFIER_PATH, 'campfire')
|
130
|
-
autoload :Presently, File.join(NOTIFIER_PATH, 'presently')
|
131
107
|
autoload :Prowl, File.join(NOTIFIER_PATH, 'prowl')
|
132
108
|
autoload :Hipchat, File.join(NOTIFIER_PATH, 'hipchat')
|
133
109
|
end
|
134
110
|
|
135
111
|
##
|
136
|
-
#
|
137
|
-
|
138
|
-
|
139
|
-
|
140
|
-
|
141
|
-
|
142
|
-
|
143
|
-
|
144
|
-
|
145
|
-
|
146
|
-
|
147
|
-
|
148
|
-
|
149
|
-
|
150
|
-
|
151
|
-
|
152
|
-
|
153
|
-
|
154
|
-
autoload :GPG, File.join(CONFIGURATION_PATH, 'encryptor', 'gpg')
|
155
|
-
end
|
156
|
-
|
157
|
-
module Compressor
|
158
|
-
autoload :Base, File.join(CONFIGURATION_PATH, 'compressor', 'base')
|
159
|
-
autoload :Gzip, File.join(CONFIGURATION_PATH, 'compressor', 'gzip')
|
160
|
-
autoload :Bzip2, File.join(CONFIGURATION_PATH, 'compressor', 'bzip2')
|
161
|
-
autoload :Pbzip2, File.join(CONFIGURATION_PATH, 'compressor', 'pbzip2')
|
162
|
-
autoload :Lzma, File.join(CONFIGURATION_PATH, 'compressor', 'lzma')
|
163
|
-
end
|
164
|
-
|
165
|
-
module Storage
|
166
|
-
autoload :Base, File.join(CONFIGURATION_PATH, 'storage', 'base')
|
167
|
-
autoload :S3, File.join(CONFIGURATION_PATH, 'storage', 's3')
|
168
|
-
autoload :CloudFiles, File.join(CONFIGURATION_PATH, 'storage', 'cloudfiles')
|
169
|
-
autoload :Ninefold, File.join(CONFIGURATION_PATH, 'storage', 'ninefold')
|
170
|
-
autoload :Dropbox, File.join(CONFIGURATION_PATH, 'storage', 'dropbox')
|
171
|
-
autoload :FTP, File.join(CONFIGURATION_PATH, 'storage', 'ftp')
|
172
|
-
autoload :SFTP, File.join(CONFIGURATION_PATH, 'storage', 'sftp')
|
173
|
-
autoload :SCP, File.join(CONFIGURATION_PATH, 'storage', 'scp')
|
174
|
-
autoload :RSync, File.join(CONFIGURATION_PATH, 'storage', 'rsync')
|
175
|
-
autoload :Local, File.join(CONFIGURATION_PATH, 'storage', 'local')
|
176
|
-
end
|
177
|
-
|
178
|
-
module Syncer
|
179
|
-
autoload :Base, File.join(CONFIGURATION_PATH, 'syncer', 'base')
|
180
|
-
autoload :Cloud, File.join(CONFIGURATION_PATH, 'syncer', 'cloud')
|
181
|
-
autoload :CloudFiles, File.join(CONFIGURATION_PATH, 'syncer', 'cloud_files')
|
182
|
-
autoload :S3, File.join(CONFIGURATION_PATH, 'syncer', 's3')
|
183
|
-
module RSync
|
184
|
-
autoload :Base, File.join(CONFIGURATION_PATH, 'syncer', 'rsync', 'base')
|
185
|
-
autoload :Local, File.join(CONFIGURATION_PATH, 'syncer', 'rsync', 'local')
|
186
|
-
autoload :Push, File.join(CONFIGURATION_PATH, 'syncer', 'rsync', 'push')
|
187
|
-
autoload :Pull, File.join(CONFIGURATION_PATH, 'syncer', 'rsync', 'pull')
|
188
|
-
end
|
189
|
-
end
|
190
|
-
|
191
|
-
module Database
|
192
|
-
autoload :Base, File.join(CONFIGURATION_PATH, 'database', 'base')
|
193
|
-
autoload :MySQL, File.join(CONFIGURATION_PATH, 'database', 'mysql')
|
194
|
-
autoload :PostgreSQL, File.join(CONFIGURATION_PATH, 'database', 'postgresql')
|
195
|
-
autoload :MongoDB, File.join(CONFIGURATION_PATH, 'database', 'mongodb')
|
196
|
-
autoload :Redis, File.join(CONFIGURATION_PATH, 'database', 'redis')
|
197
|
-
autoload :Riak, File.join(CONFIGURATION_PATH, 'database', 'riak')
|
198
|
-
end
|
199
|
-
end
|
112
|
+
# Require Backup base files
|
113
|
+
%w{
|
114
|
+
archive
|
115
|
+
binder
|
116
|
+
cleaner
|
117
|
+
config
|
118
|
+
configuration
|
119
|
+
dependency
|
120
|
+
errors
|
121
|
+
logger
|
122
|
+
model
|
123
|
+
package
|
124
|
+
packager
|
125
|
+
pipeline
|
126
|
+
splitter
|
127
|
+
template
|
128
|
+
version
|
129
|
+
}.each {|lib| require File.join(LIBRARY_PATH, lib) }
|
200
130
|
|
201
131
|
end
|
data/lib/backup/archive.rb
CHANGED
@@ -67,26 +67,34 @@ module Backup
|
|
67
67
|
# will be piped through the Compressor command and the file extension
|
68
68
|
# will be adjusted to indicate the type of compression used.
|
69
69
|
def perform!
|
70
|
-
Logger.message "#{ self.class } started
|
70
|
+
Logger.message "#{ self.class } has started archiving:\n" +
|
71
71
|
paths.map {|path| " #{path}" }.join("\n")
|
72
72
|
|
73
73
|
archive_path = File.join(Config.tmp_path, @model.trigger, 'archives')
|
74
74
|
FileUtils.mkdir_p(archive_path)
|
75
75
|
|
76
76
|
archive_ext = 'tar'
|
77
|
-
|
77
|
+
pipeline = Pipeline.new
|
78
|
+
|
79
|
+
pipeline << "#{ utility(:tar) } #{ tar_args } -cPf - " +
|
78
80
|
"#{ paths_to_exclude } #{ paths_to_package }"
|
79
81
|
|
80
82
|
if @model.compressor
|
81
83
|
@model.compressor.compress_with do |command, ext|
|
82
|
-
|
84
|
+
pipeline << command
|
83
85
|
archive_ext << ext
|
84
86
|
end
|
85
87
|
end
|
86
88
|
|
87
|
-
|
88
|
-
|
89
|
-
|
89
|
+
pipeline << "cat > '#{ File.join(archive_path, "#{name}.#{archive_ext}") }'"
|
90
|
+
pipeline.run
|
91
|
+
if pipeline.success?
|
92
|
+
Logger.message "#{ self.class } Complete!"
|
93
|
+
else
|
94
|
+
raise Errors::Archive::PipelineError,
|
95
|
+
"Failed to Create Backup Archive\n" +
|
96
|
+
pipeline.error_messages
|
97
|
+
end
|
90
98
|
end
|
91
99
|
|
92
100
|
private
|
data/lib/backup/cli/helpers.rb
CHANGED
@@ -6,45 +6,73 @@ module Backup
|
|
6
6
|
UTILITY = {}
|
7
7
|
|
8
8
|
##
|
9
|
-
# Runs a
|
10
|
-
# The STDOUT, STDERR and the returned exit code of the utility will be stored in the process_data Hash.
|
9
|
+
# Runs a system command
|
11
10
|
#
|
12
|
-
#
|
13
|
-
#
|
14
|
-
# you can pass in an array of exit codes to ignore (whitelist), for example:
|
11
|
+
# All messages generated by the command will be logged.
|
12
|
+
# Messages on STDERR will be logged as warnings.
|
15
13
|
#
|
16
|
-
#
|
14
|
+
# If the command fails to execute, or returns a non-zero exit status
|
15
|
+
# an Error will be raised.
|
17
16
|
#
|
18
|
-
#
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
17
|
+
# Returns nil
|
18
|
+
def run(command)
|
19
|
+
name = command_name(command)
|
20
|
+
Logger.message "Running system utility '#{ name }'..."
|
21
|
+
|
22
|
+
begin
|
23
|
+
out, err = '', ''
|
24
|
+
ps = Open4.popen4(command) do |pid, stdin, stdout, stderr|
|
25
|
+
stdin.close
|
26
|
+
out, err = stdout.read.strip, stderr.read.strip
|
27
|
+
end
|
28
|
+
rescue Exception => e
|
29
|
+
raise Errors::CLI::SystemCallError.wrap(e, <<-EOS)
|
30
|
+
Failed to execute system command on #{ RUBY_PLATFORM }
|
31
|
+
Command was: #{ command }
|
32
|
+
EOS
|
33
|
+
end
|
34
|
+
|
35
|
+
if ps.success?
|
36
|
+
unless out.empty?
|
37
|
+
Logger.message(
|
38
|
+
out.lines.map {|line| "#{ name }:STDOUT: #{ line }" }.join
|
39
|
+
)
|
40
|
+
end
|
24
41
|
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
process_data[:ignore_exit_codes] = ((options[:ignore_exit_codes] || Array.new) << 0).uniq
|
42
|
+
unless err.empty?
|
43
|
+
Logger.warn(
|
44
|
+
err.lines.map {|line| "#{ name }:STDERR: #{ line }" }.join
|
45
|
+
)
|
46
|
+
end
|
31
47
|
|
32
|
-
|
33
|
-
|
48
|
+
return nil
|
49
|
+
else
|
50
|
+
raise Errors::CLI::SystemCallError, <<-EOS
|
51
|
+
'#{ name }' Failed on #{ RUBY_PLATFORM }
|
52
|
+
The following information should help to determine the problem:
|
53
|
+
Command was: #{ command }
|
54
|
+
Exit Status: #{ ps.exitstatus }
|
55
|
+
STDOUT Messages: #{ out.empty? ? 'None' : "\n#{ out }" }
|
56
|
+
STDERR Messages: #{ err.empty? ? 'None' : "\n#{ err }" }
|
57
|
+
EOS
|
58
|
+
end
|
34
59
|
end
|
35
60
|
|
61
|
+
|
36
62
|
##
|
37
63
|
# Returns the full path to the specified utility.
|
38
64
|
# Raises an error if utility can not be found in the system's $PATH
|
39
65
|
def utility(name)
|
40
|
-
|
66
|
+
name = name.to_s.strip
|
67
|
+
raise Errors::CLI::UtilityNotFoundError,
|
68
|
+
'Utility Name Empty' if name.empty?
|
69
|
+
|
70
|
+
path = UTILITY[name] || %x[which #{ name } 2>/dev/null].chomp
|
41
71
|
if path.empty?
|
42
72
|
raise Errors::CLI::UtilityNotFoundError, <<-EOS
|
43
|
-
|
73
|
+
Could not locate '#{ name }'.
|
44
74
|
Make sure the specified utility is installed
|
45
75
|
and available in your system's $PATH.
|
46
|
-
If this is a database utility, you may need to specify the full path
|
47
|
-
using the Database's '<utility_name>_utility' configuration setting.
|
48
76
|
EOS
|
49
77
|
end
|
50
78
|
UTILITY[name] = path
|
@@ -58,31 +86,6 @@ module Backup
|
|
58
86
|
command.split('/')[-1]
|
59
87
|
end
|
60
88
|
|
61
|
-
##
|
62
|
-
# Inspects the exit code returned from the POpen4 child process. If the exit code isn't listed
|
63
|
-
# in the process_data[:ignore_exit_codes] array, an exception will be raised, aborting the backup process.
|
64
|
-
#
|
65
|
-
# Information regarding the error ( EXIT CODE and STDERR ) will be returned to the shell so the user can
|
66
|
-
# investigate the issue.
|
67
|
-
#
|
68
|
-
# raises Backup::Errors::CLI::SystemCallError
|
69
|
-
def raise_if_command_failed!(utility, process_data)
|
70
|
-
unless process_data[:ignore_exit_codes].include?(process_data[:status].to_i)
|
71
|
-
|
72
|
-
stderr = process_data[:stderr].empty? ?
|
73
|
-
nil : "STDERR:\n#{process_data[:stderr]}\n"
|
74
|
-
stdout = process_data[:stdout].empty? ?
|
75
|
-
nil : "STDOUT:\n#{process_data[:stdout]}\n"
|
76
|
-
|
77
|
-
raise Errors::CLI::SystemCallError, <<-EOS
|
78
|
-
Failed to run #{utility} on #{RUBY_PLATFORM}
|
79
|
-
The following information should help to determine the problem:
|
80
|
-
Exit Code: #{process_data[:status]}
|
81
|
-
#{stderr}#{stdout}
|
82
|
-
EOS
|
83
|
-
end
|
84
|
-
end
|
85
|
-
|
86
89
|
end
|
87
90
|
end
|
88
91
|
end
|