backup 3.0.23 → 3.0.24

Sign up to get free protection for your applications and to get access to all the features.
Files changed (197) hide show
  1. data/Gemfile.lock +42 -45
  2. data/Guardfile +7 -4
  3. data/README.md +10 -7
  4. data/backup.gemspec +2 -2
  5. data/lib/backup.rb +27 -97
  6. data/lib/backup/archive.rb +14 -6
  7. data/lib/backup/cli/helpers.rb +52 -49
  8. data/lib/backup/cli/utility.rb +9 -1
  9. data/lib/backup/compressor/base.rb +10 -4
  10. data/lib/backup/compressor/bzip2.rb +22 -26
  11. data/lib/backup/compressor/custom.rb +53 -0
  12. data/lib/backup/compressor/gzip.rb +22 -23
  13. data/lib/backup/compressor/lzma.rb +15 -13
  14. data/lib/backup/compressor/pbzip2.rb +20 -17
  15. data/lib/backup/config.rb +6 -3
  16. data/lib/backup/configuration.rb +33 -0
  17. data/lib/backup/configuration/helpers.rb +114 -28
  18. data/lib/backup/configuration/store.rb +24 -0
  19. data/lib/backup/database/base.rb +0 -6
  20. data/lib/backup/database/mongodb.rb +27 -11
  21. data/lib/backup/database/mysql.rb +19 -14
  22. data/lib/backup/database/postgresql.rb +16 -11
  23. data/lib/backup/database/redis.rb +7 -11
  24. data/lib/backup/database/riak.rb +3 -6
  25. data/lib/backup/dependency.rb +5 -11
  26. data/lib/backup/model.rb +14 -5
  27. data/lib/backup/notifier/campfire.rb +3 -16
  28. data/lib/backup/notifier/hipchat.rb +1 -7
  29. data/lib/backup/notifier/mail.rb +1 -1
  30. data/lib/backup/packager.rb +29 -19
  31. data/lib/backup/pipeline.rb +110 -0
  32. data/lib/backup/storage/dropbox.rb +4 -7
  33. data/lib/backup/syncer/base.rb +8 -4
  34. data/lib/backup/syncer/cloud/base.rb +247 -0
  35. data/lib/backup/syncer/cloud/cloud_files.rb +78 -0
  36. data/lib/backup/syncer/cloud/s3.rb +68 -0
  37. data/lib/backup/syncer/rsync/base.rb +1 -4
  38. data/lib/backup/syncer/rsync/local.rb +9 -5
  39. data/lib/backup/syncer/rsync/pull.rb +1 -1
  40. data/lib/backup/syncer/rsync/push.rb +10 -5
  41. data/lib/backup/version.rb +1 -1
  42. data/spec-live/.gitignore +6 -0
  43. data/spec-live/README +7 -0
  44. data/spec-live/backups/config.rb +153 -0
  45. data/spec-live/backups/config.yml.template +43 -0
  46. data/spec-live/compressor/custom_spec.rb +30 -0
  47. data/spec-live/compressor/gzip_spec.rb +30 -0
  48. data/spec-live/notifier/mail_spec.rb +85 -0
  49. data/spec-live/spec_helper.rb +85 -0
  50. data/spec-live/storage/dropbox_spec.rb +151 -0
  51. data/spec-live/storage/local_spec.rb +83 -0
  52. data/spec-live/storage/scp_spec.rb +193 -0
  53. data/spec-live/syncer/cloud/s3_spec.rb +124 -0
  54. data/spec/archive_spec.rb +86 -31
  55. data/spec/cleaner_spec.rb +8 -0
  56. data/spec/cli/helpers_spec.rb +200 -75
  57. data/spec/cli/utility_spec.rb +11 -3
  58. data/spec/compressor/base_spec.rb +31 -10
  59. data/spec/compressor/bzip2_spec.rb +212 -57
  60. data/spec/compressor/custom_spec.rb +106 -0
  61. data/spec/compressor/gzip_spec.rb +212 -57
  62. data/spec/compressor/lzma_spec.rb +75 -35
  63. data/spec/compressor/pbzip2_spec.rb +93 -52
  64. data/spec/configuration/helpers_spec.rb +406 -0
  65. data/spec/configuration/store_spec.rb +39 -0
  66. data/spec/configuration_spec.rb +62 -0
  67. data/spec/database/base_spec.rb +19 -10
  68. data/spec/database/mongodb_spec.rb +195 -70
  69. data/spec/database/mysql_spec.rb +183 -64
  70. data/spec/database/postgresql_spec.rb +167 -53
  71. data/spec/database/redis_spec.rb +121 -46
  72. data/spec/database/riak_spec.rb +96 -27
  73. data/spec/dependency_spec.rb +2 -0
  74. data/spec/encryptor/base_spec.rb +10 -0
  75. data/spec/encryptor/gpg_spec.rb +29 -13
  76. data/spec/encryptor/open_ssl_spec.rb +40 -21
  77. data/spec/logger_spec.rb +4 -0
  78. data/spec/model_spec.rb +19 -2
  79. data/spec/notifier/base_spec.rb +32 -17
  80. data/spec/notifier/campfire_spec.rb +63 -45
  81. data/spec/notifier/hipchat_spec.rb +79 -56
  82. data/spec/notifier/mail_spec.rb +82 -46
  83. data/spec/notifier/prowl_spec.rb +53 -32
  84. data/spec/notifier/twitter_spec.rb +62 -41
  85. data/spec/packager_spec.rb +95 -36
  86. data/spec/pipeline_spec.rb +259 -0
  87. data/spec/spec_helper.rb +6 -5
  88. data/spec/storage/base_spec.rb +61 -41
  89. data/spec/storage/cloudfiles_spec.rb +69 -45
  90. data/spec/storage/dropbox_spec.rb +158 -36
  91. data/spec/storage/ftp_spec.rb +69 -45
  92. data/spec/storage/local_spec.rb +47 -23
  93. data/spec/storage/ninefold_spec.rb +55 -31
  94. data/spec/storage/rsync_spec.rb +67 -50
  95. data/spec/storage/s3_spec.rb +65 -41
  96. data/spec/storage/scp_spec.rb +65 -41
  97. data/spec/storage/sftp_spec.rb +65 -41
  98. data/spec/syncer/base_spec.rb +91 -4
  99. data/spec/syncer/cloud/base_spec.rb +511 -0
  100. data/spec/syncer/cloud/cloud_files_spec.rb +181 -0
  101. data/spec/syncer/cloud/s3_spec.rb +174 -0
  102. data/spec/syncer/rsync/base_spec.rb +46 -66
  103. data/spec/syncer/rsync/local_spec.rb +55 -26
  104. data/spec/syncer/rsync/pull_spec.rb +15 -4
  105. data/spec/syncer/rsync/push_spec.rb +59 -52
  106. data/templates/cli/utility/compressor/bzip2 +1 -4
  107. data/templates/cli/utility/compressor/custom +11 -0
  108. data/templates/cli/utility/compressor/gzip +1 -4
  109. data/templates/cli/utility/compressor/lzma +3 -0
  110. data/templates/cli/utility/compressor/pbzip2 +3 -0
  111. data/templates/cli/utility/database/mysql +4 -1
  112. data/templates/cli/utility/syncer/cloud_files +17 -19
  113. data/templates/cli/utility/syncer/s3 +18 -20
  114. metadata +38 -92
  115. data/lib/backup/configuration/base.rb +0 -15
  116. data/lib/backup/configuration/compressor/base.rb +0 -9
  117. data/lib/backup/configuration/compressor/bzip2.rb +0 -23
  118. data/lib/backup/configuration/compressor/gzip.rb +0 -23
  119. data/lib/backup/configuration/compressor/lzma.rb +0 -23
  120. data/lib/backup/configuration/compressor/pbzip2.rb +0 -28
  121. data/lib/backup/configuration/database/base.rb +0 -19
  122. data/lib/backup/configuration/database/mongodb.rb +0 -49
  123. data/lib/backup/configuration/database/mysql.rb +0 -42
  124. data/lib/backup/configuration/database/postgresql.rb +0 -41
  125. data/lib/backup/configuration/database/redis.rb +0 -39
  126. data/lib/backup/configuration/database/riak.rb +0 -29
  127. data/lib/backup/configuration/encryptor/base.rb +0 -9
  128. data/lib/backup/configuration/encryptor/gpg.rb +0 -17
  129. data/lib/backup/configuration/encryptor/open_ssl.rb +0 -32
  130. data/lib/backup/configuration/notifier/base.rb +0 -28
  131. data/lib/backup/configuration/notifier/campfire.rb +0 -25
  132. data/lib/backup/configuration/notifier/hipchat.rb +0 -41
  133. data/lib/backup/configuration/notifier/mail.rb +0 -112
  134. data/lib/backup/configuration/notifier/presently.rb +0 -25
  135. data/lib/backup/configuration/notifier/prowl.rb +0 -23
  136. data/lib/backup/configuration/notifier/twitter.rb +0 -21
  137. data/lib/backup/configuration/storage/base.rb +0 -18
  138. data/lib/backup/configuration/storage/cloudfiles.rb +0 -25
  139. data/lib/backup/configuration/storage/dropbox.rb +0 -58
  140. data/lib/backup/configuration/storage/ftp.rb +0 -29
  141. data/lib/backup/configuration/storage/local.rb +0 -17
  142. data/lib/backup/configuration/storage/ninefold.rb +0 -20
  143. data/lib/backup/configuration/storage/rsync.rb +0 -29
  144. data/lib/backup/configuration/storage/s3.rb +0 -25
  145. data/lib/backup/configuration/storage/scp.rb +0 -25
  146. data/lib/backup/configuration/storage/sftp.rb +0 -25
  147. data/lib/backup/configuration/syncer/base.rb +0 -10
  148. data/lib/backup/configuration/syncer/cloud.rb +0 -23
  149. data/lib/backup/configuration/syncer/cloud_files.rb +0 -30
  150. data/lib/backup/configuration/syncer/rsync/base.rb +0 -28
  151. data/lib/backup/configuration/syncer/rsync/local.rb +0 -11
  152. data/lib/backup/configuration/syncer/rsync/pull.rb +0 -11
  153. data/lib/backup/configuration/syncer/rsync/push.rb +0 -31
  154. data/lib/backup/configuration/syncer/s3.rb +0 -23
  155. data/lib/backup/notifier/presently.rb +0 -88
  156. data/lib/backup/syncer/cloud.rb +0 -187
  157. data/lib/backup/syncer/cloud_files.rb +0 -56
  158. data/lib/backup/syncer/s3.rb +0 -47
  159. data/spec/configuration/base_spec.rb +0 -35
  160. data/spec/configuration/compressor/bzip2_spec.rb +0 -29
  161. data/spec/configuration/compressor/gzip_spec.rb +0 -29
  162. data/spec/configuration/compressor/lzma_spec.rb +0 -29
  163. data/spec/configuration/compressor/pbzip2_spec.rb +0 -32
  164. data/spec/configuration/database/base_spec.rb +0 -17
  165. data/spec/configuration/database/mongodb_spec.rb +0 -56
  166. data/spec/configuration/database/mysql_spec.rb +0 -53
  167. data/spec/configuration/database/postgresql_spec.rb +0 -53
  168. data/spec/configuration/database/redis_spec.rb +0 -50
  169. data/spec/configuration/database/riak_spec.rb +0 -35
  170. data/spec/configuration/encryptor/gpg_spec.rb +0 -26
  171. data/spec/configuration/encryptor/open_ssl_spec.rb +0 -35
  172. data/spec/configuration/notifier/base_spec.rb +0 -32
  173. data/spec/configuration/notifier/campfire_spec.rb +0 -32
  174. data/spec/configuration/notifier/hipchat_spec.rb +0 -44
  175. data/spec/configuration/notifier/mail_spec.rb +0 -71
  176. data/spec/configuration/notifier/presently_spec.rb +0 -35
  177. data/spec/configuration/notifier/prowl_spec.rb +0 -29
  178. data/spec/configuration/notifier/twitter_spec.rb +0 -35
  179. data/spec/configuration/storage/cloudfiles_spec.rb +0 -41
  180. data/spec/configuration/storage/dropbox_spec.rb +0 -38
  181. data/spec/configuration/storage/ftp_spec.rb +0 -44
  182. data/spec/configuration/storage/local_spec.rb +0 -29
  183. data/spec/configuration/storage/ninefold_spec.rb +0 -32
  184. data/spec/configuration/storage/rsync_spec.rb +0 -41
  185. data/spec/configuration/storage/s3_spec.rb +0 -38
  186. data/spec/configuration/storage/scp_spec.rb +0 -41
  187. data/spec/configuration/storage/sftp_spec.rb +0 -41
  188. data/spec/configuration/syncer/cloud_files_spec.rb +0 -44
  189. data/spec/configuration/syncer/rsync/base_spec.rb +0 -33
  190. data/spec/configuration/syncer/rsync/local_spec.rb +0 -10
  191. data/spec/configuration/syncer/rsync/pull_spec.rb +0 -10
  192. data/spec/configuration/syncer/rsync/push_spec.rb +0 -43
  193. data/spec/configuration/syncer/s3_spec.rb +0 -38
  194. data/spec/notifier/presently_spec.rb +0 -181
  195. data/spec/syncer/cloud_files_spec.rb +0 -192
  196. data/spec/syncer/s3_spec.rb +0 -192
  197. data/templates/cli/utility/notifier/presently +0 -13
data/Gemfile.lock CHANGED
@@ -2,29 +2,25 @@ PATH
2
2
  remote: .
3
3
  specs:
4
4
  backup (3.0.23)
5
- POpen4 (~> 0.1.4)
5
+ open4 (~> 1.3.0)
6
6
  thor (~> 0.14.6)
7
7
 
8
8
  GEM
9
9
  remote: http://rubygems.org/
10
10
  specs:
11
- POpen4 (0.1.4)
12
- Platform (>= 0.4.0)
13
- open4
14
- Platform (0.4.0)
15
- activesupport (3.1.3)
11
+ activesupport (3.2.2)
12
+ i18n (~> 0.6)
16
13
  multi_json (~> 1.0)
17
- addressable (2.2.6)
14
+ addressable (2.2.7)
18
15
  builder (3.0.0)
19
- crack (0.1.8)
20
16
  diff-lcs (1.1.3)
21
- dropbox-sdk (1.1)
17
+ dropbox-sdk (1.2)
22
18
  json
23
- excon (0.9.4)
24
- faraday (0.7.5)
25
- addressable (~> 2.2.6)
26
- multipart-post (~> 1.1.3)
27
- rack (>= 1.1.0, < 2)
19
+ excon (0.9.6)
20
+ faraday (0.7.6)
21
+ addressable (~> 2.2)
22
+ multipart-post (~> 1.1)
23
+ rack (~> 1.1)
28
24
  ffi (1.0.11)
29
25
  fog (1.1.2)
30
26
  builder
@@ -37,56 +33,58 @@ GEM
37
33
  nokogiri (~> 1.5.0)
38
34
  ruby-hmac
39
35
  formatador (0.2.1)
40
- fuubar (0.0.6)
36
+ fuubar (1.0.0)
41
37
  rspec (~> 2.0)
42
- rspec-instafail (~> 0.1.8)
38
+ rspec-instafail (~> 0.2.0)
43
39
  ruby-progressbar (~> 0.0.10)
44
40
  growl (1.0.3)
45
- guard (0.10.0)
41
+ guard (1.0.1)
46
42
  ffi (>= 0.5.0)
47
43
  thor (~> 0.14.6)
48
44
  guard-rspec (0.6.0)
49
45
  guard (>= 0.10.0)
50
46
  hipchat (0.4.1)
51
47
  httparty
52
- httparty (0.7.8)
53
- crack (= 0.1.8)
48
+ httparty (0.8.1)
49
+ multi_json
50
+ multi_xml
54
51
  i18n (0.6.0)
55
- json (1.5.4)
56
- libnotify (0.7.1)
57
- mail (2.4.1)
52
+ json (1.6.5)
53
+ libnotify (0.7.2)
54
+ mail (2.4.4)
58
55
  i18n (>= 0.4.0)
59
56
  mime-types (~> 1.16)
60
57
  treetop (~> 1.4.8)
61
58
  metaclass (0.0.1)
62
59
  mime-types (1.17.2)
63
- mocha (0.10.0)
60
+ mocha (0.10.5)
64
61
  metaclass (~> 0.0.1)
65
62
  multi_json (1.0.4)
66
- multipart-post (1.1.4)
63
+ multi_xml (0.4.2)
64
+ multipart-post (1.1.5)
67
65
  net-scp (1.0.4)
68
66
  net-ssh (>= 1.99.1)
69
67
  net-sftp (2.0.5)
70
68
  net-ssh (>= 2.0.9)
71
- net-ssh (2.1.4)
72
- nokogiri (1.5.0)
69
+ net-ssh (2.3.0)
70
+ nokogiri (1.5.2)
73
71
  open4 (1.3.0)
74
- parallel (0.5.12)
72
+ parallel (0.5.16)
75
73
  polyglot (0.3.3)
76
74
  prowler (1.3.1)
77
- rack (1.4.0)
78
- rb-fsevent (0.4.3.1)
75
+ rack (1.4.1)
76
+ rb-fsevent (0.9.0)
79
77
  rb-inotify (0.8.8)
80
78
  ffi (>= 0.5.0)
81
- rspec (2.8.0)
82
- rspec-core (~> 2.8.0)
83
- rspec-expectations (~> 2.8.0)
84
- rspec-mocks (~> 2.8.0)
85
- rspec-core (2.8.0)
86
- rspec-expectations (2.8.0)
87
- diff-lcs (~> 1.1.2)
88
- rspec-instafail (0.1.9)
89
- rspec-mocks (2.8.0)
79
+ rspec (2.9.0)
80
+ rspec-core (~> 2.9.0)
81
+ rspec-expectations (~> 2.9.0)
82
+ rspec-mocks (~> 2.9.0)
83
+ rspec-core (2.9.0)
84
+ rspec-expectations (2.9.0)
85
+ diff-lcs (~> 1.1.3)
86
+ rspec-instafail (0.2.2)
87
+ rspec-mocks (2.9.0)
90
88
  ruby-hmac (0.4.0)
91
89
  ruby-progressbar (0.0.10)
92
90
  simple_oauth (0.1.5)
@@ -95,7 +93,7 @@ GEM
95
93
  treetop (1.4.10)
96
94
  polyglot
97
95
  polyglot (>= 0.3.1)
98
- twitter (2.0.2)
96
+ twitter (2.1.1)
99
97
  activesupport (>= 2.3.9, < 4)
100
98
  faraday (~> 0.7)
101
99
  multi_json (~> 1.0)
@@ -106,21 +104,20 @@ PLATFORMS
106
104
 
107
105
  DEPENDENCIES
108
106
  backup!
109
- dropbox-sdk (~> 1.1.0)
110
- fog (>= 0.11.0)
107
+ dropbox-sdk (~> 1.2.0)
108
+ fog (~> 1.1.0)
111
109
  fuubar
112
110
  growl
113
111
  guard
114
112
  guard-rspec
115
113
  hipchat (~> 0.4.1)
116
- httparty (~> 0.7.4)
117
- json (~> 1.5.1)
114
+ httparty (~> 0.8.1)
118
115
  libnotify
119
- mail (>= 2.4.0)
116
+ mail (~> 2.4.0)
120
117
  mocha
121
118
  net-scp (~> 1.0.4)
122
119
  net-sftp (~> 2.0.5)
123
- net-ssh (~> 2.1.4)
120
+ net-ssh (~> 2.3.0)
124
121
  parallel (~> 0.5.12)
125
122
  prowler (>= 1.3.1)
126
123
  rb-fsevent
data/Guardfile CHANGED
@@ -12,10 +12,13 @@
12
12
  guard "rspec",
13
13
  :version => 2,
14
14
  :rvm => ["1.9.3", "1.9.2", "1.8.7"],
15
- :bundler => true,
16
- :cli => "--color --format Fuubar" do
15
+ :cli => "--color --format Fuubar",
16
+ :notification => false,
17
+ :all_after_pass => false,
18
+ :all_on_start => false do
17
19
 
20
+ watch("lib/backup.rb") { "spec" }
21
+ watch("spec/spec_helper.rb") { "spec" }
22
+ watch(%r{^lib/backup/(.+)\.rb}) {|m| "spec/#{ m[1] }_spec.rb" }
18
23
  watch(%r{^spec/.+_spec\.rb})
19
- watch(%r{^lib/(.+)\.rb}) { "spec" }
20
- watch("spec/spec_helper.rb") { "spec" }
21
24
  end
data/README.md CHANGED
@@ -69,6 +69,9 @@ Below you find a list of components that Backup currently supports. If you'd lik
69
69
  - Dropbox Web Service
70
70
  - Remote Servers *(Only Protocols: FTP, SFTP, SCP)*
71
71
  - Local Storage
72
+
73
+ [Cycling Wiki Page](https://github.com/meskyanichi/backup/wiki/Cycling)
74
+
72
75
  - **Backup Splitting, applies to:**
73
76
  - Amazon Simple Storage Service (S3)
74
77
  - Rackspace Cloud Files (Mosso)
@@ -76,13 +79,12 @@ Below you find a list of components that Backup currently supports. If you'd lik
76
79
  - Dropbox Web Service
77
80
  - Remote Servers *(Only Protocols: FTP, SFTP, SCP)*
78
81
  - Local Storage
79
- - **Incremental Backups, applies to:**
80
- - Remote Servers *(Only Protocols: RSync)*
81
-
82
- [Cycling Wiki Page](https://github.com/meskyanichi/backup/wiki/Cycling)
83
82
 
84
83
  [Splitter Wiki Page](https://github.com/meskyanichi/backup/wiki/Splitter)
85
84
 
85
+ - **Incremental Backups, applies to:**
86
+ - Remote Servers *(Only Protocols: RSync)*
87
+
86
88
  ### Syncers
87
89
 
88
90
  - RSync (Push, Pull and Local)
@@ -128,7 +130,8 @@ Below you find a list of components that Backup currently supports. If you'd lik
128
130
  A sample Backup configuration file
129
131
  ----------------------------------
130
132
 
131
- This is a Backup configuration file. Check it out and read the explanation below. Backup has a [great wiki](https://github.com/meskyanichi/backup/wiki) which explains each component of Backup in detail.
133
+ This is a Backup configuration file. Check it out and read the explanation below.
134
+ Backup has a [great wiki](https://github.com/meskyanichi/backup/wiki) which explains each component of Backup in detail.
132
135
 
133
136
  ``` rb
134
137
  Backup::Model.new(:sample_backup, 'A sample backup configuration') do
@@ -193,7 +196,7 @@ Backup::Model.new(:sample_backup, 'A sample backup configuration') do
193
196
  s3.keep = 20
194
197
  end
195
198
 
196
- sync_with S3 do |s3|
199
+ sync_with Cloud::S3 do |s3|
197
200
  s3.access_key_id = "my_access_key_id"
198
201
  s3.secret_access_key = "my_secret_access_key"
199
202
  s3.bucket = "my-bucket"
@@ -252,7 +255,7 @@ of `-aa`, `-ab` and `-ac`. These files will then be individually transfered. Thi
252
255
  Amazon S3, Rackspace Cloud Files, or other 3rd party storage services which limit you to "5GB per file" uploads. So with
253
256
  this, the backup file size is no longer a constraint.
254
257
 
255
- Additionally we have also defined a **S3 Syncer** ( `sync_with S3` ), which does not follow the above process of
258
+ Additionally we have also defined a **S3 Syncer** ( `sync_with Cloud::S3` ), which does not follow the above process of
256
259
  archiving/compression/encryption, but instead will directly sync the whole `videos` and `music` folder structures from
257
260
  your machine to your Amazon S3 account. (very efficient and cost-effective since it will only transfer files that were
258
261
  added/changed. Additionally, since we flagged it to 'mirror', it'll also remove files from S3 that no longer exist). If
data/backup.gemspec CHANGED
@@ -26,7 +26,7 @@ Gem::Specification.new do |gem|
26
26
 
27
27
  ##
28
28
  # Gem dependencies
29
- gem.add_dependency 'thor', ['~> 0.14.6']
30
- gem.add_dependency 'POpen4', ['~> 0.1.4']
29
+ gem.add_dependency 'thor', ['~> 0.14.6']
30
+ gem.add_dependency 'open4', ['~> 1.3.0']
31
31
 
32
32
  end
data/lib/backup.rb CHANGED
@@ -7,17 +7,8 @@ require 'tempfile'
7
7
  require 'yaml'
8
8
  require 'etc'
9
9
 
10
- # Attempt load to POpen4 and Thor Libraries
11
- begin
12
- gem 'POpen4', '~> 0.1.4'
13
- gem 'thor', '~> 0.14.6'
14
- require 'popen4'
15
- require 'thor'
16
- rescue LoadError
17
- puts "\nBackup requires Thor to load the CLI Utility (Command Line Interface Utility) and POpen4 to determine the status of the unix processes."
18
- puts "Please install both the Thor and POpen4 libraries first:\n\ngem install thor -v '~> 0.14.6'\ngem install POpen4 -v '~> 0.1.4'"
19
- exit 1
20
- end
10
+ require 'open4'
11
+ require 'thor'
21
12
 
22
13
  ##
23
14
  # The Backup Ruby Gem
@@ -33,25 +24,8 @@ module Backup
33
24
  ENCRYPTOR_PATH = File.join(LIBRARY_PATH, 'encryptor')
34
25
  NOTIFIER_PATH = File.join(LIBRARY_PATH, 'notifier')
35
26
  SYNCER_PATH = File.join(LIBRARY_PATH, 'syncer')
36
- CONFIGURATION_PATH = File.join(LIBRARY_PATH, 'configuration')
37
27
  TEMPLATE_PATH = File.expand_path('../../templates', __FILE__)
38
28
 
39
- ##
40
- # Autoload Backup base files
41
- autoload :Model, File.join(LIBRARY_PATH, 'model')
42
- autoload :Archive, File.join(LIBRARY_PATH, 'archive')
43
- autoload :Packager, File.join(LIBRARY_PATH, 'packager')
44
- autoload :Package, File.join(LIBRARY_PATH, 'package')
45
- autoload :Cleaner, File.join(LIBRARY_PATH, 'cleaner')
46
- autoload :Splitter, File.join(LIBRARY_PATH, 'splitter')
47
- autoload :Config, File.join(LIBRARY_PATH, 'config')
48
- autoload :Binder, File.join(LIBRARY_PATH, 'binder')
49
- autoload :Template, File.join(LIBRARY_PATH, 'template')
50
- autoload :Dependency, File.join(LIBRARY_PATH, 'dependency')
51
- autoload :Logger, File.join(LIBRARY_PATH, 'logger')
52
- autoload :Version, File.join(LIBRARY_PATH, 'version')
53
- autoload :Errors, File.join(LIBRARY_PATH, 'errors')
54
-
55
29
  ##
56
30
  # Autoload Backup CLI files
57
31
  module CLI
@@ -78,10 +52,12 @@ module Backup
78
52
  ##
79
53
  # Autoload Backup syncer files
80
54
  module Syncer
81
- autoload :Base, File.join(SYNCER_PATH, 'base')
82
- autoload :Cloud, File.join(SYNCER_PATH, 'cloud')
83
- autoload :CloudFiles, File.join(SYNCER_PATH, 'cloud_files')
84
- autoload :S3, File.join(SYNCER_PATH, 's3')
55
+ autoload :Base, File.join(SYNCER_PATH, 'base')
56
+ module Cloud
57
+ autoload :Base, File.join(SYNCER_PATH, 'cloud', 'base')
58
+ autoload :CloudFiles, File.join(SYNCER_PATH, 'cloud', 'cloud_files')
59
+ autoload :S3, File.join(SYNCER_PATH, 'cloud', 's3')
60
+ end
85
61
  module RSync
86
62
  autoload :Base, File.join(SYNCER_PATH, 'rsync', 'base')
87
63
  autoload :Local, File.join(SYNCER_PATH, 'rsync', 'local')
@@ -107,6 +83,7 @@ module Backup
107
83
  autoload :Base, File.join(COMPRESSOR_PATH, 'base')
108
84
  autoload :Gzip, File.join(COMPRESSOR_PATH, 'gzip')
109
85
  autoload :Bzip2, File.join(COMPRESSOR_PATH, 'bzip2')
86
+ autoload :Custom, File.join(COMPRESSOR_PATH, 'custom')
110
87
  autoload :Pbzip2, File.join(COMPRESSOR_PATH, 'pbzip2')
111
88
  autoload :Lzma, File.join(COMPRESSOR_PATH, 'lzma')
112
89
  end
@@ -127,75 +104,28 @@ module Backup
127
104
  autoload :Mail, File.join(NOTIFIER_PATH, 'mail')
128
105
  autoload :Twitter, File.join(NOTIFIER_PATH, 'twitter')
129
106
  autoload :Campfire, File.join(NOTIFIER_PATH, 'campfire')
130
- autoload :Presently, File.join(NOTIFIER_PATH, 'presently')
131
107
  autoload :Prowl, File.join(NOTIFIER_PATH, 'prowl')
132
108
  autoload :Hipchat, File.join(NOTIFIER_PATH, 'hipchat')
133
109
  end
134
110
 
135
111
  ##
136
- # Autoload Backup configuration files
137
- module Configuration
138
- autoload :Base, File.join(CONFIGURATION_PATH, 'base')
139
- autoload :Helpers, File.join(CONFIGURATION_PATH, 'helpers')
140
-
141
- module Notifier
142
- autoload :Base, File.join(CONFIGURATION_PATH, 'notifier', 'base')
143
- autoload :Mail, File.join(CONFIGURATION_PATH, 'notifier', 'mail')
144
- autoload :Twitter, File.join(CONFIGURATION_PATH, 'notifier', 'twitter')
145
- autoload :Campfire, File.join(CONFIGURATION_PATH, 'notifier', 'campfire')
146
- autoload :Presently, File.join(CONFIGURATION_PATH, 'notifier', 'presently')
147
- autoload :Prowl, File.join(CONFIGURATION_PATH, 'notifier', 'prowl')
148
- autoload :Hipchat, File.join(CONFIGURATION_PATH, 'notifier', 'hipchat')
149
- end
150
-
151
- module Encryptor
152
- autoload :Base, File.join(CONFIGURATION_PATH, 'encryptor', 'base')
153
- autoload :OpenSSL, File.join(CONFIGURATION_PATH, 'encryptor', 'open_ssl')
154
- autoload :GPG, File.join(CONFIGURATION_PATH, 'encryptor', 'gpg')
155
- end
156
-
157
- module Compressor
158
- autoload :Base, File.join(CONFIGURATION_PATH, 'compressor', 'base')
159
- autoload :Gzip, File.join(CONFIGURATION_PATH, 'compressor', 'gzip')
160
- autoload :Bzip2, File.join(CONFIGURATION_PATH, 'compressor', 'bzip2')
161
- autoload :Pbzip2, File.join(CONFIGURATION_PATH, 'compressor', 'pbzip2')
162
- autoload :Lzma, File.join(CONFIGURATION_PATH, 'compressor', 'lzma')
163
- end
164
-
165
- module Storage
166
- autoload :Base, File.join(CONFIGURATION_PATH, 'storage', 'base')
167
- autoload :S3, File.join(CONFIGURATION_PATH, 'storage', 's3')
168
- autoload :CloudFiles, File.join(CONFIGURATION_PATH, 'storage', 'cloudfiles')
169
- autoload :Ninefold, File.join(CONFIGURATION_PATH, 'storage', 'ninefold')
170
- autoload :Dropbox, File.join(CONFIGURATION_PATH, 'storage', 'dropbox')
171
- autoload :FTP, File.join(CONFIGURATION_PATH, 'storage', 'ftp')
172
- autoload :SFTP, File.join(CONFIGURATION_PATH, 'storage', 'sftp')
173
- autoload :SCP, File.join(CONFIGURATION_PATH, 'storage', 'scp')
174
- autoload :RSync, File.join(CONFIGURATION_PATH, 'storage', 'rsync')
175
- autoload :Local, File.join(CONFIGURATION_PATH, 'storage', 'local')
176
- end
177
-
178
- module Syncer
179
- autoload :Base, File.join(CONFIGURATION_PATH, 'syncer', 'base')
180
- autoload :Cloud, File.join(CONFIGURATION_PATH, 'syncer', 'cloud')
181
- autoload :CloudFiles, File.join(CONFIGURATION_PATH, 'syncer', 'cloud_files')
182
- autoload :S3, File.join(CONFIGURATION_PATH, 'syncer', 's3')
183
- module RSync
184
- autoload :Base, File.join(CONFIGURATION_PATH, 'syncer', 'rsync', 'base')
185
- autoload :Local, File.join(CONFIGURATION_PATH, 'syncer', 'rsync', 'local')
186
- autoload :Push, File.join(CONFIGURATION_PATH, 'syncer', 'rsync', 'push')
187
- autoload :Pull, File.join(CONFIGURATION_PATH, 'syncer', 'rsync', 'pull')
188
- end
189
- end
190
-
191
- module Database
192
- autoload :Base, File.join(CONFIGURATION_PATH, 'database', 'base')
193
- autoload :MySQL, File.join(CONFIGURATION_PATH, 'database', 'mysql')
194
- autoload :PostgreSQL, File.join(CONFIGURATION_PATH, 'database', 'postgresql')
195
- autoload :MongoDB, File.join(CONFIGURATION_PATH, 'database', 'mongodb')
196
- autoload :Redis, File.join(CONFIGURATION_PATH, 'database', 'redis')
197
- autoload :Riak, File.join(CONFIGURATION_PATH, 'database', 'riak')
198
- end
199
- end
112
+ # Require Backup base files
113
+ %w{
114
+ archive
115
+ binder
116
+ cleaner
117
+ config
118
+ configuration
119
+ dependency
120
+ errors
121
+ logger
122
+ model
123
+ package
124
+ packager
125
+ pipeline
126
+ splitter
127
+ template
128
+ version
129
+ }.each {|lib| require File.join(LIBRARY_PATH, lib) }
200
130
 
201
131
  end
@@ -67,26 +67,34 @@ module Backup
67
67
  # will be piped through the Compressor command and the file extension
68
68
  # will be adjusted to indicate the type of compression used.
69
69
  def perform!
70
- Logger.message "#{ self.class } started packaging and archiving:\n" +
70
+ Logger.message "#{ self.class } has started archiving:\n" +
71
71
  paths.map {|path| " #{path}" }.join("\n")
72
72
 
73
73
  archive_path = File.join(Config.tmp_path, @model.trigger, 'archives')
74
74
  FileUtils.mkdir_p(archive_path)
75
75
 
76
76
  archive_ext = 'tar'
77
- archive_cmd = "#{ utility(:tar) } #{ tar_args } -cf - " +
77
+ pipeline = Pipeline.new
78
+
79
+ pipeline << "#{ utility(:tar) } #{ tar_args } -cPf - " +
78
80
  "#{ paths_to_exclude } #{ paths_to_package }"
79
81
 
80
82
  if @model.compressor
81
83
  @model.compressor.compress_with do |command, ext|
82
- archive_cmd << " | #{command}"
84
+ pipeline << command
83
85
  archive_ext << ext
84
86
  end
85
87
  end
86
88
 
87
- archive_cmd << " > '#{ File.join(archive_path, "#{name}.#{archive_ext}") }'"
88
-
89
- run(archive_cmd)
89
+ pipeline << "cat > '#{ File.join(archive_path, "#{name}.#{archive_ext}") }'"
90
+ pipeline.run
91
+ if pipeline.success?
92
+ Logger.message "#{ self.class } Complete!"
93
+ else
94
+ raise Errors::Archive::PipelineError,
95
+ "Failed to Create Backup Archive\n" +
96
+ pipeline.error_messages
97
+ end
90
98
  end
91
99
 
92
100
  private
@@ -6,45 +6,73 @@ module Backup
6
6
  UTILITY = {}
7
7
 
8
8
  ##
9
- # Runs a given command in an isolated (sub) process using POpen4.
10
- # The STDOUT, STDERR and the returned exit code of the utility will be stored in the process_data Hash.
9
+ # Runs a system command
11
10
  #
12
- # If a command returns an exit code other than 0, an exception will raise and the backup process will abort.
13
- # Some utilities return exit codes other than 0 which aren't an issue in Backup's context. If this is the case,
14
- # you can pass in an array of exit codes to ignore (whitelist), for example:
11
+ # All messages generated by the command will be logged.
12
+ # Messages on STDERR will be logged as warnings.
15
13
  #
16
- # run("tar -cf /output.tar /some/folder", :ignore_exit_codes => [1])
14
+ # If the command fails to execute, or returns a non-zero exit status
15
+ # an Error will be raised.
17
16
  #
18
- # So if the `tar` utility returns in this case 1, Backup will consider it an acceptable return code.
19
- #
20
- # Note: Exit code 0 is always automatically added to the :ignore_exit_codes array, regardless of whether you specify an
21
- # array to ignore or not.
22
- def run(command, options = {})
23
- command.gsub!(/^\s+/, "")
17
+ # Returns nil
18
+ def run(command)
19
+ name = command_name(command)
20
+ Logger.message "Running system utility '#{ name }'..."
21
+
22
+ begin
23
+ out, err = '', ''
24
+ ps = Open4.popen4(command) do |pid, stdin, stdout, stderr|
25
+ stdin.close
26
+ out, err = stdout.read.strip, stderr.read.strip
27
+ end
28
+ rescue Exception => e
29
+ raise Errors::CLI::SystemCallError.wrap(e, <<-EOS)
30
+ Failed to execute system command on #{ RUBY_PLATFORM }
31
+ Command was: #{ command }
32
+ EOS
33
+ end
34
+
35
+ if ps.success?
36
+ unless out.empty?
37
+ Logger.message(
38
+ out.lines.map {|line| "#{ name }:STDOUT: #{ line }" }.join
39
+ )
40
+ end
24
41
 
25
- process_data = Hash.new
26
- pid, stdin, stdout, stderr = Open4::popen4(command)
27
- ignored, process_data[:status] = Process::waitpid2(pid)
28
- process_data[:stdout] = stdout.read
29
- process_data[:stderr] = stderr.read
30
- process_data[:ignore_exit_codes] = ((options[:ignore_exit_codes] || Array.new) << 0).uniq
42
+ unless err.empty?
43
+ Logger.warn(
44
+ err.lines.map {|line| "#{ name }:STDERR: #{ line }" }.join
45
+ )
46
+ end
31
47
 
32
- raise_if_command_failed!(command_name(command), process_data)
33
- process_data[:stdout]
48
+ return nil
49
+ else
50
+ raise Errors::CLI::SystemCallError, <<-EOS
51
+ '#{ name }' Failed on #{ RUBY_PLATFORM }
52
+ The following information should help to determine the problem:
53
+ Command was: #{ command }
54
+ Exit Status: #{ ps.exitstatus }
55
+ STDOUT Messages: #{ out.empty? ? 'None' : "\n#{ out }" }
56
+ STDERR Messages: #{ err.empty? ? 'None' : "\n#{ err }" }
57
+ EOS
58
+ end
34
59
  end
35
60
 
61
+
36
62
  ##
37
63
  # Returns the full path to the specified utility.
38
64
  # Raises an error if utility can not be found in the system's $PATH
39
65
  def utility(name)
40
- path = UTILITY[name] || %x[which #{name} 2>/dev/null].chomp
66
+ name = name.to_s.strip
67
+ raise Errors::CLI::UtilityNotFoundError,
68
+ 'Utility Name Empty' if name.empty?
69
+
70
+ path = UTILITY[name] || %x[which #{ name } 2>/dev/null].chomp
41
71
  if path.empty?
42
72
  raise Errors::CLI::UtilityNotFoundError, <<-EOS
43
- Path to '#{ name }' could not be found.
73
+ Could not locate '#{ name }'.
44
74
  Make sure the specified utility is installed
45
75
  and available in your system's $PATH.
46
- If this is a database utility, you may need to specify the full path
47
- using the Database's '<utility_name>_utility' configuration setting.
48
76
  EOS
49
77
  end
50
78
  UTILITY[name] = path
@@ -58,31 +86,6 @@ module Backup
58
86
  command.split('/')[-1]
59
87
  end
60
88
 
61
- ##
62
- # Inspects the exit code returned from the POpen4 child process. If the exit code isn't listed
63
- # in the process_data[:ignore_exit_codes] array, an exception will be raised, aborting the backup process.
64
- #
65
- # Information regarding the error ( EXIT CODE and STDERR ) will be returned to the shell so the user can
66
- # investigate the issue.
67
- #
68
- # raises Backup::Errors::CLI::SystemCallError
69
- def raise_if_command_failed!(utility, process_data)
70
- unless process_data[:ignore_exit_codes].include?(process_data[:status].to_i)
71
-
72
- stderr = process_data[:stderr].empty? ?
73
- nil : "STDERR:\n#{process_data[:stderr]}\n"
74
- stdout = process_data[:stdout].empty? ?
75
- nil : "STDOUT:\n#{process_data[:stdout]}\n"
76
-
77
- raise Errors::CLI::SystemCallError, <<-EOS
78
- Failed to run #{utility} on #{RUBY_PLATFORM}
79
- The following information should help to determine the problem:
80
- Exit Code: #{process_data[:status]}
81
- #{stderr}#{stdout}
82
- EOS
83
- end
84
- end
85
-
86
89
  end
87
90
  end
88
91
  end