backup 3.0.20 → 3.0.21

Sign up to get free protection for your applications and to get access to all the features.
Files changed (178) hide show
  1. data/Gemfile +1 -5
  2. data/Gemfile.lock +46 -50
  3. data/README.md +54 -27
  4. data/lib/backup.rb +16 -39
  5. data/lib/backup/archive.rb +42 -18
  6. data/lib/backup/cleaner.rb +110 -25
  7. data/lib/backup/cli/helpers.rb +17 -32
  8. data/lib/backup/cli/utility.rb +46 -107
  9. data/lib/backup/compressor/base.rb +14 -2
  10. data/lib/backup/compressor/bzip2.rb +10 -24
  11. data/lib/backup/compressor/gzip.rb +10 -24
  12. data/lib/backup/compressor/lzma.rb +10 -23
  13. data/lib/backup/compressor/pbzip2.rb +12 -32
  14. data/lib/backup/config.rb +171 -0
  15. data/lib/backup/configuration/compressor/base.rb +1 -2
  16. data/lib/backup/configuration/compressor/pbzip2.rb +4 -4
  17. data/lib/backup/configuration/database/base.rb +2 -1
  18. data/lib/backup/configuration/database/mongodb.rb +8 -0
  19. data/lib/backup/configuration/database/mysql.rb +4 -0
  20. data/lib/backup/configuration/database/postgresql.rb +4 -0
  21. data/lib/backup/configuration/database/redis.rb +4 -0
  22. data/lib/backup/configuration/database/riak.rb +5 -1
  23. data/lib/backup/configuration/encryptor/base.rb +1 -2
  24. data/lib/backup/configuration/encryptor/open_ssl.rb +1 -1
  25. data/lib/backup/configuration/helpers.rb +7 -2
  26. data/lib/backup/configuration/notifier/base.rb +4 -28
  27. data/lib/backup/configuration/storage/base.rb +1 -1
  28. data/lib/backup/configuration/storage/dropbox.rb +14 -4
  29. data/lib/backup/configuration/syncer/base.rb +10 -0
  30. data/lib/backup/configuration/syncer/rsync/base.rb +28 -0
  31. data/lib/backup/configuration/syncer/rsync/local.rb +11 -0
  32. data/lib/backup/configuration/syncer/rsync/pull.rb +11 -0
  33. data/lib/backup/configuration/syncer/rsync/push.rb +31 -0
  34. data/lib/backup/configuration/syncer/s3.rb +0 -4
  35. data/lib/backup/database/base.rb +25 -7
  36. data/lib/backup/database/mongodb.rb +112 -75
  37. data/lib/backup/database/mysql.rb +54 -29
  38. data/lib/backup/database/postgresql.rb +60 -42
  39. data/lib/backup/database/redis.rb +61 -39
  40. data/lib/backup/database/riak.rb +35 -11
  41. data/lib/backup/dependency.rb +4 -5
  42. data/lib/backup/encryptor/base.rb +13 -1
  43. data/lib/backup/encryptor/gpg.rb +39 -39
  44. data/lib/backup/encryptor/open_ssl.rb +28 -38
  45. data/lib/backup/logger.rb +20 -11
  46. data/lib/backup/model.rb +206 -163
  47. data/lib/backup/notifier/base.rb +27 -25
  48. data/lib/backup/notifier/campfire.rb +7 -13
  49. data/lib/backup/notifier/hipchat.rb +28 -28
  50. data/lib/backup/notifier/mail.rb +24 -26
  51. data/lib/backup/notifier/presently.rb +10 -18
  52. data/lib/backup/notifier/prowl.rb +9 -17
  53. data/lib/backup/notifier/twitter.rb +11 -18
  54. data/lib/backup/package.rb +47 -0
  55. data/lib/backup/packager.rb +81 -16
  56. data/lib/backup/splitter.rb +48 -35
  57. data/lib/backup/storage/base.rb +44 -172
  58. data/lib/backup/storage/cloudfiles.rb +31 -46
  59. data/lib/backup/storage/cycler.rb +117 -0
  60. data/lib/backup/storage/dropbox.rb +92 -76
  61. data/lib/backup/storage/ftp.rb +30 -40
  62. data/lib/backup/storage/local.rb +44 -45
  63. data/lib/backup/storage/ninefold.rb +55 -49
  64. data/lib/backup/storage/rsync.rb +49 -56
  65. data/lib/backup/storage/s3.rb +33 -44
  66. data/lib/backup/storage/scp.rb +21 -48
  67. data/lib/backup/storage/sftp.rb +26 -40
  68. data/lib/backup/syncer/base.rb +7 -0
  69. data/lib/backup/syncer/rsync/base.rb +78 -0
  70. data/lib/backup/syncer/rsync/local.rb +53 -0
  71. data/lib/backup/syncer/rsync/pull.rb +38 -0
  72. data/lib/backup/syncer/rsync/push.rb +113 -0
  73. data/lib/backup/syncer/s3.rb +42 -32
  74. data/lib/backup/version.rb +1 -1
  75. data/spec/archive_spec.rb +235 -69
  76. data/spec/cleaner_spec.rb +304 -0
  77. data/spec/cli/helpers_spec.rb +142 -1
  78. data/spec/cli/utility_spec.rb +338 -13
  79. data/spec/compressor/base_spec.rb +31 -0
  80. data/spec/compressor/bzip2_spec.rb +60 -35
  81. data/spec/compressor/gzip_spec.rb +60 -35
  82. data/spec/compressor/lzma_spec.rb +60 -35
  83. data/spec/compressor/pbzip2_spec.rb +98 -37
  84. data/spec/config_spec.rb +321 -0
  85. data/spec/configuration/base_spec.rb +4 -4
  86. data/spec/configuration/compressor/bzip2_spec.rb +1 -0
  87. data/spec/configuration/compressor/gzip_spec.rb +1 -0
  88. data/spec/configuration/compressor/lzma_spec.rb +1 -0
  89. data/spec/configuration/compressor/pbzip2_spec.rb +32 -0
  90. data/spec/configuration/database/base_spec.rb +2 -1
  91. data/spec/configuration/database/mongodb_spec.rb +26 -16
  92. data/spec/configuration/database/mysql_spec.rb +4 -0
  93. data/spec/configuration/database/postgresql_spec.rb +4 -0
  94. data/spec/configuration/database/redis_spec.rb +4 -0
  95. data/spec/configuration/database/riak_spec.rb +4 -0
  96. data/spec/configuration/encryptor/gpg_spec.rb +1 -0
  97. data/spec/configuration/encryptor/open_ssl_spec.rb +1 -0
  98. data/spec/configuration/notifier/base_spec.rb +32 -0
  99. data/spec/configuration/notifier/campfire_spec.rb +1 -0
  100. data/spec/configuration/notifier/hipchat_spec.rb +1 -0
  101. data/spec/configuration/notifier/mail_spec.rb +1 -0
  102. data/spec/configuration/notifier/presently_spec.rb +1 -0
  103. data/spec/configuration/notifier/prowl_spec.rb +1 -0
  104. data/spec/configuration/notifier/twitter_spec.rb +1 -0
  105. data/spec/configuration/storage/cloudfiles_spec.rb +1 -0
  106. data/spec/configuration/storage/dropbox_spec.rb +4 -3
  107. data/spec/configuration/storage/ftp_spec.rb +1 -0
  108. data/spec/configuration/storage/local_spec.rb +1 -0
  109. data/spec/configuration/storage/ninefold_spec.rb +1 -0
  110. data/spec/configuration/storage/rsync_spec.rb +3 -1
  111. data/spec/configuration/storage/s3_spec.rb +1 -0
  112. data/spec/configuration/storage/scp_spec.rb +1 -0
  113. data/spec/configuration/storage/sftp_spec.rb +1 -0
  114. data/spec/configuration/syncer/rsync/base_spec.rb +33 -0
  115. data/spec/configuration/syncer/rsync/local_spec.rb +10 -0
  116. data/spec/configuration/syncer/rsync/pull_spec.rb +10 -0
  117. data/spec/configuration/syncer/{rsync_spec.rb → rsync/push_spec.rb} +12 -15
  118. data/spec/configuration/syncer/s3_spec.rb +2 -3
  119. data/spec/database/base_spec.rb +35 -20
  120. data/spec/database/mongodb_spec.rb +298 -119
  121. data/spec/database/mysql_spec.rb +147 -72
  122. data/spec/database/postgresql_spec.rb +155 -100
  123. data/spec/database/redis_spec.rb +200 -97
  124. data/spec/database/riak_spec.rb +82 -24
  125. data/spec/dependency_spec.rb +49 -0
  126. data/spec/encryptor/base_spec.rb +30 -0
  127. data/spec/encryptor/gpg_spec.rb +105 -28
  128. data/spec/encryptor/open_ssl_spec.rb +85 -114
  129. data/spec/logger_spec.rb +74 -8
  130. data/spec/model_spec.rb +528 -220
  131. data/spec/notifier/base_spec.rb +89 -0
  132. data/spec/notifier/campfire_spec.rb +147 -119
  133. data/spec/notifier/hipchat_spec.rb +140 -145
  134. data/spec/notifier/mail_spec.rb +190 -248
  135. data/spec/notifier/presently_spec.rb +147 -282
  136. data/spec/notifier/prowl_spec.rb +79 -111
  137. data/spec/notifier/twitter_spec.rb +87 -106
  138. data/spec/package_spec.rb +61 -0
  139. data/spec/packager_spec.rb +154 -0
  140. data/spec/spec_helper.rb +36 -13
  141. data/spec/splitter_spec.rb +90 -41
  142. data/spec/storage/base_spec.rb +95 -239
  143. data/spec/storage/cloudfiles_spec.rb +185 -75
  144. data/spec/storage/cycler_spec.rb +239 -0
  145. data/spec/storage/dropbox_spec.rb +318 -87
  146. data/spec/storage/ftp_spec.rb +165 -152
  147. data/spec/storage/local_spec.rb +206 -54
  148. data/spec/storage/ninefold_spec.rb +264 -128
  149. data/spec/storage/rsync_spec.rb +244 -163
  150. data/spec/storage/s3_spec.rb +175 -64
  151. data/spec/storage/scp_spec.rb +156 -150
  152. data/spec/storage/sftp_spec.rb +153 -135
  153. data/spec/syncer/base_spec.rb +22 -0
  154. data/spec/syncer/rsync/base_spec.rb +118 -0
  155. data/spec/syncer/rsync/local_spec.rb +121 -0
  156. data/spec/syncer/rsync/pull_spec.rb +90 -0
  157. data/spec/syncer/rsync/push_spec.rb +327 -0
  158. data/spec/syncer/s3_spec.rb +180 -91
  159. data/templates/cli/utility/config +1 -1
  160. data/templates/cli/utility/database/mongodb +4 -0
  161. data/templates/cli/utility/database/mysql +3 -0
  162. data/templates/cli/utility/database/postgresql +3 -0
  163. data/templates/cli/utility/database/redis +3 -0
  164. data/templates/cli/utility/database/riak +3 -0
  165. data/templates/cli/utility/storage/dropbox +4 -1
  166. data/templates/cli/utility/syncer/rsync_local +12 -0
  167. data/templates/cli/utility/syncer/{rsync → rsync_pull} +2 -2
  168. data/templates/cli/utility/syncer/rsync_push +17 -0
  169. data/templates/storage/dropbox/authorization_url.erb +1 -1
  170. metadata +42 -17
  171. data/lib/backup/configuration/syncer/rsync.rb +0 -45
  172. data/lib/backup/finder.rb +0 -87
  173. data/lib/backup/storage/object.rb +0 -47
  174. data/lib/backup/syncer/rsync.rb +0 -152
  175. data/spec/backup_spec.rb +0 -11
  176. data/spec/finder_spec.rb +0 -91
  177. data/spec/storage/object_spec.rb +0 -74
  178. data/spec/syncer/rsync_spec.rb +0 -195
data/Gemfile CHANGED
@@ -6,12 +6,8 @@ source 'http://rubygems.org'
6
6
  # Include gem dependencies from the gemspec for development purposes
7
7
  gemspec
8
8
 
9
- # Load Backup::Dependency
10
- ["cli/helpers", "dependency"].each do |library|
11
- require File.expand_path("../lib/backup/#{library}", __FILE__)
12
- end
13
-
14
9
  # Dynamically define the dependencies specified in Backup::Dependency.all
10
+ require File.expand_path("../lib/backup/dependency", __FILE__)
15
11
  Backup::Dependency.all.each do |name, gemspec|
16
12
  gem(name, gemspec[:version])
17
13
  end
@@ -1,7 +1,7 @@
1
1
  PATH
2
2
  remote: .
3
3
  specs:
4
- backup (3.0.19)
4
+ backup (3.0.21)
5
5
  POpen4 (~> 0.1.4)
6
6
  thor (~> 0.14.6)
7
7
 
@@ -12,82 +12,80 @@ GEM
12
12
  Platform (>= 0.4.0)
13
13
  open4
14
14
  Platform (0.4.0)
15
+ activesupport (3.1.3)
16
+ multi_json (~> 1.0)
15
17
  addressable (2.2.6)
16
18
  builder (3.0.0)
17
19
  crack (0.1.8)
18
- diff-lcs (1.1.2)
19
- dropbox (1.3.0)
20
- json (>= 1.2.0)
21
- multipart-post (>= 1.1.0)
22
- oauth (>= 0.3.6)
23
- excon (0.6.6)
24
- faraday (0.7.4)
20
+ diff-lcs (1.1.3)
21
+ dropbox-sdk (1.1)
22
+ json
23
+ excon (0.9.4)
24
+ faraday (0.7.5)
25
25
  addressable (~> 2.2.6)
26
- multipart-post (~> 1.1.0)
26
+ multipart-post (~> 1.1.3)
27
27
  rack (>= 1.1.0, < 2)
28
- faraday_middleware (0.7.0)
29
- faraday (~> 0.7.3)
30
- ffi (1.0.9)
31
- fog (0.11.0)
28
+ ffi (1.0.11)
29
+ fog (1.1.2)
32
30
  builder
33
- excon (~> 0.6.5)
31
+ excon (~> 0.9.0)
34
32
  formatador (~> 0.2.0)
35
33
  mime-types
36
34
  multi_json (~> 1.0.3)
37
35
  net-scp (~> 1.0.4)
38
- net-ssh (~> 2.1.4)
36
+ net-ssh (>= 2.1.3)
39
37
  nokogiri (~> 1.5.0)
40
38
  ruby-hmac
41
39
  formatador (0.2.1)
42
- fuubar (0.0.5)
40
+ fuubar (0.0.6)
43
41
  rspec (~> 2.0)
44
- rspec-instafail (~> 0.1.4)
42
+ rspec-instafail (~> 0.1.8)
45
43
  ruby-progressbar (~> 0.0.10)
46
44
  growl (1.0.3)
47
- guard (0.3.4)
45
+ guard (0.10.0)
46
+ ffi (>= 0.5.0)
48
47
  thor (~> 0.14.6)
49
- guard-rspec (0.3.1)
50
- guard (>= 0.2.2)
51
- hashie (1.1.0)
48
+ guard-rspec (0.6.0)
49
+ guard (>= 0.10.0)
52
50
  hipchat (0.4.1)
53
51
  httparty
54
- httparty (0.7.4)
52
+ httparty (0.7.8)
55
53
  crack (= 0.1.8)
56
54
  i18n (0.6.0)
57
55
  json (1.5.4)
58
- libnotify (0.5.5)
56
+ libnotify (0.7.1)
59
57
  mail (2.3.0)
60
58
  i18n (>= 0.4.0)
61
59
  mime-types (~> 1.16)
62
60
  treetop (~> 1.4.8)
63
- mime-types (1.16)
64
- mocha (0.9.12)
65
- multi_json (1.0.3)
66
- multi_xml (0.3.0)
67
- multipart-post (1.1.3)
61
+ metaclass (0.0.1)
62
+ mime-types (1.17.2)
63
+ mocha (0.10.0)
64
+ metaclass (~> 0.0.1)
65
+ multi_json (1.0.4)
66
+ multipart-post (1.1.4)
68
67
  net-scp (1.0.4)
69
68
  net-ssh (>= 1.99.1)
70
69
  net-sftp (2.0.5)
71
70
  net-ssh (>= 2.0.9)
72
71
  net-ssh (2.1.4)
73
72
  nokogiri (1.5.0)
74
- oauth (0.4.5)
75
- open4 (1.2.0)
76
- polyglot (0.3.2)
73
+ open4 (1.3.0)
74
+ polyglot (0.3.3)
77
75
  prowler (1.3.1)
78
- rack (1.3.2)
79
- rb-fsevent (0.4.0)
80
- rb-inotify (0.8.5)
76
+ rack (1.4.0)
77
+ rb-fsevent (0.4.3.1)
78
+ rb-inotify (0.8.8)
81
79
  ffi (>= 0.5.0)
82
- rspec (2.5.0)
83
- rspec-core (~> 2.5.0)
84
- rspec-expectations (~> 2.5.0)
85
- rspec-mocks (~> 2.5.0)
86
- rspec-core (2.5.1)
87
- rspec-expectations (2.5.0)
80
+ rspec (2.8.0)
81
+ rspec-core (~> 2.8.0)
82
+ rspec-expectations (~> 2.8.0)
83
+ rspec-mocks (~> 2.8.0)
84
+ rspec-core (2.8.0)
85
+ rspec-expectations (2.8.0)
88
86
  diff-lcs (~> 1.1.2)
89
- rspec-instafail (0.1.7)
90
- rspec-mocks (2.5.0)
87
+ rspec-instafail (0.1.9)
88
+ rspec-mocks (2.8.0)
91
89
  ruby-hmac (0.4.0)
92
90
  ruby-progressbar (0.0.10)
93
91
  simple_oauth (0.1.5)
@@ -96,20 +94,18 @@ GEM
96
94
  treetop (1.4.10)
97
95
  polyglot
98
96
  polyglot (>= 0.3.1)
99
- twitter (1.7.1)
100
- faraday (~> 0.7.4)
101
- faraday_middleware (~> 0.7.0)
102
- hashie (~> 1.1.0)
103
- multi_json (~> 1.0.0)
104
- multi_xml (~> 0.3.0)
105
- simple_oauth (~> 0.1.5)
97
+ twitter (2.0.2)
98
+ activesupport (>= 2.3.9, < 4)
99
+ faraday (~> 0.7)
100
+ multi_json (~> 1.0)
101
+ simple_oauth (~> 0.1)
106
102
 
107
103
  PLATFORMS
108
104
  ruby
109
105
 
110
106
  DEPENDENCIES
111
107
  backup!
112
- dropbox (~> 1.3.0)
108
+ dropbox-sdk (~> 1.1.0)
113
109
  fog (>= 0.11.0)
114
110
  fuubar
115
111
  growl
data/README.md CHANGED
@@ -15,12 +15,6 @@ Author
15
15
  Drop me a message for any questions, suggestions, requests, bugs or submit them to the [issue log](https://github.com/meskyanichi/backup/issues).
16
16
 
17
17
 
18
- Core Contributor
19
- ----------------
20
-
21
- **[Brian Burns](https://github.com/burns)**
22
-
23
-
24
18
  Installation
25
19
  ------------
26
20
 
@@ -92,14 +86,14 @@ Storage Features
92
86
  - **Incremental Backups, applies to:**
93
87
  - Remote Servers *(Only Protocols: RSync)*
94
88
 
95
- [Storage Wiki Page](https://github.com/meskyanichi/backup/wiki/Storages)
89
+ [Cycling Wiki Page](https://github.com/meskyanichi/backup/wiki/Cycling)
96
90
 
97
91
  [Splitter Wiki Page](https://github.com/meskyanichi/backup/wiki/Splitter)
98
92
 
99
93
  Syncers
100
94
  -------
101
95
 
102
- - RSync
96
+ - RSync (Push, Pull and Local)
103
97
  - Amazon Simple Storage Service (S3)
104
98
 
105
99
  [Syncer Wiki Page](https://github.com/meskyanichi/backup/wiki/Syncers)
@@ -240,21 +234,50 @@ end
240
234
 
241
235
  ### Brief explanation for the above example configuration
242
236
 
243
- It will dump two databases (MySQL and MongoDB), it'll create two (.t)archives (user_avatars and logs). It'll package the two database and two archives together in a single (.t)archive. It'll run the Gzip compressor to compress that archive, and then it'll run the OpenSSL encryptor to encrypt the compressed archive. Then that encrypted archive will be stored to your Amazon S3 account. If all goes well, and no exceptions are raised, you'll be notified via the Twitter notifier that the backup succeeded. If there was an exception raised during the backup process, then you'd receive an email in your inbox containing detailed exception information, as well as receive a simple Twitter message that something went wrong.
244
-
245
- Aside of S3, we have also defined two `SFTP` storage methods, and given them two unique identifiers `Server A` and `Server B` to distinguish between the two. With these in place, a copy of the backup will now also be stored on two separate servers: `a.my-backup-server.com` and `b.my-backup-server.com`.
246
-
247
- As you can see, you can freely mix and match **archives**, **databases**, **compressors**, **encryptors**, **storages** and **notifiers** for your backups. You could even specify 4 storage locations if you wanted: Amazon S3, Rackspace Cloud Files, Ninefold and Dropbox, it'd then store your packaged backup to 4 separate locations for high redundancy. This also applies to compressors (like Gzip, Bzip2, Lzma) and encryptors, you could double encrypt your backup with OpenSSL followed by GPG if you wanted.
248
-
249
- Also, notice the `split_into_chunks_of 4000` at the top of the configuration. This tells Backup to split any backups that exceed in 4000 MEGABYTES of size in to multiple smaller chunks. Assuming your backup file is 12000 MEGABYTES (12GB) in size, then Backup will go ahead and split it in to 3 chunks of 4000 MEGABYTES and transfer them individually. This is useful for when you are using Amazon S3, Rackspace Cloud Files, or other 3rd party storage services which limit you to "5GB per file" uploads. So with this, the backup file size is no longer a constraint.
250
-
251
- Additionally we have also defined a **S3 Syncer** ( `sync_with S3` ), which does not follow the above process of archiving/compression/encryption, but instead will directly sync the whole `videos` and `music` folder structures from your machine to your Amazon S3 account. (very efficient and cost-effective since it will only transfer files that were added/changed. Additionally, since we flagged it to 'mirror', it'll also remove files from S3 that no longer exist). If you simply wanted to sync to a separate backup server that you own, you could also use the RSync syncer for even more efficient backups that only transfer the **bytes** of each file that changed.
252
-
253
- There are more **archives**, **databases**, **compressors**, **encryptors**, **storages** and **notifiers** than displayed in the example, all available components are listed at the top of this README, as well as in the [Wiki](https://github.com/meskyanichi/backup/wiki) with more detailed information.
237
+ First, it will dump the two Databases (MySQL and MongoDB). The MySQL dump will be piped through the Gzip Compressor into
238
+ `sample_backup/databases/MySQL/my_sample_mysql_db.sql.gz`. The MongoDB dump will be dumped into
239
+ `sample_backup/databases/MongoDB/`, which will then be packaged into `sample_backup/databases/MongoDB-#####.tar.gz`
240
+ (`#####` will be a simple unique identifier, in case multiple dumps are performed.)
241
+ Next, it will create two _tar_ Archives (user_avatars and logs). Each will be piped through the Gzip Compressor into
242
+ `sample_backup/archives/` as `user_archives.tar.gz` and `logs.tar.gz`.
243
+ Finally, the `sample_backup` directory will be packaged into an uncompressed _tar_ archive, which will be piped through
244
+ the OpenSSL Encryptor to encrypt this final package into `YYYY-MM-DD-hh-mm-ss.sample_backup.tar.enc`. This final
245
+ encrypted archive will then be transfered to your Amazon S3 account. If all goes well, and no exceptions are raised,
246
+ you'll be notified via the Twitter notifier that the backup succeeded. If any warnings were issued or there was an
247
+ exception raised during the backup process, then you'd receive an email in your inbox containing detailed exception
248
+ information, as well as receive a simple Twitter message that something went wrong.
249
+
250
+ Aside of S3, we have also defined two `SFTP` storage methods, and given them two unique identifiers `Server A` and
251
+ `Server B` to distinguish between the two. With these in place, a copy of the backup will now also be stored on two
252
+ separate servers: `a.my-backup-server.com` and `b.my-backup-server.com`.
253
+
254
+ As you can see, you can freely mix and match **archives**, **databases**, **compressors**, **encryptors**, **storages**
255
+ and **notifiers** for your backups. You could even specify 4 storage locations if you wanted: Amazon S3, Rackspace Cloud
256
+ Files, Ninefold and Dropbox, it'd then store your packaged backup to 4 separate locations for high redundancy.
257
+
258
+ Also, notice the `split_into_chunks_of 4000` at the top of the configuration. This tells Backup to split any backups
259
+ that exceed in 4000 MEGABYTES of size in to multiple smaller chunks. Assuming your backup file is 12000 MEGABYTES (12GB)
260
+ in size, then Backup will take the output which was piped from _tar_ into the OpenSSL Compressor and additionally pipe
261
+ that output through the _split_ utility, which will result in 3 chunks of 4000 MEGABYTES with additional file extensions
262
+ of `-aa`, `-ab` and `-ac`. These files will then be individually transfered. This is useful for when you are using
263
+ Amazon S3, Rackspace Cloud Files, or other 3rd party storage services which limit you to "5GB per file" uploads. So with
264
+ this, the backup file size is no longer a constraint.
265
+
266
+ Additionally we have also defined a **S3 Syncer** ( `sync_with S3` ), which does not follow the above process of
267
+ archiving/compression/encryption, but instead will directly sync the whole `videos` and `music` folder structures from
268
+ your machine to your Amazon S3 account. (very efficient and cost-effective since it will only transfer files that were
269
+ added/changed. Additionally, since we flagged it to 'mirror', it'll also remove files from S3 that no longer exist). If
270
+ you simply wanted to sync to a separate backup server that you own, you could also use the RSync syncer for even more
271
+ efficient backups that only transfer the **bytes** of each file that changed.
272
+
273
+ There are more **archives**, **databases**, **compressors**, **encryptors**, **storages** and **notifiers** than
274
+ displayed in the example, all available components are listed at the top of this README, as well as in the
275
+ [Wiki](https://github.com/meskyanichi/backup/wiki) with more detailed information.
254
276
 
255
277
  ### Running the example
256
278
 
257
- Notice the `Backup::Model.new(:sample_backup, 'A sample backup configuration') do` at the top of the above example. The `:sample_backup` is called the **trigger**. This is used to identify the backup procedure/file and initialize it.
279
+ Notice the `Backup::Model.new(:sample_backup, 'A sample backup configuration') do` at the top of the above example. The
280
+ `:sample_backup` is called the **trigger**. This is used to identify the backup procedure/file and initialize it.
258
281
 
259
282
  ``` sh
260
283
  backup perform -t [--trigger] sample_backup
@@ -264,7 +287,9 @@ Now it'll run the backup, it's as simple as that.
264
287
 
265
288
  ### Automatic backups
266
289
 
267
- Since Backup is an easy-to-use command line utility, you should write a crontask to invoke it periodically. I recommend using [Whenever](https://github.com/javan/whenever) to manage your crontab. It'll allow you to write to the crontab using pure Ruby, and it provides an elegant DSL to do so. Here's an example:
290
+ Since Backup is an easy-to-use command line utility, you should write a crontask to invoke it periodically. I recommend
291
+ using [Whenever](https://github.com/javan/whenever) to manage your crontab. It'll allow you to write to the crontab
292
+ using pure Ruby, and it provides an elegant DSL to do so. Here's an example:
268
293
 
269
294
  ``` rb
270
295
  every 6.hours do
@@ -272,12 +297,14 @@ every 6.hours do
272
297
  end
273
298
  ```
274
299
 
275
- With this in place, run `whenever --update-crontab backup` to write the equivalent of the above Ruby syntax to the crontab in cron-syntax. Cron will now invoke `backup perform --trigger sample_backup` every 6 hours. Check out the Whenever project page for more information.
300
+ With this in place, run `whenever --update-crontab backup` to write the equivalent of the above Ruby syntax to the
301
+ crontab in cron-syntax. Cron will now invoke `backup perform --trigger sample_backup` every 6 hours. Check out the
302
+ Whenever project page for more information.
276
303
 
277
304
  Documentation
278
305
  -------------
279
306
 
280
- See the [Wiki Pages](https://github.com/meskyanichi/backup/wiki). The subjects labeled **without** the "Backup 2)"-prefix are meant for Backup 3 users.
307
+ See the [Wiki Pages](https://github.com/meskyanichi/backup/wiki).
281
308
 
282
309
 
283
310
  Suggestions, Bugs, Requests, Questions
@@ -293,6 +320,10 @@ Contributors
293
320
  <th>Contributor</th>
294
321
  <th>Contribution</th>
295
322
  </tr>
323
+ <tr>
324
+ <td><a href="https://github.com/burns" target="_blank"><b>Brian D. Burns ( burns )</b></a></td>
325
+ <td><b>Core Contributor</b></td>
326
+ </tr>
296
327
  <tr>
297
328
  <td><a href="https://github.com/asanghi" target="_blank">Aditya Sanghi ( asanghi )</a></td>
298
329
  <td>Twitter Notifier, Dropbox Timeout Configuration</td>
@@ -409,10 +440,6 @@ Contributors
409
440
  <td><a href="https://github.com/szymonpk" target="_blank">Szymon ( szymonpk )</a></td>
410
441
  <td>Pbzip2 compressor</td>
411
442
  </tr>
412
- <tr>
413
- <td><a href="https://github.com/burns" target="_blank">burns ( burns )</a></td>
414
- <td>Improved Backup cycling implementation by refreshing all user configuration during the cycle procedure</td>
415
- </tr>
416
443
  </table>
417
444
 
418
445
 
@@ -23,22 +23,6 @@ end
23
23
  # The Backup Ruby Gem
24
24
  module Backup
25
25
 
26
- ##
27
- # List the available database, storage, compressor, encryptor and notifier constants.
28
- # These are used to dynamically define these constants as classes inside Backup::Finder
29
- # to provide a nicer configuration file DSL syntax to the users. Adding existing constants
30
- # to the arrays below will enable the user to use a constant instead of a string.
31
- # Example, instead of:
32
- # database "MySQL" do |mysql|
33
- # You can do:
34
- # database MySQL do |mysql|
35
- DATABASES = ['MySQL', 'PostgreSQL', 'MongoDB', 'Redis', 'Riak']
36
- STORAGES = ['S3', 'CloudFiles', 'Ninefold', 'Dropbox', 'FTP', 'SFTP', 'SCP', 'RSync', 'Local']
37
- COMPRESSORS = ['Gzip', 'Bzip2', 'Pbzip2', 'Lzma']
38
- ENCRYPTORS = ['OpenSSL', 'GPG']
39
- SYNCERS = ['RSync', 'S3']
40
- NOTIFIERS = ['Mail', 'Twitter', 'Campfire', 'Presently', 'Prowl', 'Hipchat']
41
-
42
26
  ##
43
27
  # Backup's internal paths
44
28
  LIBRARY_PATH = File.join(File.dirname(__FILE__), 'backup')
@@ -52,25 +36,15 @@ module Backup
52
36
  CONFIGURATION_PATH = File.join(LIBRARY_PATH, 'configuration')
53
37
  TEMPLATE_PATH = File.expand_path('../../templates', __FILE__)
54
38
 
55
- ##
56
- # Backup's Environment paths
57
- USER = ENV['USER'] || Etc.getpwuid.name
58
- HOME = File.expand_path(ENV['HOME'] || '')
59
- PATH = File.join(HOME, 'Backup')
60
- CONFIG_FILE = File.join(PATH, 'config.rb')
61
- DATA_PATH = File.join(PATH, 'data')
62
- LOG_PATH = File.join(PATH, 'log')
63
- CACHE_PATH = File.join(PATH, '.cache')
64
- TMP_PATH = File.join(PATH, '.tmp')
65
-
66
39
  ##
67
40
  # Autoload Backup base files
68
41
  autoload :Model, File.join(LIBRARY_PATH, 'model')
69
42
  autoload :Archive, File.join(LIBRARY_PATH, 'archive')
70
43
  autoload :Packager, File.join(LIBRARY_PATH, 'packager')
44
+ autoload :Package, File.join(LIBRARY_PATH, 'package')
71
45
  autoload :Cleaner, File.join(LIBRARY_PATH, 'cleaner')
72
46
  autoload :Splitter, File.join(LIBRARY_PATH, 'splitter')
73
- autoload :Finder, File.join(LIBRARY_PATH, 'finder')
47
+ autoload :Config, File.join(LIBRARY_PATH, 'config')
74
48
  autoload :Binder, File.join(LIBRARY_PATH, 'binder')
75
49
  autoload :Template, File.join(LIBRARY_PATH, 'template')
76
50
  autoload :Dependency, File.join(LIBRARY_PATH, 'dependency')
@@ -89,7 +63,7 @@ module Backup
89
63
  # Autoload Backup storage files
90
64
  module Storage
91
65
  autoload :Base, File.join(STORAGE_PATH, 'base')
92
- autoload :Object, File.join(STORAGE_PATH, 'object')
66
+ autoload :Cycler, File.join(STORAGE_PATH, 'cycler')
93
67
  autoload :S3, File.join(STORAGE_PATH, 's3')
94
68
  autoload :CloudFiles, File.join(STORAGE_PATH, 'cloudfiles')
95
69
  autoload :Ninefold, File.join(STORAGE_PATH, 'ninefold')
@@ -105,8 +79,13 @@ module Backup
105
79
  # Autoload Backup syncer files
106
80
  module Syncer
107
81
  autoload :Base, File.join(SYNCER_PATH, 'base')
108
- autoload :RSync, File.join(SYNCER_PATH, 'rsync')
109
82
  autoload :S3, File.join(SYNCER_PATH, 's3')
83
+ module RSync
84
+ autoload :Base, File.join(SYNCER_PATH, 'rsync', 'base')
85
+ autoload :Local, File.join(SYNCER_PATH, 'rsync', 'local')
86
+ autoload :Push, File.join(SYNCER_PATH, 'rsync', 'push')
87
+ autoload :Pull, File.join(SYNCER_PATH, 'rsync', 'pull')
88
+ end
110
89
  end
111
90
 
112
91
  ##
@@ -195,8 +174,14 @@ module Backup
195
174
  end
196
175
 
197
176
  module Syncer
198
- autoload :RSync, File.join(CONFIGURATION_PATH, 'syncer', 'rsync')
177
+ autoload :Base, File.join(CONFIGURATION_PATH, 'syncer', 'base')
199
178
  autoload :S3, File.join(CONFIGURATION_PATH, 'syncer', 's3')
179
+ module RSync
180
+ autoload :Base, File.join(CONFIGURATION_PATH, 'syncer', 'rsync', 'base')
181
+ autoload :Local, File.join(CONFIGURATION_PATH, 'syncer', 'rsync', 'local')
182
+ autoload :Push, File.join(CONFIGURATION_PATH, 'syncer', 'rsync', 'push')
183
+ autoload :Pull, File.join(CONFIGURATION_PATH, 'syncer', 'rsync', 'pull')
184
+ end
200
185
  end
201
186
 
202
187
  module Database
@@ -209,12 +194,4 @@ module Backup
209
194
  end
210
195
  end
211
196
 
212
- ##
213
- # Dynamically defines all the available database, storage, compressor, encryptor and notifier
214
- # classes inside Backup::Finder to improve the DSL for the configuration file
215
- (DATABASES + STORAGES + COMPRESSORS + ENCRYPTORS + NOTIFIERS + SYNCERS).each do |constant|
216
- unless Backup::Finder.const_defined?(constant)
217
- Backup::Finder.const_set(constant, Class.new)
218
- end
219
- end
220
197
  end
@@ -17,24 +17,34 @@ module Backup
17
17
  attr_accessor :excludes
18
18
 
19
19
  ##
20
- # Stores the path to the archive directory
21
- attr_accessor :archive_path
20
+ # String of additional arguments for the `tar` command
21
+ attr_accessor :tar_args
22
22
 
23
23
  ##
24
24
  # Takes the name of the archive and the configuration block
25
- def initialize(name, &block)
26
- @name = name.to_sym
25
+ def initialize(model, name, &block)
26
+ @model = model
27
+ @name = name.to_s
27
28
  @paths = Array.new
28
29
  @excludes = Array.new
29
- @tar_options = ''
30
+ @tar_args = ''
30
31
 
31
- instance_eval(&block)
32
+ instance_eval(&block) if block_given?
32
33
  end
33
34
 
34
35
  ##
35
36
  # Adds new paths to the @paths instance variable array
36
37
  def add(path)
37
- @paths << File.expand_path(path)
38
+ path = File.expand_path(path)
39
+ if File.exist?(path)
40
+ @paths << path
41
+ else
42
+ Logger.warn Errors::Archive::NotFoundError.new(<<-EOS)
43
+ The following path was not found:
44
+ #{ path }
45
+ This path will be omitted from the '#{ name }' Archive.
46
+ EOS
47
+ end
38
48
  end
39
49
 
40
50
  ##
@@ -47,39 +57,53 @@ module Backup
47
57
  # Adds the given String of +options+ to the `tar` command.
48
58
  # e.g. '-h --xattrs'
49
59
  def tar_options(options)
50
- @tar_options = options
60
+ @tar_args = options
51
61
  end
52
62
 
53
63
  ##
54
64
  # Archives all the provided paths in to a single .tar file
55
65
  # and places that .tar file in the folder which later will be packaged
66
+ # If the model is configured with a Compressor, the tar command output
67
+ # will be piped through the Compressor command and the file extension
68
+ # will be adjusted to indicate the type of compression used.
56
69
  def perform!
57
- @archive_path = File.join(TMP_PATH, TRIGGER, 'archive')
58
- mkdir(archive_path)
59
-
60
70
  Logger.message "#{ self.class } started packaging and archiving:\n" +
61
71
  paths.map {|path| " #{path}" }.join("\n")
62
72
 
63
- run("#{ utility(:tar) } #{ @tar_options } -cf '#{ File.join(archive_path, "#{name}.tar") }' " +
64
- "#{ paths_to_exclude } #{ paths_to_package }", :ignore_exit_codes => [1])
73
+ archive_path = File.join(Config.tmp_path, @model.trigger, 'archives')
74
+ FileUtils.mkdir_p(archive_path)
75
+
76
+ archive_ext = 'tar'
77
+ archive_cmd = "#{ utility(:tar) } #{ tar_args } -cf - " +
78
+ "#{ paths_to_exclude } #{ paths_to_package }"
79
+
80
+ if @model.compressor
81
+ @model.compressor.compress_with do |command, ext|
82
+ archive_cmd << " | #{command}"
83
+ archive_ext << ext
84
+ end
85
+ end
86
+
87
+ archive_cmd << " > '#{ File.join(archive_path, "#{name}.#{archive_ext}") }'"
88
+
89
+ run(archive_cmd)
65
90
  end
66
91
 
67
- private
92
+ private
68
93
 
69
94
  ##
70
95
  # Returns a "tar-ready" string of all the specified paths combined
71
96
  def paths_to_package
72
- paths.map do |path|
73
- "'#{path}'"
74
- end.join("\s")
97
+ paths.map {|path| "'#{path}'" }.join(' ')
75
98
  end
76
99
 
77
100
  ##
78
101
  # Returns a "tar-ready" string of all the specified excludes combined
79
102
  def paths_to_exclude
80
103
  if excludes.any?
81
- excludes.map{ |e| "--exclude='#{e}'" }.join(" ")
104
+ excludes.map {|path| "--exclude='#{path}'" }.join(' ')
82
105
  end
83
106
  end
107
+
84
108
  end
85
109
  end