backup-agoddard 3.0.27

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of backup-agoddard might be problematic. Click here for more details.

Files changed (190) hide show
  1. data/.gitignore +8 -0
  2. data/.travis.yml +10 -0
  3. data/Gemfile +28 -0
  4. data/Guardfile +23 -0
  5. data/LICENSE.md +24 -0
  6. data/README.md +478 -0
  7. data/backup.gemspec +32 -0
  8. data/bin/backup +11 -0
  9. data/lib/backup.rb +133 -0
  10. data/lib/backup/archive.rb +117 -0
  11. data/lib/backup/binder.rb +22 -0
  12. data/lib/backup/cleaner.rb +121 -0
  13. data/lib/backup/cli/helpers.rb +93 -0
  14. data/lib/backup/cli/utility.rb +255 -0
  15. data/lib/backup/compressor/base.rb +35 -0
  16. data/lib/backup/compressor/bzip2.rb +50 -0
  17. data/lib/backup/compressor/custom.rb +53 -0
  18. data/lib/backup/compressor/gzip.rb +50 -0
  19. data/lib/backup/compressor/lzma.rb +52 -0
  20. data/lib/backup/compressor/pbzip2.rb +59 -0
  21. data/lib/backup/config.rb +174 -0
  22. data/lib/backup/configuration.rb +33 -0
  23. data/lib/backup/configuration/helpers.rb +130 -0
  24. data/lib/backup/configuration/store.rb +24 -0
  25. data/lib/backup/database/base.rb +53 -0
  26. data/lib/backup/database/mongodb.rb +230 -0
  27. data/lib/backup/database/mysql.rb +160 -0
  28. data/lib/backup/database/postgresql.rb +144 -0
  29. data/lib/backup/database/redis.rb +136 -0
  30. data/lib/backup/database/riak.rb +67 -0
  31. data/lib/backup/dependency.rb +108 -0
  32. data/lib/backup/encryptor/base.rb +29 -0
  33. data/lib/backup/encryptor/gpg.rb +760 -0
  34. data/lib/backup/encryptor/open_ssl.rb +72 -0
  35. data/lib/backup/errors.rb +124 -0
  36. data/lib/backup/hooks.rb +68 -0
  37. data/lib/backup/logger.rb +152 -0
  38. data/lib/backup/model.rb +409 -0
  39. data/lib/backup/notifier/base.rb +81 -0
  40. data/lib/backup/notifier/campfire.rb +155 -0
  41. data/lib/backup/notifier/hipchat.rb +93 -0
  42. data/lib/backup/notifier/mail.rb +206 -0
  43. data/lib/backup/notifier/prowl.rb +65 -0
  44. data/lib/backup/notifier/pushover.rb +88 -0
  45. data/lib/backup/notifier/twitter.rb +70 -0
  46. data/lib/backup/package.rb +47 -0
  47. data/lib/backup/packager.rb +100 -0
  48. data/lib/backup/pipeline.rb +110 -0
  49. data/lib/backup/splitter.rb +75 -0
  50. data/lib/backup/storage/base.rb +99 -0
  51. data/lib/backup/storage/cloudfiles.rb +87 -0
  52. data/lib/backup/storage/cycler.rb +117 -0
  53. data/lib/backup/storage/dropbox.rb +178 -0
  54. data/lib/backup/storage/ftp.rb +119 -0
  55. data/lib/backup/storage/local.rb +82 -0
  56. data/lib/backup/storage/ninefold.rb +116 -0
  57. data/lib/backup/storage/rsync.rb +149 -0
  58. data/lib/backup/storage/s3.rb +94 -0
  59. data/lib/backup/storage/scp.rb +99 -0
  60. data/lib/backup/storage/sftp.rb +108 -0
  61. data/lib/backup/syncer/base.rb +46 -0
  62. data/lib/backup/syncer/cloud/base.rb +247 -0
  63. data/lib/backup/syncer/cloud/cloud_files.rb +78 -0
  64. data/lib/backup/syncer/cloud/s3.rb +68 -0
  65. data/lib/backup/syncer/rsync/base.rb +49 -0
  66. data/lib/backup/syncer/rsync/local.rb +55 -0
  67. data/lib/backup/syncer/rsync/pull.rb +36 -0
  68. data/lib/backup/syncer/rsync/push.rb +116 -0
  69. data/lib/backup/template.rb +46 -0
  70. data/lib/backup/version.rb +43 -0
  71. data/spec-live/.gitignore +6 -0
  72. data/spec-live/README +7 -0
  73. data/spec-live/backups/config.rb +83 -0
  74. data/spec-live/backups/config.yml.template +46 -0
  75. data/spec-live/backups/models.rb +184 -0
  76. data/spec-live/compressor/custom_spec.rb +30 -0
  77. data/spec-live/compressor/gzip_spec.rb +30 -0
  78. data/spec-live/encryptor/gpg_keys.rb +239 -0
  79. data/spec-live/encryptor/gpg_spec.rb +287 -0
  80. data/spec-live/notifier/mail_spec.rb +121 -0
  81. data/spec-live/spec_helper.rb +151 -0
  82. data/spec-live/storage/dropbox_spec.rb +151 -0
  83. data/spec-live/storage/local_spec.rb +83 -0
  84. data/spec-live/storage/scp_spec.rb +193 -0
  85. data/spec-live/syncer/cloud/s3_spec.rb +124 -0
  86. data/spec/archive_spec.rb +335 -0
  87. data/spec/cleaner_spec.rb +312 -0
  88. data/spec/cli/helpers_spec.rb +301 -0
  89. data/spec/cli/utility_spec.rb +411 -0
  90. data/spec/compressor/base_spec.rb +52 -0
  91. data/spec/compressor/bzip2_spec.rb +217 -0
  92. data/spec/compressor/custom_spec.rb +106 -0
  93. data/spec/compressor/gzip_spec.rb +217 -0
  94. data/spec/compressor/lzma_spec.rb +123 -0
  95. data/spec/compressor/pbzip2_spec.rb +165 -0
  96. data/spec/config_spec.rb +321 -0
  97. data/spec/configuration/helpers_spec.rb +247 -0
  98. data/spec/configuration/store_spec.rb +39 -0
  99. data/spec/configuration_spec.rb +62 -0
  100. data/spec/database/base_spec.rb +63 -0
  101. data/spec/database/mongodb_spec.rb +510 -0
  102. data/spec/database/mysql_spec.rb +411 -0
  103. data/spec/database/postgresql_spec.rb +353 -0
  104. data/spec/database/redis_spec.rb +334 -0
  105. data/spec/database/riak_spec.rb +176 -0
  106. data/spec/dependency_spec.rb +51 -0
  107. data/spec/encryptor/base_spec.rb +40 -0
  108. data/spec/encryptor/gpg_spec.rb +909 -0
  109. data/spec/encryptor/open_ssl_spec.rb +148 -0
  110. data/spec/errors_spec.rb +306 -0
  111. data/spec/hooks_spec.rb +35 -0
  112. data/spec/logger_spec.rb +367 -0
  113. data/spec/model_spec.rb +694 -0
  114. data/spec/notifier/base_spec.rb +104 -0
  115. data/spec/notifier/campfire_spec.rb +217 -0
  116. data/spec/notifier/hipchat_spec.rb +211 -0
  117. data/spec/notifier/mail_spec.rb +316 -0
  118. data/spec/notifier/prowl_spec.rb +138 -0
  119. data/spec/notifier/pushover_spec.rb +123 -0
  120. data/spec/notifier/twitter_spec.rb +153 -0
  121. data/spec/package_spec.rb +61 -0
  122. data/spec/packager_spec.rb +213 -0
  123. data/spec/pipeline_spec.rb +259 -0
  124. data/spec/spec_helper.rb +60 -0
  125. data/spec/splitter_spec.rb +120 -0
  126. data/spec/storage/base_spec.rb +166 -0
  127. data/spec/storage/cloudfiles_spec.rb +254 -0
  128. data/spec/storage/cycler_spec.rb +247 -0
  129. data/spec/storage/dropbox_spec.rb +480 -0
  130. data/spec/storage/ftp_spec.rb +271 -0
  131. data/spec/storage/local_spec.rb +259 -0
  132. data/spec/storage/ninefold_spec.rb +343 -0
  133. data/spec/storage/rsync_spec.rb +362 -0
  134. data/spec/storage/s3_spec.rb +245 -0
  135. data/spec/storage/scp_spec.rb +233 -0
  136. data/spec/storage/sftp_spec.rb +244 -0
  137. data/spec/syncer/base_spec.rb +109 -0
  138. data/spec/syncer/cloud/base_spec.rb +515 -0
  139. data/spec/syncer/cloud/cloud_files_spec.rb +181 -0
  140. data/spec/syncer/cloud/s3_spec.rb +174 -0
  141. data/spec/syncer/rsync/base_spec.rb +98 -0
  142. data/spec/syncer/rsync/local_spec.rb +149 -0
  143. data/spec/syncer/rsync/pull_spec.rb +98 -0
  144. data/spec/syncer/rsync/push_spec.rb +333 -0
  145. data/spec/version_spec.rb +21 -0
  146. data/templates/cli/utility/archive +25 -0
  147. data/templates/cli/utility/compressor/bzip2 +4 -0
  148. data/templates/cli/utility/compressor/custom +11 -0
  149. data/templates/cli/utility/compressor/gzip +4 -0
  150. data/templates/cli/utility/compressor/lzma +10 -0
  151. data/templates/cli/utility/compressor/pbzip2 +10 -0
  152. data/templates/cli/utility/config +32 -0
  153. data/templates/cli/utility/database/mongodb +18 -0
  154. data/templates/cli/utility/database/mysql +21 -0
  155. data/templates/cli/utility/database/postgresql +17 -0
  156. data/templates/cli/utility/database/redis +16 -0
  157. data/templates/cli/utility/database/riak +11 -0
  158. data/templates/cli/utility/encryptor/gpg +27 -0
  159. data/templates/cli/utility/encryptor/openssl +9 -0
  160. data/templates/cli/utility/model.erb +23 -0
  161. data/templates/cli/utility/notifier/campfire +12 -0
  162. data/templates/cli/utility/notifier/hipchat +15 -0
  163. data/templates/cli/utility/notifier/mail +22 -0
  164. data/templates/cli/utility/notifier/prowl +11 -0
  165. data/templates/cli/utility/notifier/pushover +11 -0
  166. data/templates/cli/utility/notifier/twitter +13 -0
  167. data/templates/cli/utility/splitter +7 -0
  168. data/templates/cli/utility/storage/cloud_files +22 -0
  169. data/templates/cli/utility/storage/dropbox +20 -0
  170. data/templates/cli/utility/storage/ftp +12 -0
  171. data/templates/cli/utility/storage/local +7 -0
  172. data/templates/cli/utility/storage/ninefold +9 -0
  173. data/templates/cli/utility/storage/rsync +11 -0
  174. data/templates/cli/utility/storage/s3 +19 -0
  175. data/templates/cli/utility/storage/scp +11 -0
  176. data/templates/cli/utility/storage/sftp +11 -0
  177. data/templates/cli/utility/syncer/cloud_files +46 -0
  178. data/templates/cli/utility/syncer/rsync_local +12 -0
  179. data/templates/cli/utility/syncer/rsync_pull +17 -0
  180. data/templates/cli/utility/syncer/rsync_push +17 -0
  181. data/templates/cli/utility/syncer/s3 +43 -0
  182. data/templates/general/links +11 -0
  183. data/templates/general/version.erb +2 -0
  184. data/templates/notifier/mail/failure.erb +9 -0
  185. data/templates/notifier/mail/success.erb +7 -0
  186. data/templates/notifier/mail/warning.erb +9 -0
  187. data/templates/storage/dropbox/authorization_url.erb +6 -0
  188. data/templates/storage/dropbox/authorized.erb +4 -0
  189. data/templates/storage/dropbox/cache_file_written.erb +10 -0
  190. metadata +277 -0
@@ -0,0 +1,244 @@
1
+ # encoding: utf-8
2
+
3
+ require File.expand_path('../../spec_helper.rb', __FILE__)
4
+
5
+ describe Backup::Storage::SFTP do
6
+ let(:model) { Backup::Model.new(:test_trigger, 'test label') }
7
+ let(:storage) do
8
+ Backup::Storage::SFTP.new(model) do |sftp|
9
+ sftp.username = 'my_username'
10
+ sftp.password = 'my_password'
11
+ sftp.ip = '123.45.678.90'
12
+ sftp.keep = 5
13
+ end
14
+ end
15
+
16
+ it 'should be a subclass of Storage::Base' do
17
+ Backup::Storage::SFTP.
18
+ superclass.should == Backup::Storage::Base
19
+ end
20
+
21
+ describe '#initialize' do
22
+ after { Backup::Storage::SFTP.clear_defaults! }
23
+
24
+ it 'should load pre-configured defaults through Base' do
25
+ Backup::Storage::SFTP.any_instance.expects(:load_defaults!)
26
+ storage
27
+ end
28
+
29
+ it 'should pass the model reference to Base' do
30
+ storage.instance_variable_get(:@model).should == model
31
+ end
32
+
33
+ it 'should pass the storage_id to Base' do
34
+ storage = Backup::Storage::SFTP.new(model, 'my_storage_id')
35
+ storage.storage_id.should == 'my_storage_id'
36
+ end
37
+
38
+ it 'should remove any preceeding tilde and slash from the path' do
39
+ storage = Backup::Storage::SFTP.new(model) do |sftp|
40
+ sftp.path = '~/my_backups/path'
41
+ end
42
+ storage.path.should == 'my_backups/path'
43
+ end
44
+
45
+ context 'when no pre-configured defaults have been set' do
46
+ it 'should use the values given' do
47
+ storage.username.should == 'my_username'
48
+ storage.password.should == 'my_password'
49
+ storage.ip.should == '123.45.678.90'
50
+ storage.port.should == 22
51
+ storage.path.should == 'backups'
52
+
53
+ storage.storage_id.should be_nil
54
+ storage.keep.should == 5
55
+ end
56
+
57
+ it 'should use default values if none are given' do
58
+ storage = Backup::Storage::SFTP.new(model)
59
+
60
+ storage.username.should be_nil
61
+ storage.password.should be_nil
62
+ storage.ip.should be_nil
63
+ storage.port.should == 22
64
+ storage.path.should == 'backups'
65
+
66
+ storage.storage_id.should be_nil
67
+ storage.keep.should be_nil
68
+ end
69
+ end # context 'when no pre-configured defaults have been set'
70
+
71
+ context 'when pre-configured defaults have been set' do
72
+ before do
73
+ Backup::Storage::SFTP.defaults do |s|
74
+ s.username = 'some_username'
75
+ s.password = 'some_password'
76
+ s.ip = 'some_ip'
77
+ s.port = 'some_port'
78
+ s.path = 'some_path'
79
+ s.keep = 'some_keep'
80
+ end
81
+ end
82
+
83
+ it 'should use pre-configured defaults' do
84
+ storage = Backup::Storage::SFTP.new(model)
85
+
86
+ storage.username.should == 'some_username'
87
+ storage.password.should == 'some_password'
88
+ storage.ip.should == 'some_ip'
89
+ storage.port.should == 'some_port'
90
+ storage.path.should == 'some_path'
91
+
92
+ storage.storage_id.should be_nil
93
+ storage.keep.should == 'some_keep'
94
+ end
95
+
96
+ it 'should override pre-configured defaults' do
97
+ storage = Backup::Storage::SFTP.new(model) do |s|
98
+ s.username = 'new_username'
99
+ s.password = 'new_password'
100
+ s.ip = 'new_ip'
101
+ s.port = 'new_port'
102
+ s.path = 'new_path'
103
+ s.keep = 'new_keep'
104
+ end
105
+
106
+ storage.username.should == 'new_username'
107
+ storage.password.should == 'new_password'
108
+ storage.ip.should == 'new_ip'
109
+ storage.port.should == 'new_port'
110
+ storage.path.should == 'new_path'
111
+
112
+ storage.storage_id.should be_nil
113
+ storage.keep.should == 'new_keep'
114
+ end
115
+ end # context 'when pre-configured defaults have been set'
116
+ end # describe '#initialize'
117
+
118
+ describe '#connection' do
119
+ let(:connection) { mock }
120
+
121
+ it 'should yield a connection to the remote server' do
122
+ Net::SFTP.expects(:start).with(
123
+ '123.45.678.90', 'my_username', :password => 'my_password', :port => 22
124
+ ).yields(connection)
125
+
126
+ storage.send(:connection) do |sftp|
127
+ sftp.should be(connection)
128
+ end
129
+ end
130
+ end
131
+
132
+ describe '#transfer!' do
133
+ let(:connection) { mock }
134
+ let(:package) { mock }
135
+ let(:s) { sequence '' }
136
+
137
+ before do
138
+ storage.instance_variable_set(:@package, package)
139
+ storage.stubs(:storage_name).returns('Storage::SFTP')
140
+ storage.stubs(:local_path).returns('/local/path')
141
+ storage.stubs(:connection).yields(connection)
142
+ end
143
+
144
+ it 'should transfer the package files' do
145
+ storage.expects(:remote_path_for).in_sequence(s).with(package).
146
+ returns('remote/path')
147
+ storage.expects(:create_remote_path).in_sequence(s).with(
148
+ 'remote/path', connection
149
+ )
150
+
151
+ storage.expects(:files_to_transfer_for).in_sequence(s).with(package).
152
+ multiple_yields(
153
+ ['2011.12.31.11.00.02.backup.tar.enc-aa', 'backup.tar.enc-aa'],
154
+ ['2011.12.31.11.00.02.backup.tar.enc-ab', 'backup.tar.enc-ab']
155
+ )
156
+ # first yield
157
+ Backup::Logger.expects(:message).in_sequence(s).with(
158
+ "Storage::SFTP started transferring " +
159
+ "'2011.12.31.11.00.02.backup.tar.enc-aa' to '123.45.678.90'."
160
+ )
161
+ connection.expects(:upload!).in_sequence(s).with(
162
+ File.join('/local/path', '2011.12.31.11.00.02.backup.tar.enc-aa'),
163
+ File.join('remote/path', 'backup.tar.enc-aa')
164
+ )
165
+ # second yield
166
+ Backup::Logger.expects(:message).in_sequence(s).with(
167
+ "Storage::SFTP started transferring " +
168
+ "'2011.12.31.11.00.02.backup.tar.enc-ab' to '123.45.678.90'."
169
+ )
170
+ connection.expects(:upload!).in_sequence(s).with(
171
+ File.join('/local/path', '2011.12.31.11.00.02.backup.tar.enc-ab'),
172
+ File.join('remote/path', 'backup.tar.enc-ab')
173
+ )
174
+
175
+ storage.send(:transfer!)
176
+ end
177
+ end # describe '#transfer!'
178
+
179
+ describe '#remove!' do
180
+ let(:package) { mock }
181
+ let(:connection) { mock }
182
+ let(:s) { sequence '' }
183
+
184
+ before do
185
+ storage.stubs(:storage_name).returns('Storage::SFTP')
186
+ storage.stubs(:connection).yields(connection)
187
+ end
188
+
189
+ it 'should remove the package files' do
190
+ storage.expects(:remote_path_for).in_sequence(s).with(package).
191
+ returns('remote/path')
192
+
193
+ storage.expects(:transferred_files_for).in_sequence(s).with(package).
194
+ multiple_yields(
195
+ ['2011.12.31.11.00.02.backup.tar.enc-aa', 'backup.tar.enc-aa'],
196
+ ['2011.12.31.11.00.02.backup.tar.enc-ab', 'backup.tar.enc-ab']
197
+ )
198
+ # first yield
199
+ Backup::Logger.expects(:message).in_sequence(s).with(
200
+ "Storage::SFTP started removing " +
201
+ "'2011.12.31.11.00.02.backup.tar.enc-aa' from '123.45.678.90'."
202
+ )
203
+ connection.expects(:remove!).in_sequence(s).with(
204
+ File.join('remote/path', 'backup.tar.enc-aa')
205
+ )
206
+ # second yield
207
+ Backup::Logger.expects(:message).in_sequence(s).with(
208
+ "Storage::SFTP started removing " +
209
+ "'2011.12.31.11.00.02.backup.tar.enc-ab' from '123.45.678.90'."
210
+ )
211
+ connection.expects(:remove!).in_sequence(s).with(
212
+ File.join('remote/path', 'backup.tar.enc-ab')
213
+ )
214
+
215
+ connection.expects(:rmdir!).with('remote/path').in_sequence(s)
216
+
217
+ storage.send(:remove!, package)
218
+ end
219
+ end # describe '#remove!'
220
+
221
+ describe '#create_remote_path' do
222
+ let(:connection) { mock }
223
+ let(:remote_path) { 'backups/folder/another_folder' }
224
+ let(:s) { sequence '' }
225
+ let(:sftp_response) { stub(:code => 11, :message => nil) }
226
+ let(:sftp_status_exception) { Net::SFTP::StatusException.new(sftp_response) }
227
+
228
+ context 'while properly creating remote directories one by one' do
229
+ it 'should rescue any SFTP::StatusException and continue' do
230
+ connection.expects(:mkdir!).in_sequence(s).
231
+ with("backups").raises(sftp_status_exception)
232
+ connection.expects(:mkdir!).in_sequence(s).
233
+ with("backups/folder")
234
+ connection.expects(:mkdir!).in_sequence(s).
235
+ with("backups/folder/another_folder")
236
+
237
+ expect do
238
+ storage.send(:create_remote_path, remote_path, connection)
239
+ end.not_to raise_error
240
+ end
241
+ end
242
+ end
243
+
244
+ end
@@ -0,0 +1,109 @@
1
+ # encoding: utf-8
2
+
3
+ require File.expand_path('../../spec_helper.rb', __FILE__)
4
+
5
+ describe Backup::Syncer::Base do
6
+ let(:syncer) { Backup::Syncer::Base.new }
7
+
8
+ it 'should include CLI::Helpers' do
9
+ Backup::Syncer::Base.
10
+ include?(Backup::CLI::Helpers).should be_true
11
+ end
12
+
13
+ it 'should include Configuration::Helpers' do
14
+ Backup::Syncer::Base.
15
+ include?(Backup::Configuration::Helpers).should be_true
16
+ end
17
+
18
+ describe '#initialize' do
19
+ after { Backup::Syncer::Base.clear_defaults! }
20
+
21
+ it 'should load pre-configured defaults through Base' do
22
+ Backup::Syncer::Base.any_instance.expects(:load_defaults!)
23
+ syncer
24
+ end
25
+
26
+ it 'should establish a new array for @directories' do
27
+ syncer.directories.should == []
28
+ end
29
+
30
+ context 'when no pre-configured defaults have been set' do
31
+ it 'should set default values' do
32
+ syncer.path.should == 'backups'
33
+ syncer.mirror.should == false
34
+ end
35
+ end # context 'when no pre-configured defaults have been set'
36
+
37
+ context 'when pre-configured defaults have been set' do
38
+ before do
39
+ Backup::Syncer::Base.defaults do |s|
40
+ s.path = 'some_path'
41
+ s.mirror = 'some_mirror'
42
+ end
43
+ end
44
+
45
+ it 'should use pre-configured defaults' do
46
+ syncer.path.should == 'some_path'
47
+ syncer.mirror.should == 'some_mirror'
48
+ end
49
+ end # context 'when pre-configured defaults have been set'
50
+ end # describe '#initialize'
51
+
52
+ describe '#directories' do
53
+ before do
54
+ syncer.instance_variable_set(
55
+ :@directories, ['/some/directory', '/another/directory']
56
+ )
57
+ end
58
+
59
+ context 'when no block is given' do
60
+ it 'should return @directories' do
61
+ syncer.directories.should ==
62
+ ['/some/directory', '/another/directory']
63
+ end
64
+ end
65
+
66
+ context 'when a block is given' do
67
+ it 'should evalute the block, allowing #add to add directories' do
68
+ syncer.directories do
69
+ add '/new/path'
70
+ add '/another/new/path'
71
+ end
72
+ syncer.directories.should == [
73
+ '/some/directory',
74
+ '/another/directory',
75
+ '/new/path',
76
+ '/another/new/path'
77
+ ]
78
+ end
79
+ end
80
+ end # describe '#directories'
81
+
82
+ describe '#add' do
83
+ before do
84
+ syncer.instance_variable_set(
85
+ :@directories, ['/some/directory', '/another/directory']
86
+ )
87
+ end
88
+
89
+ it 'should add the given path to @directories' do
90
+ syncer.add '/my/path'
91
+ syncer.directories.should ==
92
+ ['/some/directory', '/another/directory', '/my/path']
93
+ end
94
+
95
+ # Note: Each Syncer should handle this as needed.
96
+ # For example, expanding these here would break RSync::Pull
97
+ it 'should not expand the given paths' do
98
+ syncer.add 'relative/path'
99
+ syncer.directories.should ==
100
+ ['/some/directory', '/another/directory', 'relative/path']
101
+ end
102
+ end
103
+
104
+ describe '#syncer_name' do
105
+ it 'should return the class name with the Backup:: namespace removed' do
106
+ syncer.send(:syncer_name).should == 'Syncer::Base'
107
+ end
108
+ end
109
+ end
@@ -0,0 +1,515 @@
1
+ # encoding: utf-8
2
+ require File.expand_path('../../../spec_helper.rb', __FILE__)
3
+
4
+ describe 'Backup::Syncer::Cloud::Base' do
5
+ let(:syncer) { Backup::Syncer::Cloud::Base.new }
6
+ let(:s) { sequence '' }
7
+
8
+ it 'should be a subclass of Syncer::Base' do
9
+ Backup::Syncer::Cloud::Base.
10
+ superclass.should == Backup::Syncer::Base
11
+ end
12
+
13
+ it 'should establish a class constant for a Mutex' do
14
+ Backup::Syncer::Cloud::Base::MUTEX.should be_an_instance_of Mutex
15
+ end
16
+
17
+ describe '#initialize' do
18
+ after { Backup::Syncer::Cloud::Base.clear_defaults! }
19
+
20
+ it 'should load pre-configured defaults through Syncer::Base' do
21
+ Backup::Syncer::Cloud::Base.any_instance.expects(:load_defaults!)
22
+ syncer
23
+ end
24
+
25
+ context 'when no pre-configured defaults have been set' do
26
+ it 'should use default values if none are given' do
27
+ syncer.path.should == 'backups'
28
+ syncer.mirror.should == false
29
+ syncer.concurrency_type.should == false
30
+ syncer.concurrency_level.should == 2
31
+ end
32
+ end # context 'when no pre-configured defaults have been set'
33
+
34
+ context 'when pre-configured defaults have been set' do
35
+ before do
36
+ Backup::Syncer::Cloud::Base.defaults do |cloud|
37
+ cloud.concurrency_type = 'default_concurrency_type'
38
+ cloud.concurrency_level = 'default_concurrency_level'
39
+ end
40
+ end
41
+
42
+ it 'should use pre-configured defaults' do
43
+ syncer.path.should == 'backups'
44
+ syncer.mirror.should == false
45
+ syncer.concurrency_type.should == 'default_concurrency_type'
46
+ syncer.concurrency_level.should == 'default_concurrency_level'
47
+ end
48
+ end # context 'when pre-configured defaults have been set'
49
+ end # describe '#initialize'
50
+
51
+ describe '#perform' do
52
+ let(:sync_context) { mock }
53
+
54
+ before do
55
+ syncer.stubs(:repository_object).returns(:a_repository_object)
56
+
57
+ Backup::Logger.expects(:message).with(
58
+ "Syncer::Cloud::Base started the syncing process:\n" +
59
+ "\s\sConcurrency: false Level: 2"
60
+ )
61
+ Backup::Logger.expects(:message).with(
62
+ 'Syncer::Cloud::Base Syncing Complete!'
63
+ )
64
+ end
65
+
66
+ it 'should sync each directory' do
67
+ syncer.directories do
68
+ add '/dir/one'
69
+ add '/dir/two'
70
+ end
71
+
72
+ Backup::Syncer::Cloud::Base::SyncContext.expects(:new).in_sequence(s).with(
73
+ '/dir/one', :a_repository_object, 'backups'
74
+ ).returns(sync_context)
75
+ sync_context.expects(:sync!).in_sequence(s).with(
76
+ false, false, 2
77
+ )
78
+ Backup::Syncer::Cloud::Base::SyncContext.expects(:new).in_sequence(s).with(
79
+ '/dir/two', :a_repository_object, 'backups'
80
+ ).returns(sync_context)
81
+ sync_context.expects(:sync!).in_sequence(s).with(
82
+ false, false, 2
83
+ )
84
+
85
+ syncer.perform!
86
+ end
87
+
88
+ it 'should ensure each directory path is expanded with no trailing slash' do
89
+ syncer.directories do
90
+ add '/dir/one/'
91
+ add 'dir/two'
92
+ end
93
+
94
+ Backup::Syncer::Cloud::Base::SyncContext.expects(:new).with(
95
+ '/dir/one', :a_repository_object, 'backups'
96
+ ).returns(sync_context)
97
+
98
+ Backup::Syncer::Cloud::Base::SyncContext.expects(:new).with(
99
+ File.expand_path('dir/two'), :a_repository_object, 'backups'
100
+ ).returns(sync_context)
101
+
102
+ sync_context.stubs(:sync!)
103
+
104
+ syncer.perform!
105
+ end
106
+ end # describe '#perform'
107
+
108
+ describe 'Cloud::Base::SyncContext' do
109
+ let(:bucket) { mock }
110
+ let(:sync_context) do
111
+ Backup::Syncer::Cloud::Base::SyncContext.new(
112
+ '/dir/to/sync', bucket, 'backups'
113
+ )
114
+ end
115
+
116
+ describe '#initialize' do
117
+ it 'should set variables' do
118
+ sync_context.directory.should == '/dir/to/sync'
119
+ sync_context.bucket.should == bucket
120
+ sync_context.path.should == 'backups'
121
+ sync_context.remote_base.should == 'backups/sync'
122
+ end
123
+ end
124
+
125
+ describe '#sync!' do
126
+ let(:all_files_array) { mock }
127
+
128
+ before do
129
+ sync_context.stubs(:all_file_names).returns(all_files_array)
130
+ end
131
+
132
+ context 'when concurrency_type is set to `false`' do
133
+ it 'syncs files without concurrency' do
134
+ all_files_array.expects(:each).in_sequence(s).
135
+ multiple_yields('foo.file', 'foo_dir/foo.file')
136
+
137
+ sync_context.expects(:sync_file).in_sequence(s).
138
+ with('foo.file', :mirror)
139
+ sync_context.expects(:sync_file).in_sequence(s).
140
+ with('foo_dir/foo.file', :mirror)
141
+
142
+ sync_context.sync!(:mirror, false, :foo)
143
+ end
144
+ end
145
+
146
+ context 'when concurrency_type is set to `:threads`' do
147
+ it 'uses `concurrency_level` number of threads for concurrency' do
148
+ Parallel.expects(:each).in_sequence(s).with(
149
+ all_files_array, :in_threads => :num_of_threads
150
+ ).multiple_yields('foo.file', 'foo_dir/foo.file')
151
+
152
+ sync_context.expects(:sync_file).in_sequence(s).
153
+ with('foo.file', :mirror)
154
+ sync_context.expects(:sync_file).in_sequence(s).
155
+ with('foo_dir/foo.file', :mirror)
156
+
157
+ sync_context.sync!(:mirror, :threads, :num_of_threads)
158
+ end
159
+ end
160
+
161
+ context 'when concurrency_type is set to `:processes`' do
162
+ it 'uses `concurrency_level` number of processes for concurrency' do
163
+ Parallel.expects(:each).in_sequence(s).with(
164
+ all_files_array, :in_processes => :num_of_processes
165
+ ).multiple_yields('foo.file', 'foo_dir/foo.file')
166
+
167
+ sync_context.expects(:sync_file).in_sequence(s).
168
+ with('foo.file', :mirror)
169
+ sync_context.expects(:sync_file).in_sequence(s).
170
+ with('foo_dir/foo.file', :mirror)
171
+
172
+ sync_context.sync!(:mirror, :processes, :num_of_processes)
173
+ end
174
+ end
175
+
176
+ context 'when concurrency_type setting is invalid' do
177
+ it 'should raise an error' do
178
+ expect do
179
+ sync_context.sync!(:foo, 'unknown type', :foo)
180
+ end.to raise_error(
181
+ Backup::Errors::Syncer::Cloud::ConfigurationError,
182
+ 'Syncer::Cloud::ConfigurationError: ' +
183
+ "Unknown concurrency_type setting: \"unknown type\""
184
+ )
185
+ end
186
+ end
187
+ end # describe '#sync!'
188
+
189
+ describe '#all_file_names' do
190
+ let(:local_files_hash) do
191
+ { 'file_b' => :foo, 'file_a' => :foo, 'dir_a/file_b' => :foo }
192
+ end
193
+ let(:remote_files_hash) do
194
+ { 'file_c' => :foo, 'file_a' => :foo, 'dir_a/file_a' => :foo }
195
+ end
196
+ let(:local_remote_union_array) do
197
+ ['dir_a/file_a', 'dir_a/file_b', 'file_a', 'file_b', 'file_c']
198
+ end
199
+
200
+ it 'returns and caches a sorted union of local and remote file names' do
201
+ sync_context.expects(:local_files).once.returns(local_files_hash)
202
+ sync_context.expects(:remote_files).once.returns(remote_files_hash)
203
+
204
+ sync_context.send(:all_file_names).should == local_remote_union_array
205
+ sync_context.instance_variable_get(:@all_file_names).
206
+ should == local_remote_union_array
207
+ sync_context.send(:all_file_names).should == local_remote_union_array
208
+ end
209
+ end # describe '#all_file_names'
210
+
211
+ describe '#local_files' do
212
+ let(:local_file_class) { Backup::Syncer::Cloud::Base::LocalFile }
213
+ let(:local_hashes_data) { "line1\nline2\nbad\xFFline\nline3" }
214
+
215
+ let(:local_file_a) { stub(:relative_path => 'file_a') }
216
+ let(:local_file_b) { stub(:relative_path => 'file_b') }
217
+ let(:local_file_c) { stub(:relative_path => 'file_c') }
218
+ let(:local_files_hash) do
219
+ { 'file_a' => local_file_a,
220
+ 'file_b' => local_file_b,
221
+ 'file_c' => local_file_c }
222
+ end
223
+
224
+ it 'should return and caches a hash of LocalFile objects' do
225
+ sync_context.expects(:local_hashes).once.returns(local_hashes_data)
226
+
227
+ local_file_class.expects(:new).once.with('/dir/to/sync', "line1\n").
228
+ returns(local_file_a)
229
+ local_file_class.expects(:new).once.with('/dir/to/sync', "line2\n").
230
+ returns(local_file_b)
231
+ local_file_class.expects(:new).once.with('/dir/to/sync', "bad\xFFline\n").
232
+ returns(nil)
233
+ local_file_class.expects(:new).once.with('/dir/to/sync', "line3").
234
+ returns(local_file_c)
235
+
236
+ sync_context.send(:local_files).should == local_files_hash
237
+ sync_context.instance_variable_get(:@local_files).
238
+ should == local_files_hash
239
+ sync_context.send(:local_files).should == local_files_hash
240
+ end
241
+
242
+ # Note: don't use methods that validate encoding
243
+ it 'will raise an Exception if String#split is used',
244
+ :if => RUBY_VERSION >= '1.9' do
245
+ expect do
246
+ "line1\nbad\xFFline\nline3".split("\n")
247
+ end.to raise_error(ArgumentError, 'invalid byte sequence in UTF-8')
248
+ end
249
+ end # describe '#local_files'
250
+
251
+ describe '#local_hashes' do
252
+ it 'should collect file paths and MD5 checksums for @directory' do
253
+ Backup::Logger.expects(:message).with(
254
+ "\s\sGenerating checksums for '/dir/to/sync'"
255
+ )
256
+ sync_context.expects(:`).with(
257
+ "find '/dir/to/sync' -print0 | xargs -0 openssl md5 2> /dev/null"
258
+ ).returns('MD5(tmp/foo)= 0123456789abcdefghijklmnopqrstuv')
259
+
260
+ sync_context.send(:local_hashes).should ==
261
+ 'MD5(tmp/foo)= 0123456789abcdefghijklmnopqrstuv'
262
+ end
263
+ end
264
+
265
+ describe '#remote_files' do
266
+ let(:repository_object) { mock }
267
+ let(:repository_files) { mock }
268
+ let(:file_objects) { mock }
269
+ let(:file_obj_a) { stub(:key => 'file_a') }
270
+ let(:file_obj_b) { stub(:key => 'file_b') }
271
+ let(:file_obj_c) { stub(:key => 'dir/file_c') }
272
+ let(:remote_files_hash) do
273
+ { 'file_a' => file_obj_a,
274
+ 'file_b' => file_obj_b,
275
+ 'dir/file_c' => file_obj_c }
276
+ end
277
+
278
+ before do
279
+ sync_context.instance_variable_set(:@bucket, repository_object)
280
+
281
+ repository_object.expects(:files).once.returns(repository_files)
282
+ repository_files.expects(:all).once.with(:prefix => 'backups/sync').
283
+ returns(file_objects)
284
+ file_objects.expects(:each).once.multiple_yields(
285
+ file_obj_a, file_obj_b, file_obj_c
286
+ )
287
+
288
+ # this is to avoid: unexpected invocation: #<Mock>.to_a()
289
+ # only 1.9.2 seems affected by this
290
+ if RUBY_VERSION == '1.9.2'
291
+ file_obj_a.stubs(:to_a)
292
+ file_obj_b.stubs(:to_a)
293
+ file_obj_c.stubs(:to_a)
294
+ end
295
+ end
296
+
297
+ context 'when it returns and caches a hash of repository file objects' do
298
+ it 'should remove the @remote_base from the path for the hash key' do
299
+ sync_context.send(:remote_files).should == remote_files_hash
300
+ sync_context.instance_variable_get(:@remote_files).
301
+ should == remote_files_hash
302
+ sync_context.send(:remote_files).should == remote_files_hash
303
+ end
304
+ end
305
+ end # describe '#remote_files'
306
+
307
+ describe '#sync_file' do
308
+ let(:local_file) do
309
+ stub(
310
+ :path => '/dir/to/sync/sync.file',
311
+ :md5 => '0123456789abcdefghijklmnopqrstuv')
312
+ end
313
+ let(:remote_file) do
314
+ stub(:path => 'backups/sync/sync.file')
315
+ end
316
+ let(:file) { mock }
317
+ let(:repository_object) { mock }
318
+ let(:repository_files) { mock }
319
+
320
+ before do
321
+ sync_context.instance_variable_set(:@bucket, repository_object)
322
+ repository_object.stubs(:files).returns(repository_files)
323
+ end
324
+
325
+ context 'when the requested file to sync exists locally' do
326
+ before do
327
+ sync_context.stubs(:local_files).returns(
328
+ { 'sync.file' => local_file }
329
+ )
330
+ File.expects(:exist?).with('/dir/to/sync/sync.file').returns(true)
331
+ end
332
+
333
+ context 'when the MD5 checksum matches the remote file' do
334
+ before do
335
+ remote_file.stubs(:etag).returns('0123456789abcdefghijklmnopqrstuv')
336
+ sync_context.stubs(:remote_files).returns(
337
+ { 'sync.file' => remote_file }
338
+ )
339
+ end
340
+
341
+ it 'should skip the file' do
342
+ File.expects(:open).never
343
+ Backup::Syncer::Cloud::Base::MUTEX.expects(:synchronize).yields
344
+ Backup::Logger.expects(:message).with(
345
+ "\s\s[skipping] 'backups/sync/sync.file'"
346
+ )
347
+
348
+ sync_context.send(:sync_file, 'sync.file', :foo)
349
+ end
350
+ end
351
+
352
+ context 'when the MD5 checksum does not match the remote file' do
353
+ before do
354
+ remote_file.stubs(:etag).returns('vutsrqponmlkjihgfedcba9876543210')
355
+ sync_context.stubs(:remote_files).returns(
356
+ { 'sync.file' => remote_file }
357
+ )
358
+ end
359
+
360
+ it 'should upload the file' do
361
+ Backup::Syncer::Cloud::Base::MUTEX.expects(:synchronize).yields
362
+ Backup::Logger.expects(:message).with(
363
+ "\s\s[transferring] 'backups/sync/sync.file'"
364
+ )
365
+
366
+ File.expects(:open).with('/dir/to/sync/sync.file', 'r').yields(file)
367
+ repository_files.expects(:create).with(
368
+ :key => 'backups/sync/sync.file',
369
+ :body => file
370
+ )
371
+
372
+ sync_context.send(:sync_file, 'sync.file', :foo)
373
+ end
374
+ end
375
+
376
+ context 'when the requested file does not exist on the remote' do
377
+ before do
378
+ sync_context.stubs(:remote_files).returns({})
379
+ end
380
+
381
+ it 'should upload the file' do
382
+ Backup::Syncer::Cloud::Base::MUTEX.expects(:synchronize).yields
383
+ Backup::Logger.expects(:message).with(
384
+ "\s\s[transferring] 'backups/sync/sync.file'"
385
+ )
386
+
387
+ File.expects(:open).with('/dir/to/sync/sync.file', 'r').yields(file)
388
+ repository_files.expects(:create).with(
389
+ :key => 'backups/sync/sync.file',
390
+ :body => file
391
+ )
392
+
393
+ sync_context.send(:sync_file, 'sync.file', :foo)
394
+ end
395
+ end
396
+ end
397
+
398
+ context 'when the requested file does not exist locally' do
399
+ before do
400
+ sync_context.stubs(:remote_files).returns(
401
+ { 'sync.file' => remote_file }
402
+ )
403
+ sync_context.stubs(:local_files).returns({})
404
+ end
405
+
406
+ context 'when the `mirror` option is set to true' do
407
+ it 'should remove the file from the remote' do
408
+ Backup::Syncer::Cloud::Base::MUTEX.expects(:synchronize).yields
409
+ Backup::Logger.expects(:message).with(
410
+ "\s\s[removing] 'backups/sync/sync.file'"
411
+ )
412
+
413
+ remote_file.expects(:destroy)
414
+
415
+ sync_context.send(:sync_file, 'sync.file', true)
416
+ end
417
+ end
418
+
419
+ context 'when the `mirror` option is set to false' do
420
+ it 'should leave the file on the remote' do
421
+ Backup::Syncer::Cloud::Base::MUTEX.expects(:synchronize).yields
422
+ Backup::Logger.expects(:message).with(
423
+ "\s\s[leaving] 'backups/sync/sync.file'"
424
+ )
425
+
426
+ remote_file.expects(:destroy).never
427
+
428
+ sync_context.send(:sync_file, 'sync.file', false)
429
+ end
430
+ end
431
+ end
432
+ end # describe '#sync_file'
433
+ end # describe 'Cloud::Base::SyncContext'
434
+
435
+ describe 'Cloud::Base::LocalFile' do
436
+ let(:local_file_class) { Backup::Syncer::Cloud::Base::LocalFile }
437
+
438
+ describe '#new' do
439
+ describe 'wrapping #initialize and using #sanitize to validate objects' do
440
+ context 'when the path is valid UTF-8' do
441
+ let(:local_file) do
442
+ local_file_class.new(
443
+ 'foo',
444
+ 'MD5(foo)= 0123456789abcdefghijklmnopqrstuv'
445
+ )
446
+ end
447
+
448
+ it 'should return the new object' do
449
+ Backup::Logger.expects(:warn).never
450
+
451
+ local_file.should be_an_instance_of local_file_class
452
+ end
453
+ end
454
+
455
+ context 'when the path contains invalid UTF-8' do
456
+ let(:local_file) do
457
+ local_file_class.new(
458
+ "/bad/pa\xFFth",
459
+ "MD5(/bad/pa\xFFth/to/file)= 0123456789abcdefghijklmnopqrstuv"
460
+ )
461
+ end
462
+ it 'should return nil and log a warning' do
463
+ Backup::Logger.expects(:warn).with(
464
+ "\s\s[skipping] /bad/pa\xEF\xBF\xBDth/to/file\n" +
465
+ "\s\sPath Contains Invalid UTF-8 byte sequences"
466
+ )
467
+
468
+ local_file.should be_nil
469
+ end
470
+ end
471
+ end
472
+ end # describe '#new'
473
+
474
+ describe '#initialize' do
475
+ let(:local_file) do
476
+ local_file_class.new(:directory, :line)
477
+ end
478
+
479
+ before do
480
+ local_file_class.any_instance.expects(:sanitize).with(:directory).
481
+ returns('/dir/to/sync')
482
+ local_file_class.any_instance.expects(:sanitize).with(:line).
483
+ returns("MD5(/dir/to/sync/subdir/sync.file)= 0123456789abcdefghijklmnopqrstuv\n")
484
+ end
485
+
486
+ it 'should determine @path, @relative_path and @md5' do
487
+ local_file.path.should == '/dir/to/sync/subdir/sync.file'
488
+ local_file.relative_path.should == 'subdir/sync.file'
489
+ local_file.md5.should == '0123456789abcdefghijklmnopqrstuv'
490
+ end
491
+
492
+ it 'should return nil if the object is invalid' do
493
+ local_file_class.any_instance.expects(:invalid?).returns(true)
494
+ Backup::Logger.expects(:warn)
495
+ local_file.should be_nil
496
+ end
497
+ end # describe '#initialize'
498
+
499
+ describe '#sanitize' do
500
+ let(:local_file) do
501
+ local_file_class.new('foo', 'MD5(foo)= 0123456789abcdefghijklmnopqrstuv')
502
+ end
503
+
504
+ it 'should replace any invalid UTF-8 characters' do
505
+ local_file.send(:sanitize, "/path/to/d\xFFir/subdir/sync\xFFfile").
506
+ should == "/path/to/d\xEF\xBF\xBDir/subdir/sync\xEF\xBF\xBDfile"
507
+ end
508
+
509
+ it 'should flag the LocalFile object as invalid' do
510
+ local_file.send(:sanitize, "/path/to/d\xFFir/subdir/sync\xFFfile")
511
+ local_file.invalid?.should be_true
512
+ end
513
+ end # describe '#sanitize'
514
+ end # describe 'Cloud::Base::LocalFile'
515
+ end