backup-agoddard 3.0.27

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (190) hide show
  1. data/.gitignore +8 -0
  2. data/.travis.yml +10 -0
  3. data/Gemfile +28 -0
  4. data/Guardfile +23 -0
  5. data/LICENSE.md +24 -0
  6. data/README.md +478 -0
  7. data/backup.gemspec +32 -0
  8. data/bin/backup +11 -0
  9. data/lib/backup.rb +133 -0
  10. data/lib/backup/archive.rb +117 -0
  11. data/lib/backup/binder.rb +22 -0
  12. data/lib/backup/cleaner.rb +121 -0
  13. data/lib/backup/cli/helpers.rb +93 -0
  14. data/lib/backup/cli/utility.rb +255 -0
  15. data/lib/backup/compressor/base.rb +35 -0
  16. data/lib/backup/compressor/bzip2.rb +50 -0
  17. data/lib/backup/compressor/custom.rb +53 -0
  18. data/lib/backup/compressor/gzip.rb +50 -0
  19. data/lib/backup/compressor/lzma.rb +52 -0
  20. data/lib/backup/compressor/pbzip2.rb +59 -0
  21. data/lib/backup/config.rb +174 -0
  22. data/lib/backup/configuration.rb +33 -0
  23. data/lib/backup/configuration/helpers.rb +130 -0
  24. data/lib/backup/configuration/store.rb +24 -0
  25. data/lib/backup/database/base.rb +53 -0
  26. data/lib/backup/database/mongodb.rb +230 -0
  27. data/lib/backup/database/mysql.rb +160 -0
  28. data/lib/backup/database/postgresql.rb +144 -0
  29. data/lib/backup/database/redis.rb +136 -0
  30. data/lib/backup/database/riak.rb +67 -0
  31. data/lib/backup/dependency.rb +108 -0
  32. data/lib/backup/encryptor/base.rb +29 -0
  33. data/lib/backup/encryptor/gpg.rb +760 -0
  34. data/lib/backup/encryptor/open_ssl.rb +72 -0
  35. data/lib/backup/errors.rb +124 -0
  36. data/lib/backup/hooks.rb +68 -0
  37. data/lib/backup/logger.rb +152 -0
  38. data/lib/backup/model.rb +409 -0
  39. data/lib/backup/notifier/base.rb +81 -0
  40. data/lib/backup/notifier/campfire.rb +155 -0
  41. data/lib/backup/notifier/hipchat.rb +93 -0
  42. data/lib/backup/notifier/mail.rb +206 -0
  43. data/lib/backup/notifier/prowl.rb +65 -0
  44. data/lib/backup/notifier/pushover.rb +88 -0
  45. data/lib/backup/notifier/twitter.rb +70 -0
  46. data/lib/backup/package.rb +47 -0
  47. data/lib/backup/packager.rb +100 -0
  48. data/lib/backup/pipeline.rb +110 -0
  49. data/lib/backup/splitter.rb +75 -0
  50. data/lib/backup/storage/base.rb +99 -0
  51. data/lib/backup/storage/cloudfiles.rb +87 -0
  52. data/lib/backup/storage/cycler.rb +117 -0
  53. data/lib/backup/storage/dropbox.rb +178 -0
  54. data/lib/backup/storage/ftp.rb +119 -0
  55. data/lib/backup/storage/local.rb +82 -0
  56. data/lib/backup/storage/ninefold.rb +116 -0
  57. data/lib/backup/storage/rsync.rb +149 -0
  58. data/lib/backup/storage/s3.rb +94 -0
  59. data/lib/backup/storage/scp.rb +99 -0
  60. data/lib/backup/storage/sftp.rb +108 -0
  61. data/lib/backup/syncer/base.rb +46 -0
  62. data/lib/backup/syncer/cloud/base.rb +247 -0
  63. data/lib/backup/syncer/cloud/cloud_files.rb +78 -0
  64. data/lib/backup/syncer/cloud/s3.rb +68 -0
  65. data/lib/backup/syncer/rsync/base.rb +49 -0
  66. data/lib/backup/syncer/rsync/local.rb +55 -0
  67. data/lib/backup/syncer/rsync/pull.rb +36 -0
  68. data/lib/backup/syncer/rsync/push.rb +116 -0
  69. data/lib/backup/template.rb +46 -0
  70. data/lib/backup/version.rb +43 -0
  71. data/spec-live/.gitignore +6 -0
  72. data/spec-live/README +7 -0
  73. data/spec-live/backups/config.rb +83 -0
  74. data/spec-live/backups/config.yml.template +46 -0
  75. data/spec-live/backups/models.rb +184 -0
  76. data/spec-live/compressor/custom_spec.rb +30 -0
  77. data/spec-live/compressor/gzip_spec.rb +30 -0
  78. data/spec-live/encryptor/gpg_keys.rb +239 -0
  79. data/spec-live/encryptor/gpg_spec.rb +287 -0
  80. data/spec-live/notifier/mail_spec.rb +121 -0
  81. data/spec-live/spec_helper.rb +151 -0
  82. data/spec-live/storage/dropbox_spec.rb +151 -0
  83. data/spec-live/storage/local_spec.rb +83 -0
  84. data/spec-live/storage/scp_spec.rb +193 -0
  85. data/spec-live/syncer/cloud/s3_spec.rb +124 -0
  86. data/spec/archive_spec.rb +335 -0
  87. data/spec/cleaner_spec.rb +312 -0
  88. data/spec/cli/helpers_spec.rb +301 -0
  89. data/spec/cli/utility_spec.rb +411 -0
  90. data/spec/compressor/base_spec.rb +52 -0
  91. data/spec/compressor/bzip2_spec.rb +217 -0
  92. data/spec/compressor/custom_spec.rb +106 -0
  93. data/spec/compressor/gzip_spec.rb +217 -0
  94. data/spec/compressor/lzma_spec.rb +123 -0
  95. data/spec/compressor/pbzip2_spec.rb +165 -0
  96. data/spec/config_spec.rb +321 -0
  97. data/spec/configuration/helpers_spec.rb +247 -0
  98. data/spec/configuration/store_spec.rb +39 -0
  99. data/spec/configuration_spec.rb +62 -0
  100. data/spec/database/base_spec.rb +63 -0
  101. data/spec/database/mongodb_spec.rb +510 -0
  102. data/spec/database/mysql_spec.rb +411 -0
  103. data/spec/database/postgresql_spec.rb +353 -0
  104. data/spec/database/redis_spec.rb +334 -0
  105. data/spec/database/riak_spec.rb +176 -0
  106. data/spec/dependency_spec.rb +51 -0
  107. data/spec/encryptor/base_spec.rb +40 -0
  108. data/spec/encryptor/gpg_spec.rb +909 -0
  109. data/spec/encryptor/open_ssl_spec.rb +148 -0
  110. data/spec/errors_spec.rb +306 -0
  111. data/spec/hooks_spec.rb +35 -0
  112. data/spec/logger_spec.rb +367 -0
  113. data/spec/model_spec.rb +694 -0
  114. data/spec/notifier/base_spec.rb +104 -0
  115. data/spec/notifier/campfire_spec.rb +217 -0
  116. data/spec/notifier/hipchat_spec.rb +211 -0
  117. data/spec/notifier/mail_spec.rb +316 -0
  118. data/spec/notifier/prowl_spec.rb +138 -0
  119. data/spec/notifier/pushover_spec.rb +123 -0
  120. data/spec/notifier/twitter_spec.rb +153 -0
  121. data/spec/package_spec.rb +61 -0
  122. data/spec/packager_spec.rb +213 -0
  123. data/spec/pipeline_spec.rb +259 -0
  124. data/spec/spec_helper.rb +60 -0
  125. data/spec/splitter_spec.rb +120 -0
  126. data/spec/storage/base_spec.rb +166 -0
  127. data/spec/storage/cloudfiles_spec.rb +254 -0
  128. data/spec/storage/cycler_spec.rb +247 -0
  129. data/spec/storage/dropbox_spec.rb +480 -0
  130. data/spec/storage/ftp_spec.rb +271 -0
  131. data/spec/storage/local_spec.rb +259 -0
  132. data/spec/storage/ninefold_spec.rb +343 -0
  133. data/spec/storage/rsync_spec.rb +362 -0
  134. data/spec/storage/s3_spec.rb +245 -0
  135. data/spec/storage/scp_spec.rb +233 -0
  136. data/spec/storage/sftp_spec.rb +244 -0
  137. data/spec/syncer/base_spec.rb +109 -0
  138. data/spec/syncer/cloud/base_spec.rb +515 -0
  139. data/spec/syncer/cloud/cloud_files_spec.rb +181 -0
  140. data/spec/syncer/cloud/s3_spec.rb +174 -0
  141. data/spec/syncer/rsync/base_spec.rb +98 -0
  142. data/spec/syncer/rsync/local_spec.rb +149 -0
  143. data/spec/syncer/rsync/pull_spec.rb +98 -0
  144. data/spec/syncer/rsync/push_spec.rb +333 -0
  145. data/spec/version_spec.rb +21 -0
  146. data/templates/cli/utility/archive +25 -0
  147. data/templates/cli/utility/compressor/bzip2 +4 -0
  148. data/templates/cli/utility/compressor/custom +11 -0
  149. data/templates/cli/utility/compressor/gzip +4 -0
  150. data/templates/cli/utility/compressor/lzma +10 -0
  151. data/templates/cli/utility/compressor/pbzip2 +10 -0
  152. data/templates/cli/utility/config +32 -0
  153. data/templates/cli/utility/database/mongodb +18 -0
  154. data/templates/cli/utility/database/mysql +21 -0
  155. data/templates/cli/utility/database/postgresql +17 -0
  156. data/templates/cli/utility/database/redis +16 -0
  157. data/templates/cli/utility/database/riak +11 -0
  158. data/templates/cli/utility/encryptor/gpg +27 -0
  159. data/templates/cli/utility/encryptor/openssl +9 -0
  160. data/templates/cli/utility/model.erb +23 -0
  161. data/templates/cli/utility/notifier/campfire +12 -0
  162. data/templates/cli/utility/notifier/hipchat +15 -0
  163. data/templates/cli/utility/notifier/mail +22 -0
  164. data/templates/cli/utility/notifier/prowl +11 -0
  165. data/templates/cli/utility/notifier/pushover +11 -0
  166. data/templates/cli/utility/notifier/twitter +13 -0
  167. data/templates/cli/utility/splitter +7 -0
  168. data/templates/cli/utility/storage/cloud_files +22 -0
  169. data/templates/cli/utility/storage/dropbox +20 -0
  170. data/templates/cli/utility/storage/ftp +12 -0
  171. data/templates/cli/utility/storage/local +7 -0
  172. data/templates/cli/utility/storage/ninefold +9 -0
  173. data/templates/cli/utility/storage/rsync +11 -0
  174. data/templates/cli/utility/storage/s3 +19 -0
  175. data/templates/cli/utility/storage/scp +11 -0
  176. data/templates/cli/utility/storage/sftp +11 -0
  177. data/templates/cli/utility/syncer/cloud_files +46 -0
  178. data/templates/cli/utility/syncer/rsync_local +12 -0
  179. data/templates/cli/utility/syncer/rsync_pull +17 -0
  180. data/templates/cli/utility/syncer/rsync_push +17 -0
  181. data/templates/cli/utility/syncer/s3 +43 -0
  182. data/templates/general/links +11 -0
  183. data/templates/general/version.erb +2 -0
  184. data/templates/notifier/mail/failure.erb +9 -0
  185. data/templates/notifier/mail/success.erb +7 -0
  186. data/templates/notifier/mail/warning.erb +9 -0
  187. data/templates/storage/dropbox/authorization_url.erb +6 -0
  188. data/templates/storage/dropbox/authorized.erb +4 -0
  189. data/templates/storage/dropbox/cache_file_written.erb +10 -0
  190. metadata +277 -0
@@ -0,0 +1,244 @@
1
+ # encoding: utf-8
2
+
3
+ require File.expand_path('../../spec_helper.rb', __FILE__)
4
+
5
+ describe Backup::Storage::SFTP do
6
+ let(:model) { Backup::Model.new(:test_trigger, 'test label') }
7
+ let(:storage) do
8
+ Backup::Storage::SFTP.new(model) do |sftp|
9
+ sftp.username = 'my_username'
10
+ sftp.password = 'my_password'
11
+ sftp.ip = '123.45.678.90'
12
+ sftp.keep = 5
13
+ end
14
+ end
15
+
16
+ it 'should be a subclass of Storage::Base' do
17
+ Backup::Storage::SFTP.
18
+ superclass.should == Backup::Storage::Base
19
+ end
20
+
21
+ describe '#initialize' do
22
+ after { Backup::Storage::SFTP.clear_defaults! }
23
+
24
+ it 'should load pre-configured defaults through Base' do
25
+ Backup::Storage::SFTP.any_instance.expects(:load_defaults!)
26
+ storage
27
+ end
28
+
29
+ it 'should pass the model reference to Base' do
30
+ storage.instance_variable_get(:@model).should == model
31
+ end
32
+
33
+ it 'should pass the storage_id to Base' do
34
+ storage = Backup::Storage::SFTP.new(model, 'my_storage_id')
35
+ storage.storage_id.should == 'my_storage_id'
36
+ end
37
+
38
+ it 'should remove any preceeding tilde and slash from the path' do
39
+ storage = Backup::Storage::SFTP.new(model) do |sftp|
40
+ sftp.path = '~/my_backups/path'
41
+ end
42
+ storage.path.should == 'my_backups/path'
43
+ end
44
+
45
+ context 'when no pre-configured defaults have been set' do
46
+ it 'should use the values given' do
47
+ storage.username.should == 'my_username'
48
+ storage.password.should == 'my_password'
49
+ storage.ip.should == '123.45.678.90'
50
+ storage.port.should == 22
51
+ storage.path.should == 'backups'
52
+
53
+ storage.storage_id.should be_nil
54
+ storage.keep.should == 5
55
+ end
56
+
57
+ it 'should use default values if none are given' do
58
+ storage = Backup::Storage::SFTP.new(model)
59
+
60
+ storage.username.should be_nil
61
+ storage.password.should be_nil
62
+ storage.ip.should be_nil
63
+ storage.port.should == 22
64
+ storage.path.should == 'backups'
65
+
66
+ storage.storage_id.should be_nil
67
+ storage.keep.should be_nil
68
+ end
69
+ end # context 'when no pre-configured defaults have been set'
70
+
71
+ context 'when pre-configured defaults have been set' do
72
+ before do
73
+ Backup::Storage::SFTP.defaults do |s|
74
+ s.username = 'some_username'
75
+ s.password = 'some_password'
76
+ s.ip = 'some_ip'
77
+ s.port = 'some_port'
78
+ s.path = 'some_path'
79
+ s.keep = 'some_keep'
80
+ end
81
+ end
82
+
83
+ it 'should use pre-configured defaults' do
84
+ storage = Backup::Storage::SFTP.new(model)
85
+
86
+ storage.username.should == 'some_username'
87
+ storage.password.should == 'some_password'
88
+ storage.ip.should == 'some_ip'
89
+ storage.port.should == 'some_port'
90
+ storage.path.should == 'some_path'
91
+
92
+ storage.storage_id.should be_nil
93
+ storage.keep.should == 'some_keep'
94
+ end
95
+
96
+ it 'should override pre-configured defaults' do
97
+ storage = Backup::Storage::SFTP.new(model) do |s|
98
+ s.username = 'new_username'
99
+ s.password = 'new_password'
100
+ s.ip = 'new_ip'
101
+ s.port = 'new_port'
102
+ s.path = 'new_path'
103
+ s.keep = 'new_keep'
104
+ end
105
+
106
+ storage.username.should == 'new_username'
107
+ storage.password.should == 'new_password'
108
+ storage.ip.should == 'new_ip'
109
+ storage.port.should == 'new_port'
110
+ storage.path.should == 'new_path'
111
+
112
+ storage.storage_id.should be_nil
113
+ storage.keep.should == 'new_keep'
114
+ end
115
+ end # context 'when pre-configured defaults have been set'
116
+ end # describe '#initialize'
117
+
118
+ describe '#connection' do
119
+ let(:connection) { mock }
120
+
121
+ it 'should yield a connection to the remote server' do
122
+ Net::SFTP.expects(:start).with(
123
+ '123.45.678.90', 'my_username', :password => 'my_password', :port => 22
124
+ ).yields(connection)
125
+
126
+ storage.send(:connection) do |sftp|
127
+ sftp.should be(connection)
128
+ end
129
+ end
130
+ end
131
+
132
+ describe '#transfer!' do
133
+ let(:connection) { mock }
134
+ let(:package) { mock }
135
+ let(:s) { sequence '' }
136
+
137
+ before do
138
+ storage.instance_variable_set(:@package, package)
139
+ storage.stubs(:storage_name).returns('Storage::SFTP')
140
+ storage.stubs(:local_path).returns('/local/path')
141
+ storage.stubs(:connection).yields(connection)
142
+ end
143
+
144
+ it 'should transfer the package files' do
145
+ storage.expects(:remote_path_for).in_sequence(s).with(package).
146
+ returns('remote/path')
147
+ storage.expects(:create_remote_path).in_sequence(s).with(
148
+ 'remote/path', connection
149
+ )
150
+
151
+ storage.expects(:files_to_transfer_for).in_sequence(s).with(package).
152
+ multiple_yields(
153
+ ['2011.12.31.11.00.02.backup.tar.enc-aa', 'backup.tar.enc-aa'],
154
+ ['2011.12.31.11.00.02.backup.tar.enc-ab', 'backup.tar.enc-ab']
155
+ )
156
+ # first yield
157
+ Backup::Logger.expects(:message).in_sequence(s).with(
158
+ "Storage::SFTP started transferring " +
159
+ "'2011.12.31.11.00.02.backup.tar.enc-aa' to '123.45.678.90'."
160
+ )
161
+ connection.expects(:upload!).in_sequence(s).with(
162
+ File.join('/local/path', '2011.12.31.11.00.02.backup.tar.enc-aa'),
163
+ File.join('remote/path', 'backup.tar.enc-aa')
164
+ )
165
+ # second yield
166
+ Backup::Logger.expects(:message).in_sequence(s).with(
167
+ "Storage::SFTP started transferring " +
168
+ "'2011.12.31.11.00.02.backup.tar.enc-ab' to '123.45.678.90'."
169
+ )
170
+ connection.expects(:upload!).in_sequence(s).with(
171
+ File.join('/local/path', '2011.12.31.11.00.02.backup.tar.enc-ab'),
172
+ File.join('remote/path', 'backup.tar.enc-ab')
173
+ )
174
+
175
+ storage.send(:transfer!)
176
+ end
177
+ end # describe '#transfer!'
178
+
179
+ describe '#remove!' do
180
+ let(:package) { mock }
181
+ let(:connection) { mock }
182
+ let(:s) { sequence '' }
183
+
184
+ before do
185
+ storage.stubs(:storage_name).returns('Storage::SFTP')
186
+ storage.stubs(:connection).yields(connection)
187
+ end
188
+
189
+ it 'should remove the package files' do
190
+ storage.expects(:remote_path_for).in_sequence(s).with(package).
191
+ returns('remote/path')
192
+
193
+ storage.expects(:transferred_files_for).in_sequence(s).with(package).
194
+ multiple_yields(
195
+ ['2011.12.31.11.00.02.backup.tar.enc-aa', 'backup.tar.enc-aa'],
196
+ ['2011.12.31.11.00.02.backup.tar.enc-ab', 'backup.tar.enc-ab']
197
+ )
198
+ # first yield
199
+ Backup::Logger.expects(:message).in_sequence(s).with(
200
+ "Storage::SFTP started removing " +
201
+ "'2011.12.31.11.00.02.backup.tar.enc-aa' from '123.45.678.90'."
202
+ )
203
+ connection.expects(:remove!).in_sequence(s).with(
204
+ File.join('remote/path', 'backup.tar.enc-aa')
205
+ )
206
+ # second yield
207
+ Backup::Logger.expects(:message).in_sequence(s).with(
208
+ "Storage::SFTP started removing " +
209
+ "'2011.12.31.11.00.02.backup.tar.enc-ab' from '123.45.678.90'."
210
+ )
211
+ connection.expects(:remove!).in_sequence(s).with(
212
+ File.join('remote/path', 'backup.tar.enc-ab')
213
+ )
214
+
215
+ connection.expects(:rmdir!).with('remote/path').in_sequence(s)
216
+
217
+ storage.send(:remove!, package)
218
+ end
219
+ end # describe '#remove!'
220
+
221
+ describe '#create_remote_path' do
222
+ let(:connection) { mock }
223
+ let(:remote_path) { 'backups/folder/another_folder' }
224
+ let(:s) { sequence '' }
225
+ let(:sftp_response) { stub(:code => 11, :message => nil) }
226
+ let(:sftp_status_exception) { Net::SFTP::StatusException.new(sftp_response) }
227
+
228
+ context 'while properly creating remote directories one by one' do
229
+ it 'should rescue any SFTP::StatusException and continue' do
230
+ connection.expects(:mkdir!).in_sequence(s).
231
+ with("backups").raises(sftp_status_exception)
232
+ connection.expects(:mkdir!).in_sequence(s).
233
+ with("backups/folder")
234
+ connection.expects(:mkdir!).in_sequence(s).
235
+ with("backups/folder/another_folder")
236
+
237
+ expect do
238
+ storage.send(:create_remote_path, remote_path, connection)
239
+ end.not_to raise_error
240
+ end
241
+ end
242
+ end
243
+
244
+ end
@@ -0,0 +1,109 @@
1
+ # encoding: utf-8
2
+
3
+ require File.expand_path('../../spec_helper.rb', __FILE__)
4
+
5
+ describe Backup::Syncer::Base do
6
+ let(:syncer) { Backup::Syncer::Base.new }
7
+
8
+ it 'should include CLI::Helpers' do
9
+ Backup::Syncer::Base.
10
+ include?(Backup::CLI::Helpers).should be_true
11
+ end
12
+
13
+ it 'should include Configuration::Helpers' do
14
+ Backup::Syncer::Base.
15
+ include?(Backup::Configuration::Helpers).should be_true
16
+ end
17
+
18
+ describe '#initialize' do
19
+ after { Backup::Syncer::Base.clear_defaults! }
20
+
21
+ it 'should load pre-configured defaults through Base' do
22
+ Backup::Syncer::Base.any_instance.expects(:load_defaults!)
23
+ syncer
24
+ end
25
+
26
+ it 'should establish a new array for @directories' do
27
+ syncer.directories.should == []
28
+ end
29
+
30
+ context 'when no pre-configured defaults have been set' do
31
+ it 'should set default values' do
32
+ syncer.path.should == 'backups'
33
+ syncer.mirror.should == false
34
+ end
35
+ end # context 'when no pre-configured defaults have been set'
36
+
37
+ context 'when pre-configured defaults have been set' do
38
+ before do
39
+ Backup::Syncer::Base.defaults do |s|
40
+ s.path = 'some_path'
41
+ s.mirror = 'some_mirror'
42
+ end
43
+ end
44
+
45
+ it 'should use pre-configured defaults' do
46
+ syncer.path.should == 'some_path'
47
+ syncer.mirror.should == 'some_mirror'
48
+ end
49
+ end # context 'when pre-configured defaults have been set'
50
+ end # describe '#initialize'
51
+
52
+ describe '#directories' do
53
+ before do
54
+ syncer.instance_variable_set(
55
+ :@directories, ['/some/directory', '/another/directory']
56
+ )
57
+ end
58
+
59
+ context 'when no block is given' do
60
+ it 'should return @directories' do
61
+ syncer.directories.should ==
62
+ ['/some/directory', '/another/directory']
63
+ end
64
+ end
65
+
66
+ context 'when a block is given' do
67
+ it 'should evalute the block, allowing #add to add directories' do
68
+ syncer.directories do
69
+ add '/new/path'
70
+ add '/another/new/path'
71
+ end
72
+ syncer.directories.should == [
73
+ '/some/directory',
74
+ '/another/directory',
75
+ '/new/path',
76
+ '/another/new/path'
77
+ ]
78
+ end
79
+ end
80
+ end # describe '#directories'
81
+
82
+ describe '#add' do
83
+ before do
84
+ syncer.instance_variable_set(
85
+ :@directories, ['/some/directory', '/another/directory']
86
+ )
87
+ end
88
+
89
+ it 'should add the given path to @directories' do
90
+ syncer.add '/my/path'
91
+ syncer.directories.should ==
92
+ ['/some/directory', '/another/directory', '/my/path']
93
+ end
94
+
95
+ # Note: Each Syncer should handle this as needed.
96
+ # For example, expanding these here would break RSync::Pull
97
+ it 'should not expand the given paths' do
98
+ syncer.add 'relative/path'
99
+ syncer.directories.should ==
100
+ ['/some/directory', '/another/directory', 'relative/path']
101
+ end
102
+ end
103
+
104
+ describe '#syncer_name' do
105
+ it 'should return the class name with the Backup:: namespace removed' do
106
+ syncer.send(:syncer_name).should == 'Syncer::Base'
107
+ end
108
+ end
109
+ end
@@ -0,0 +1,515 @@
1
+ # encoding: utf-8
2
+ require File.expand_path('../../../spec_helper.rb', __FILE__)
3
+
4
+ describe 'Backup::Syncer::Cloud::Base' do
5
+ let(:syncer) { Backup::Syncer::Cloud::Base.new }
6
+ let(:s) { sequence '' }
7
+
8
+ it 'should be a subclass of Syncer::Base' do
9
+ Backup::Syncer::Cloud::Base.
10
+ superclass.should == Backup::Syncer::Base
11
+ end
12
+
13
+ it 'should establish a class constant for a Mutex' do
14
+ Backup::Syncer::Cloud::Base::MUTEX.should be_an_instance_of Mutex
15
+ end
16
+
17
+ describe '#initialize' do
18
+ after { Backup::Syncer::Cloud::Base.clear_defaults! }
19
+
20
+ it 'should load pre-configured defaults through Syncer::Base' do
21
+ Backup::Syncer::Cloud::Base.any_instance.expects(:load_defaults!)
22
+ syncer
23
+ end
24
+
25
+ context 'when no pre-configured defaults have been set' do
26
+ it 'should use default values if none are given' do
27
+ syncer.path.should == 'backups'
28
+ syncer.mirror.should == false
29
+ syncer.concurrency_type.should == false
30
+ syncer.concurrency_level.should == 2
31
+ end
32
+ end # context 'when no pre-configured defaults have been set'
33
+
34
+ context 'when pre-configured defaults have been set' do
35
+ before do
36
+ Backup::Syncer::Cloud::Base.defaults do |cloud|
37
+ cloud.concurrency_type = 'default_concurrency_type'
38
+ cloud.concurrency_level = 'default_concurrency_level'
39
+ end
40
+ end
41
+
42
+ it 'should use pre-configured defaults' do
43
+ syncer.path.should == 'backups'
44
+ syncer.mirror.should == false
45
+ syncer.concurrency_type.should == 'default_concurrency_type'
46
+ syncer.concurrency_level.should == 'default_concurrency_level'
47
+ end
48
+ end # context 'when pre-configured defaults have been set'
49
+ end # describe '#initialize'
50
+
51
+ describe '#perform' do
52
+ let(:sync_context) { mock }
53
+
54
+ before do
55
+ syncer.stubs(:repository_object).returns(:a_repository_object)
56
+
57
+ Backup::Logger.expects(:message).with(
58
+ "Syncer::Cloud::Base started the syncing process:\n" +
59
+ "\s\sConcurrency: false Level: 2"
60
+ )
61
+ Backup::Logger.expects(:message).with(
62
+ 'Syncer::Cloud::Base Syncing Complete!'
63
+ )
64
+ end
65
+
66
+ it 'should sync each directory' do
67
+ syncer.directories do
68
+ add '/dir/one'
69
+ add '/dir/two'
70
+ end
71
+
72
+ Backup::Syncer::Cloud::Base::SyncContext.expects(:new).in_sequence(s).with(
73
+ '/dir/one', :a_repository_object, 'backups'
74
+ ).returns(sync_context)
75
+ sync_context.expects(:sync!).in_sequence(s).with(
76
+ false, false, 2
77
+ )
78
+ Backup::Syncer::Cloud::Base::SyncContext.expects(:new).in_sequence(s).with(
79
+ '/dir/two', :a_repository_object, 'backups'
80
+ ).returns(sync_context)
81
+ sync_context.expects(:sync!).in_sequence(s).with(
82
+ false, false, 2
83
+ )
84
+
85
+ syncer.perform!
86
+ end
87
+
88
+ it 'should ensure each directory path is expanded with no trailing slash' do
89
+ syncer.directories do
90
+ add '/dir/one/'
91
+ add 'dir/two'
92
+ end
93
+
94
+ Backup::Syncer::Cloud::Base::SyncContext.expects(:new).with(
95
+ '/dir/one', :a_repository_object, 'backups'
96
+ ).returns(sync_context)
97
+
98
+ Backup::Syncer::Cloud::Base::SyncContext.expects(:new).with(
99
+ File.expand_path('dir/two'), :a_repository_object, 'backups'
100
+ ).returns(sync_context)
101
+
102
+ sync_context.stubs(:sync!)
103
+
104
+ syncer.perform!
105
+ end
106
+ end # describe '#perform'
107
+
108
+ describe 'Cloud::Base::SyncContext' do
109
+ let(:bucket) { mock }
110
+ let(:sync_context) do
111
+ Backup::Syncer::Cloud::Base::SyncContext.new(
112
+ '/dir/to/sync', bucket, 'backups'
113
+ )
114
+ end
115
+
116
+ describe '#initialize' do
117
+ it 'should set variables' do
118
+ sync_context.directory.should == '/dir/to/sync'
119
+ sync_context.bucket.should == bucket
120
+ sync_context.path.should == 'backups'
121
+ sync_context.remote_base.should == 'backups/sync'
122
+ end
123
+ end
124
+
125
+ describe '#sync!' do
126
+ let(:all_files_array) { mock }
127
+
128
+ before do
129
+ sync_context.stubs(:all_file_names).returns(all_files_array)
130
+ end
131
+
132
+ context 'when concurrency_type is set to `false`' do
133
+ it 'syncs files without concurrency' do
134
+ all_files_array.expects(:each).in_sequence(s).
135
+ multiple_yields('foo.file', 'foo_dir/foo.file')
136
+
137
+ sync_context.expects(:sync_file).in_sequence(s).
138
+ with('foo.file', :mirror)
139
+ sync_context.expects(:sync_file).in_sequence(s).
140
+ with('foo_dir/foo.file', :mirror)
141
+
142
+ sync_context.sync!(:mirror, false, :foo)
143
+ end
144
+ end
145
+
146
+ context 'when concurrency_type is set to `:threads`' do
147
+ it 'uses `concurrency_level` number of threads for concurrency' do
148
+ Parallel.expects(:each).in_sequence(s).with(
149
+ all_files_array, :in_threads => :num_of_threads
150
+ ).multiple_yields('foo.file', 'foo_dir/foo.file')
151
+
152
+ sync_context.expects(:sync_file).in_sequence(s).
153
+ with('foo.file', :mirror)
154
+ sync_context.expects(:sync_file).in_sequence(s).
155
+ with('foo_dir/foo.file', :mirror)
156
+
157
+ sync_context.sync!(:mirror, :threads, :num_of_threads)
158
+ end
159
+ end
160
+
161
+ context 'when concurrency_type is set to `:processes`' do
162
+ it 'uses `concurrency_level` number of processes for concurrency' do
163
+ Parallel.expects(:each).in_sequence(s).with(
164
+ all_files_array, :in_processes => :num_of_processes
165
+ ).multiple_yields('foo.file', 'foo_dir/foo.file')
166
+
167
+ sync_context.expects(:sync_file).in_sequence(s).
168
+ with('foo.file', :mirror)
169
+ sync_context.expects(:sync_file).in_sequence(s).
170
+ with('foo_dir/foo.file', :mirror)
171
+
172
+ sync_context.sync!(:mirror, :processes, :num_of_processes)
173
+ end
174
+ end
175
+
176
+ context 'when concurrency_type setting is invalid' do
177
+ it 'should raise an error' do
178
+ expect do
179
+ sync_context.sync!(:foo, 'unknown type', :foo)
180
+ end.to raise_error(
181
+ Backup::Errors::Syncer::Cloud::ConfigurationError,
182
+ 'Syncer::Cloud::ConfigurationError: ' +
183
+ "Unknown concurrency_type setting: \"unknown type\""
184
+ )
185
+ end
186
+ end
187
+ end # describe '#sync!'
188
+
189
+ describe '#all_file_names' do
190
+ let(:local_files_hash) do
191
+ { 'file_b' => :foo, 'file_a' => :foo, 'dir_a/file_b' => :foo }
192
+ end
193
+ let(:remote_files_hash) do
194
+ { 'file_c' => :foo, 'file_a' => :foo, 'dir_a/file_a' => :foo }
195
+ end
196
+ let(:local_remote_union_array) do
197
+ ['dir_a/file_a', 'dir_a/file_b', 'file_a', 'file_b', 'file_c']
198
+ end
199
+
200
+ it 'returns and caches a sorted union of local and remote file names' do
201
+ sync_context.expects(:local_files).once.returns(local_files_hash)
202
+ sync_context.expects(:remote_files).once.returns(remote_files_hash)
203
+
204
+ sync_context.send(:all_file_names).should == local_remote_union_array
205
+ sync_context.instance_variable_get(:@all_file_names).
206
+ should == local_remote_union_array
207
+ sync_context.send(:all_file_names).should == local_remote_union_array
208
+ end
209
+ end # describe '#all_file_names'
210
+
211
+ describe '#local_files' do
212
+ let(:local_file_class) { Backup::Syncer::Cloud::Base::LocalFile }
213
+ let(:local_hashes_data) { "line1\nline2\nbad\xFFline\nline3" }
214
+
215
+ let(:local_file_a) { stub(:relative_path => 'file_a') }
216
+ let(:local_file_b) { stub(:relative_path => 'file_b') }
217
+ let(:local_file_c) { stub(:relative_path => 'file_c') }
218
+ let(:local_files_hash) do
219
+ { 'file_a' => local_file_a,
220
+ 'file_b' => local_file_b,
221
+ 'file_c' => local_file_c }
222
+ end
223
+
224
+ it 'should return and caches a hash of LocalFile objects' do
225
+ sync_context.expects(:local_hashes).once.returns(local_hashes_data)
226
+
227
+ local_file_class.expects(:new).once.with('/dir/to/sync', "line1\n").
228
+ returns(local_file_a)
229
+ local_file_class.expects(:new).once.with('/dir/to/sync', "line2\n").
230
+ returns(local_file_b)
231
+ local_file_class.expects(:new).once.with('/dir/to/sync', "bad\xFFline\n").
232
+ returns(nil)
233
+ local_file_class.expects(:new).once.with('/dir/to/sync', "line3").
234
+ returns(local_file_c)
235
+
236
+ sync_context.send(:local_files).should == local_files_hash
237
+ sync_context.instance_variable_get(:@local_files).
238
+ should == local_files_hash
239
+ sync_context.send(:local_files).should == local_files_hash
240
+ end
241
+
242
+ # Note: don't use methods that validate encoding
243
+ it 'will raise an Exception if String#split is used',
244
+ :if => RUBY_VERSION >= '1.9' do
245
+ expect do
246
+ "line1\nbad\xFFline\nline3".split("\n")
247
+ end.to raise_error(ArgumentError, 'invalid byte sequence in UTF-8')
248
+ end
249
+ end # describe '#local_files'
250
+
251
+ describe '#local_hashes' do
252
+ it 'should collect file paths and MD5 checksums for @directory' do
253
+ Backup::Logger.expects(:message).with(
254
+ "\s\sGenerating checksums for '/dir/to/sync'"
255
+ )
256
+ sync_context.expects(:`).with(
257
+ "find '/dir/to/sync' -print0 | xargs -0 openssl md5 2> /dev/null"
258
+ ).returns('MD5(tmp/foo)= 0123456789abcdefghijklmnopqrstuv')
259
+
260
+ sync_context.send(:local_hashes).should ==
261
+ 'MD5(tmp/foo)= 0123456789abcdefghijklmnopqrstuv'
262
+ end
263
+ end
264
+
265
+ describe '#remote_files' do
266
+ let(:repository_object) { mock }
267
+ let(:repository_files) { mock }
268
+ let(:file_objects) { mock }
269
+ let(:file_obj_a) { stub(:key => 'file_a') }
270
+ let(:file_obj_b) { stub(:key => 'file_b') }
271
+ let(:file_obj_c) { stub(:key => 'dir/file_c') }
272
+ let(:remote_files_hash) do
273
+ { 'file_a' => file_obj_a,
274
+ 'file_b' => file_obj_b,
275
+ 'dir/file_c' => file_obj_c }
276
+ end
277
+
278
+ before do
279
+ sync_context.instance_variable_set(:@bucket, repository_object)
280
+
281
+ repository_object.expects(:files).once.returns(repository_files)
282
+ repository_files.expects(:all).once.with(:prefix => 'backups/sync').
283
+ returns(file_objects)
284
+ file_objects.expects(:each).once.multiple_yields(
285
+ file_obj_a, file_obj_b, file_obj_c
286
+ )
287
+
288
+ # this is to avoid: unexpected invocation: #<Mock>.to_a()
289
+ # only 1.9.2 seems affected by this
290
+ if RUBY_VERSION == '1.9.2'
291
+ file_obj_a.stubs(:to_a)
292
+ file_obj_b.stubs(:to_a)
293
+ file_obj_c.stubs(:to_a)
294
+ end
295
+ end
296
+
297
+ context 'when it returns and caches a hash of repository file objects' do
298
+ it 'should remove the @remote_base from the path for the hash key' do
299
+ sync_context.send(:remote_files).should == remote_files_hash
300
+ sync_context.instance_variable_get(:@remote_files).
301
+ should == remote_files_hash
302
+ sync_context.send(:remote_files).should == remote_files_hash
303
+ end
304
+ end
305
+ end # describe '#remote_files'
306
+
307
+ describe '#sync_file' do
308
+ let(:local_file) do
309
+ stub(
310
+ :path => '/dir/to/sync/sync.file',
311
+ :md5 => '0123456789abcdefghijklmnopqrstuv')
312
+ end
313
+ let(:remote_file) do
314
+ stub(:path => 'backups/sync/sync.file')
315
+ end
316
+ let(:file) { mock }
317
+ let(:repository_object) { mock }
318
+ let(:repository_files) { mock }
319
+
320
+ before do
321
+ sync_context.instance_variable_set(:@bucket, repository_object)
322
+ repository_object.stubs(:files).returns(repository_files)
323
+ end
324
+
325
+ context 'when the requested file to sync exists locally' do
326
+ before do
327
+ sync_context.stubs(:local_files).returns(
328
+ { 'sync.file' => local_file }
329
+ )
330
+ File.expects(:exist?).with('/dir/to/sync/sync.file').returns(true)
331
+ end
332
+
333
+ context 'when the MD5 checksum matches the remote file' do
334
+ before do
335
+ remote_file.stubs(:etag).returns('0123456789abcdefghijklmnopqrstuv')
336
+ sync_context.stubs(:remote_files).returns(
337
+ { 'sync.file' => remote_file }
338
+ )
339
+ end
340
+
341
+ it 'should skip the file' do
342
+ File.expects(:open).never
343
+ Backup::Syncer::Cloud::Base::MUTEX.expects(:synchronize).yields
344
+ Backup::Logger.expects(:message).with(
345
+ "\s\s[skipping] 'backups/sync/sync.file'"
346
+ )
347
+
348
+ sync_context.send(:sync_file, 'sync.file', :foo)
349
+ end
350
+ end
351
+
352
+ context 'when the MD5 checksum does not match the remote file' do
353
+ before do
354
+ remote_file.stubs(:etag).returns('vutsrqponmlkjihgfedcba9876543210')
355
+ sync_context.stubs(:remote_files).returns(
356
+ { 'sync.file' => remote_file }
357
+ )
358
+ end
359
+
360
+ it 'should upload the file' do
361
+ Backup::Syncer::Cloud::Base::MUTEX.expects(:synchronize).yields
362
+ Backup::Logger.expects(:message).with(
363
+ "\s\s[transferring] 'backups/sync/sync.file'"
364
+ )
365
+
366
+ File.expects(:open).with('/dir/to/sync/sync.file', 'r').yields(file)
367
+ repository_files.expects(:create).with(
368
+ :key => 'backups/sync/sync.file',
369
+ :body => file
370
+ )
371
+
372
+ sync_context.send(:sync_file, 'sync.file', :foo)
373
+ end
374
+ end
375
+
376
+ context 'when the requested file does not exist on the remote' do
377
+ before do
378
+ sync_context.stubs(:remote_files).returns({})
379
+ end
380
+
381
+ it 'should upload the file' do
382
+ Backup::Syncer::Cloud::Base::MUTEX.expects(:synchronize).yields
383
+ Backup::Logger.expects(:message).with(
384
+ "\s\s[transferring] 'backups/sync/sync.file'"
385
+ )
386
+
387
+ File.expects(:open).with('/dir/to/sync/sync.file', 'r').yields(file)
388
+ repository_files.expects(:create).with(
389
+ :key => 'backups/sync/sync.file',
390
+ :body => file
391
+ )
392
+
393
+ sync_context.send(:sync_file, 'sync.file', :foo)
394
+ end
395
+ end
396
+ end
397
+
398
+ context 'when the requested file does not exist locally' do
399
+ before do
400
+ sync_context.stubs(:remote_files).returns(
401
+ { 'sync.file' => remote_file }
402
+ )
403
+ sync_context.stubs(:local_files).returns({})
404
+ end
405
+
406
+ context 'when the `mirror` option is set to true' do
407
+ it 'should remove the file from the remote' do
408
+ Backup::Syncer::Cloud::Base::MUTEX.expects(:synchronize).yields
409
+ Backup::Logger.expects(:message).with(
410
+ "\s\s[removing] 'backups/sync/sync.file'"
411
+ )
412
+
413
+ remote_file.expects(:destroy)
414
+
415
+ sync_context.send(:sync_file, 'sync.file', true)
416
+ end
417
+ end
418
+
419
+ context 'when the `mirror` option is set to false' do
420
+ it 'should leave the file on the remote' do
421
+ Backup::Syncer::Cloud::Base::MUTEX.expects(:synchronize).yields
422
+ Backup::Logger.expects(:message).with(
423
+ "\s\s[leaving] 'backups/sync/sync.file'"
424
+ )
425
+
426
+ remote_file.expects(:destroy).never
427
+
428
+ sync_context.send(:sync_file, 'sync.file', false)
429
+ end
430
+ end
431
+ end
432
+ end # describe '#sync_file'
433
+ end # describe 'Cloud::Base::SyncContext'
434
+
435
+ describe 'Cloud::Base::LocalFile' do
436
+ let(:local_file_class) { Backup::Syncer::Cloud::Base::LocalFile }
437
+
438
+ describe '#new' do
439
+ describe 'wrapping #initialize and using #sanitize to validate objects' do
440
+ context 'when the path is valid UTF-8' do
441
+ let(:local_file) do
442
+ local_file_class.new(
443
+ 'foo',
444
+ 'MD5(foo)= 0123456789abcdefghijklmnopqrstuv'
445
+ )
446
+ end
447
+
448
+ it 'should return the new object' do
449
+ Backup::Logger.expects(:warn).never
450
+
451
+ local_file.should be_an_instance_of local_file_class
452
+ end
453
+ end
454
+
455
+ context 'when the path contains invalid UTF-8' do
456
+ let(:local_file) do
457
+ local_file_class.new(
458
+ "/bad/pa\xFFth",
459
+ "MD5(/bad/pa\xFFth/to/file)= 0123456789abcdefghijklmnopqrstuv"
460
+ )
461
+ end
462
+ it 'should return nil and log a warning' do
463
+ Backup::Logger.expects(:warn).with(
464
+ "\s\s[skipping] /bad/pa\xEF\xBF\xBDth/to/file\n" +
465
+ "\s\sPath Contains Invalid UTF-8 byte sequences"
466
+ )
467
+
468
+ local_file.should be_nil
469
+ end
470
+ end
471
+ end
472
+ end # describe '#new'
473
+
474
+ describe '#initialize' do
475
+ let(:local_file) do
476
+ local_file_class.new(:directory, :line)
477
+ end
478
+
479
+ before do
480
+ local_file_class.any_instance.expects(:sanitize).with(:directory).
481
+ returns('/dir/to/sync')
482
+ local_file_class.any_instance.expects(:sanitize).with(:line).
483
+ returns("MD5(/dir/to/sync/subdir/sync.file)= 0123456789abcdefghijklmnopqrstuv\n")
484
+ end
485
+
486
+ it 'should determine @path, @relative_path and @md5' do
487
+ local_file.path.should == '/dir/to/sync/subdir/sync.file'
488
+ local_file.relative_path.should == 'subdir/sync.file'
489
+ local_file.md5.should == '0123456789abcdefghijklmnopqrstuv'
490
+ end
491
+
492
+ it 'should return nil if the object is invalid' do
493
+ local_file_class.any_instance.expects(:invalid?).returns(true)
494
+ Backup::Logger.expects(:warn)
495
+ local_file.should be_nil
496
+ end
497
+ end # describe '#initialize'
498
+
499
+ describe '#sanitize' do
500
+ let(:local_file) do
501
+ local_file_class.new('foo', 'MD5(foo)= 0123456789abcdefghijklmnopqrstuv')
502
+ end
503
+
504
+ it 'should replace any invalid UTF-8 characters' do
505
+ local_file.send(:sanitize, "/path/to/d\xFFir/subdir/sync\xFFfile").
506
+ should == "/path/to/d\xEF\xBF\xBDir/subdir/sync\xEF\xBF\xBDfile"
507
+ end
508
+
509
+ it 'should flag the LocalFile object as invalid' do
510
+ local_file.send(:sanitize, "/path/to/d\xFFir/subdir/sync\xFFfile")
511
+ local_file.invalid?.should be_true
512
+ end
513
+ end # describe '#sanitize'
514
+ end # describe 'Cloud::Base::LocalFile'
515
+ end