backup 3.0.27 → 3.1.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (166) hide show
  1. data/LICENSE.md +1 -1
  2. data/README.md +139 -386
  3. data/bin/backup +1 -7
  4. data/lib/backup.rb +3 -9
  5. data/lib/backup/archive.rb +26 -20
  6. data/lib/backup/cleaner.rb +2 -2
  7. data/lib/backup/cli.rb +366 -0
  8. data/lib/backup/compressor/base.rb +2 -2
  9. data/lib/backup/compressor/gzip.rb +35 -1
  10. data/lib/backup/config.rb +1 -2
  11. data/lib/backup/database/base.rb +2 -2
  12. data/lib/backup/database/mongodb.rb +3 -3
  13. data/lib/backup/database/mysql.rb +3 -2
  14. data/lib/backup/database/postgresql.rb +3 -2
  15. data/lib/backup/database/riak.rb +18 -5
  16. data/lib/backup/dependency.rb +144 -93
  17. data/lib/backup/encryptor/base.rb +2 -2
  18. data/lib/backup/logger.rb +108 -110
  19. data/lib/backup/logger/console.rb +51 -0
  20. data/lib/backup/logger/logfile.rb +113 -0
  21. data/lib/backup/logger/syslog.rb +116 -0
  22. data/lib/backup/model.rb +67 -65
  23. data/lib/backup/notifier/base.rb +1 -1
  24. data/lib/backup/notifier/hipchat.rb +1 -1
  25. data/lib/backup/notifier/mail.rb +1 -1
  26. data/lib/backup/notifier/pushover.rb +6 -3
  27. data/lib/backup/packager.rb +4 -4
  28. data/lib/backup/pipeline.rb +17 -3
  29. data/lib/backup/splitter.rb +2 -2
  30. data/lib/backup/storage/base.rb +2 -2
  31. data/lib/backup/storage/cloudfiles.rb +2 -2
  32. data/lib/backup/storage/dropbox.rb +4 -4
  33. data/lib/backup/storage/ftp.rb +2 -2
  34. data/lib/backup/storage/local.rb +2 -2
  35. data/lib/backup/storage/ninefold.rb +2 -2
  36. data/lib/backup/storage/rsync.rb +3 -3
  37. data/lib/backup/storage/s3.rb +2 -2
  38. data/lib/backup/storage/scp.rb +2 -6
  39. data/lib/backup/storage/sftp.rb +2 -5
  40. data/lib/backup/syncer/base.rb +1 -1
  41. data/lib/backup/syncer/cloud/base.rb +15 -8
  42. data/lib/backup/syncer/rsync/local.rb +1 -1
  43. data/lib/backup/syncer/rsync/pull.rb +1 -1
  44. data/lib/backup/syncer/rsync/push.rb +1 -1
  45. data/lib/backup/utilities.rb +211 -0
  46. data/lib/backup/version.rb +1 -1
  47. data/templates/cli/{utility/archive → archive} +4 -8
  48. data/templates/cli/{utility/compressor → compressor}/bzip2 +0 -0
  49. data/templates/cli/{utility/compressor → compressor}/custom +0 -0
  50. data/templates/cli/{utility/compressor → compressor}/gzip +0 -0
  51. data/templates/cli/{utility/compressor → compressor}/lzma +0 -0
  52. data/templates/cli/{utility/compressor → compressor}/pbzip2 +0 -0
  53. data/templates/cli/config +68 -0
  54. data/templates/cli/{utility/database → database}/mongodb +1 -1
  55. data/templates/cli/{utility/database → database}/mysql +1 -1
  56. data/templates/cli/{utility/database → database}/postgresql +1 -1
  57. data/templates/cli/{utility/database → database}/redis +0 -0
  58. data/templates/cli/database/riak +20 -0
  59. data/templates/cli/{utility/encryptor → encryptor}/gpg +0 -0
  60. data/templates/cli/{utility/encryptor → encryptor}/openssl +0 -0
  61. data/templates/cli/{utility/model.erb → model.erb} +4 -4
  62. data/templates/cli/{utility/notifier → notifier}/campfire +0 -0
  63. data/templates/cli/{utility/notifier → notifier}/hipchat +0 -0
  64. data/templates/cli/{utility/notifier → notifier}/mail +0 -0
  65. data/templates/cli/{utility/notifier → notifier}/prowl +0 -0
  66. data/templates/cli/{utility/notifier → notifier}/pushover +0 -0
  67. data/templates/cli/{utility/notifier → notifier}/twitter +0 -0
  68. data/templates/cli/{utility/splitter → splitter} +0 -0
  69. data/templates/cli/{utility/storage → storage}/cloud_files +0 -0
  70. data/templates/cli/{utility/storage → storage}/dropbox +0 -0
  71. data/templates/cli/{utility/storage → storage}/ftp +0 -0
  72. data/templates/cli/{utility/storage → storage}/local +0 -0
  73. data/templates/cli/{utility/storage → storage}/ninefold +0 -0
  74. data/templates/cli/{utility/storage → storage}/rsync +0 -0
  75. data/templates/cli/{utility/storage → storage}/s3 +0 -0
  76. data/templates/cli/{utility/storage → storage}/scp +0 -0
  77. data/templates/cli/{utility/storage → storage}/sftp +0 -0
  78. data/templates/cli/{utility/syncer → syncer}/cloud_files +0 -0
  79. data/templates/cli/{utility/syncer → syncer}/rsync_local +0 -0
  80. data/templates/cli/{utility/syncer → syncer}/rsync_pull +0 -0
  81. data/templates/cli/{utility/syncer → syncer}/rsync_push +0 -0
  82. data/templates/cli/{utility/syncer → syncer}/s3 +0 -0
  83. metadata +55 -131
  84. data/.gitignore +0 -8
  85. data/.travis.yml +0 -10
  86. data/Gemfile +0 -28
  87. data/Guardfile +0 -23
  88. data/backup.gemspec +0 -32
  89. data/lib/backup/cli/helpers.rb +0 -93
  90. data/lib/backup/cli/utility.rb +0 -255
  91. data/spec-live/.gitignore +0 -6
  92. data/spec-live/README +0 -7
  93. data/spec-live/backups/config.rb +0 -83
  94. data/spec-live/backups/config.yml.template +0 -46
  95. data/spec-live/backups/models.rb +0 -184
  96. data/spec-live/compressor/custom_spec.rb +0 -30
  97. data/spec-live/compressor/gzip_spec.rb +0 -30
  98. data/spec-live/encryptor/gpg_keys.rb +0 -239
  99. data/spec-live/encryptor/gpg_spec.rb +0 -287
  100. data/spec-live/notifier/mail_spec.rb +0 -121
  101. data/spec-live/spec_helper.rb +0 -151
  102. data/spec-live/storage/dropbox_spec.rb +0 -151
  103. data/spec-live/storage/local_spec.rb +0 -83
  104. data/spec-live/storage/scp_spec.rb +0 -193
  105. data/spec-live/syncer/cloud/s3_spec.rb +0 -124
  106. data/spec/archive_spec.rb +0 -335
  107. data/spec/cleaner_spec.rb +0 -312
  108. data/spec/cli/helpers_spec.rb +0 -301
  109. data/spec/cli/utility_spec.rb +0 -411
  110. data/spec/compressor/base_spec.rb +0 -52
  111. data/spec/compressor/bzip2_spec.rb +0 -217
  112. data/spec/compressor/custom_spec.rb +0 -106
  113. data/spec/compressor/gzip_spec.rb +0 -217
  114. data/spec/compressor/lzma_spec.rb +0 -123
  115. data/spec/compressor/pbzip2_spec.rb +0 -165
  116. data/spec/config_spec.rb +0 -321
  117. data/spec/configuration/helpers_spec.rb +0 -247
  118. data/spec/configuration/store_spec.rb +0 -39
  119. data/spec/configuration_spec.rb +0 -62
  120. data/spec/database/base_spec.rb +0 -63
  121. data/spec/database/mongodb_spec.rb +0 -510
  122. data/spec/database/mysql_spec.rb +0 -411
  123. data/spec/database/postgresql_spec.rb +0 -353
  124. data/spec/database/redis_spec.rb +0 -334
  125. data/spec/database/riak_spec.rb +0 -176
  126. data/spec/dependency_spec.rb +0 -51
  127. data/spec/encryptor/base_spec.rb +0 -40
  128. data/spec/encryptor/gpg_spec.rb +0 -909
  129. data/spec/encryptor/open_ssl_spec.rb +0 -148
  130. data/spec/errors_spec.rb +0 -306
  131. data/spec/logger_spec.rb +0 -367
  132. data/spec/model_spec.rb +0 -666
  133. data/spec/notifier/base_spec.rb +0 -104
  134. data/spec/notifier/campfire_spec.rb +0 -217
  135. data/spec/notifier/hipchat_spec.rb +0 -211
  136. data/spec/notifier/mail_spec.rb +0 -316
  137. data/spec/notifier/prowl_spec.rb +0 -138
  138. data/spec/notifier/pushover_spec.rb +0 -123
  139. data/spec/notifier/twitter_spec.rb +0 -153
  140. data/spec/package_spec.rb +0 -61
  141. data/spec/packager_spec.rb +0 -213
  142. data/spec/pipeline_spec.rb +0 -259
  143. data/spec/spec_helper.rb +0 -60
  144. data/spec/splitter_spec.rb +0 -120
  145. data/spec/storage/base_spec.rb +0 -166
  146. data/spec/storage/cloudfiles_spec.rb +0 -254
  147. data/spec/storage/cycler_spec.rb +0 -247
  148. data/spec/storage/dropbox_spec.rb +0 -480
  149. data/spec/storage/ftp_spec.rb +0 -271
  150. data/spec/storage/local_spec.rb +0 -259
  151. data/spec/storage/ninefold_spec.rb +0 -343
  152. data/spec/storage/rsync_spec.rb +0 -362
  153. data/spec/storage/s3_spec.rb +0 -245
  154. data/spec/storage/scp_spec.rb +0 -233
  155. data/spec/storage/sftp_spec.rb +0 -244
  156. data/spec/syncer/base_spec.rb +0 -109
  157. data/spec/syncer/cloud/base_spec.rb +0 -515
  158. data/spec/syncer/cloud/cloud_files_spec.rb +0 -181
  159. data/spec/syncer/cloud/s3_spec.rb +0 -174
  160. data/spec/syncer/rsync/base_spec.rb +0 -98
  161. data/spec/syncer/rsync/local_spec.rb +0 -149
  162. data/spec/syncer/rsync/pull_spec.rb +0 -98
  163. data/spec/syncer/rsync/push_spec.rb +0 -333
  164. data/spec/version_spec.rb +0 -21
  165. data/templates/cli/utility/config +0 -32
  166. data/templates/cli/utility/database/riak +0 -11
@@ -1,244 +0,0 @@
1
- # encoding: utf-8
2
-
3
- require File.expand_path('../../spec_helper.rb', __FILE__)
4
-
5
- describe Backup::Storage::SFTP do
6
- let(:model) { Backup::Model.new(:test_trigger, 'test label') }
7
- let(:storage) do
8
- Backup::Storage::SFTP.new(model) do |sftp|
9
- sftp.username = 'my_username'
10
- sftp.password = 'my_password'
11
- sftp.ip = '123.45.678.90'
12
- sftp.keep = 5
13
- end
14
- end
15
-
16
- it 'should be a subclass of Storage::Base' do
17
- Backup::Storage::SFTP.
18
- superclass.should == Backup::Storage::Base
19
- end
20
-
21
- describe '#initialize' do
22
- after { Backup::Storage::SFTP.clear_defaults! }
23
-
24
- it 'should load pre-configured defaults through Base' do
25
- Backup::Storage::SFTP.any_instance.expects(:load_defaults!)
26
- storage
27
- end
28
-
29
- it 'should pass the model reference to Base' do
30
- storage.instance_variable_get(:@model).should == model
31
- end
32
-
33
- it 'should pass the storage_id to Base' do
34
- storage = Backup::Storage::SFTP.new(model, 'my_storage_id')
35
- storage.storage_id.should == 'my_storage_id'
36
- end
37
-
38
- it 'should remove any preceeding tilde and slash from the path' do
39
- storage = Backup::Storage::SFTP.new(model) do |sftp|
40
- sftp.path = '~/my_backups/path'
41
- end
42
- storage.path.should == 'my_backups/path'
43
- end
44
-
45
- context 'when no pre-configured defaults have been set' do
46
- it 'should use the values given' do
47
- storage.username.should == 'my_username'
48
- storage.password.should == 'my_password'
49
- storage.ip.should == '123.45.678.90'
50
- storage.port.should == 22
51
- storage.path.should == 'backups'
52
-
53
- storage.storage_id.should be_nil
54
- storage.keep.should == 5
55
- end
56
-
57
- it 'should use default values if none are given' do
58
- storage = Backup::Storage::SFTP.new(model)
59
-
60
- storage.username.should be_nil
61
- storage.password.should be_nil
62
- storage.ip.should be_nil
63
- storage.port.should == 22
64
- storage.path.should == 'backups'
65
-
66
- storage.storage_id.should be_nil
67
- storage.keep.should be_nil
68
- end
69
- end # context 'when no pre-configured defaults have been set'
70
-
71
- context 'when pre-configured defaults have been set' do
72
- before do
73
- Backup::Storage::SFTP.defaults do |s|
74
- s.username = 'some_username'
75
- s.password = 'some_password'
76
- s.ip = 'some_ip'
77
- s.port = 'some_port'
78
- s.path = 'some_path'
79
- s.keep = 'some_keep'
80
- end
81
- end
82
-
83
- it 'should use pre-configured defaults' do
84
- storage = Backup::Storage::SFTP.new(model)
85
-
86
- storage.username.should == 'some_username'
87
- storage.password.should == 'some_password'
88
- storage.ip.should == 'some_ip'
89
- storage.port.should == 'some_port'
90
- storage.path.should == 'some_path'
91
-
92
- storage.storage_id.should be_nil
93
- storage.keep.should == 'some_keep'
94
- end
95
-
96
- it 'should override pre-configured defaults' do
97
- storage = Backup::Storage::SFTP.new(model) do |s|
98
- s.username = 'new_username'
99
- s.password = 'new_password'
100
- s.ip = 'new_ip'
101
- s.port = 'new_port'
102
- s.path = 'new_path'
103
- s.keep = 'new_keep'
104
- end
105
-
106
- storage.username.should == 'new_username'
107
- storage.password.should == 'new_password'
108
- storage.ip.should == 'new_ip'
109
- storage.port.should == 'new_port'
110
- storage.path.should == 'new_path'
111
-
112
- storage.storage_id.should be_nil
113
- storage.keep.should == 'new_keep'
114
- end
115
- end # context 'when pre-configured defaults have been set'
116
- end # describe '#initialize'
117
-
118
- describe '#connection' do
119
- let(:connection) { mock }
120
-
121
- it 'should yield a connection to the remote server' do
122
- Net::SFTP.expects(:start).with(
123
- '123.45.678.90', 'my_username', :password => 'my_password', :port => 22
124
- ).yields(connection)
125
-
126
- storage.send(:connection) do |sftp|
127
- sftp.should be(connection)
128
- end
129
- end
130
- end
131
-
132
- describe '#transfer!' do
133
- let(:connection) { mock }
134
- let(:package) { mock }
135
- let(:s) { sequence '' }
136
-
137
- before do
138
- storage.instance_variable_set(:@package, package)
139
- storage.stubs(:storage_name).returns('Storage::SFTP')
140
- storage.stubs(:local_path).returns('/local/path')
141
- storage.stubs(:connection).yields(connection)
142
- end
143
-
144
- it 'should transfer the package files' do
145
- storage.expects(:remote_path_for).in_sequence(s).with(package).
146
- returns('remote/path')
147
- storage.expects(:create_remote_path).in_sequence(s).with(
148
- 'remote/path', connection
149
- )
150
-
151
- storage.expects(:files_to_transfer_for).in_sequence(s).with(package).
152
- multiple_yields(
153
- ['2011.12.31.11.00.02.backup.tar.enc-aa', 'backup.tar.enc-aa'],
154
- ['2011.12.31.11.00.02.backup.tar.enc-ab', 'backup.tar.enc-ab']
155
- )
156
- # first yield
157
- Backup::Logger.expects(:message).in_sequence(s).with(
158
- "Storage::SFTP started transferring " +
159
- "'2011.12.31.11.00.02.backup.tar.enc-aa' to '123.45.678.90'."
160
- )
161
- connection.expects(:upload!).in_sequence(s).with(
162
- File.join('/local/path', '2011.12.31.11.00.02.backup.tar.enc-aa'),
163
- File.join('remote/path', 'backup.tar.enc-aa')
164
- )
165
- # second yield
166
- Backup::Logger.expects(:message).in_sequence(s).with(
167
- "Storage::SFTP started transferring " +
168
- "'2011.12.31.11.00.02.backup.tar.enc-ab' to '123.45.678.90'."
169
- )
170
- connection.expects(:upload!).in_sequence(s).with(
171
- File.join('/local/path', '2011.12.31.11.00.02.backup.tar.enc-ab'),
172
- File.join('remote/path', 'backup.tar.enc-ab')
173
- )
174
-
175
- storage.send(:transfer!)
176
- end
177
- end # describe '#transfer!'
178
-
179
- describe '#remove!' do
180
- let(:package) { mock }
181
- let(:connection) { mock }
182
- let(:s) { sequence '' }
183
-
184
- before do
185
- storage.stubs(:storage_name).returns('Storage::SFTP')
186
- storage.stubs(:connection).yields(connection)
187
- end
188
-
189
- it 'should remove the package files' do
190
- storage.expects(:remote_path_for).in_sequence(s).with(package).
191
- returns('remote/path')
192
-
193
- storage.expects(:transferred_files_for).in_sequence(s).with(package).
194
- multiple_yields(
195
- ['2011.12.31.11.00.02.backup.tar.enc-aa', 'backup.tar.enc-aa'],
196
- ['2011.12.31.11.00.02.backup.tar.enc-ab', 'backup.tar.enc-ab']
197
- )
198
- # first yield
199
- Backup::Logger.expects(:message).in_sequence(s).with(
200
- "Storage::SFTP started removing " +
201
- "'2011.12.31.11.00.02.backup.tar.enc-aa' from '123.45.678.90'."
202
- )
203
- connection.expects(:remove!).in_sequence(s).with(
204
- File.join('remote/path', 'backup.tar.enc-aa')
205
- )
206
- # second yield
207
- Backup::Logger.expects(:message).in_sequence(s).with(
208
- "Storage::SFTP started removing " +
209
- "'2011.12.31.11.00.02.backup.tar.enc-ab' from '123.45.678.90'."
210
- )
211
- connection.expects(:remove!).in_sequence(s).with(
212
- File.join('remote/path', 'backup.tar.enc-ab')
213
- )
214
-
215
- connection.expects(:rmdir!).with('remote/path').in_sequence(s)
216
-
217
- storage.send(:remove!, package)
218
- end
219
- end # describe '#remove!'
220
-
221
- describe '#create_remote_path' do
222
- let(:connection) { mock }
223
- let(:remote_path) { 'backups/folder/another_folder' }
224
- let(:s) { sequence '' }
225
- let(:sftp_response) { stub(:code => 11, :message => nil) }
226
- let(:sftp_status_exception) { Net::SFTP::StatusException.new(sftp_response) }
227
-
228
- context 'while properly creating remote directories one by one' do
229
- it 'should rescue any SFTP::StatusException and continue' do
230
- connection.expects(:mkdir!).in_sequence(s).
231
- with("backups").raises(sftp_status_exception)
232
- connection.expects(:mkdir!).in_sequence(s).
233
- with("backups/folder")
234
- connection.expects(:mkdir!).in_sequence(s).
235
- with("backups/folder/another_folder")
236
-
237
- expect do
238
- storage.send(:create_remote_path, remote_path, connection)
239
- end.not_to raise_error
240
- end
241
- end
242
- end
243
-
244
- end
@@ -1,109 +0,0 @@
1
- # encoding: utf-8
2
-
3
- require File.expand_path('../../spec_helper.rb', __FILE__)
4
-
5
- describe Backup::Syncer::Base do
6
- let(:syncer) { Backup::Syncer::Base.new }
7
-
8
- it 'should include CLI::Helpers' do
9
- Backup::Syncer::Base.
10
- include?(Backup::CLI::Helpers).should be_true
11
- end
12
-
13
- it 'should include Configuration::Helpers' do
14
- Backup::Syncer::Base.
15
- include?(Backup::Configuration::Helpers).should be_true
16
- end
17
-
18
- describe '#initialize' do
19
- after { Backup::Syncer::Base.clear_defaults! }
20
-
21
- it 'should load pre-configured defaults through Base' do
22
- Backup::Syncer::Base.any_instance.expects(:load_defaults!)
23
- syncer
24
- end
25
-
26
- it 'should establish a new array for @directories' do
27
- syncer.directories.should == []
28
- end
29
-
30
- context 'when no pre-configured defaults have been set' do
31
- it 'should set default values' do
32
- syncer.path.should == 'backups'
33
- syncer.mirror.should == false
34
- end
35
- end # context 'when no pre-configured defaults have been set'
36
-
37
- context 'when pre-configured defaults have been set' do
38
- before do
39
- Backup::Syncer::Base.defaults do |s|
40
- s.path = 'some_path'
41
- s.mirror = 'some_mirror'
42
- end
43
- end
44
-
45
- it 'should use pre-configured defaults' do
46
- syncer.path.should == 'some_path'
47
- syncer.mirror.should == 'some_mirror'
48
- end
49
- end # context 'when pre-configured defaults have been set'
50
- end # describe '#initialize'
51
-
52
- describe '#directories' do
53
- before do
54
- syncer.instance_variable_set(
55
- :@directories, ['/some/directory', '/another/directory']
56
- )
57
- end
58
-
59
- context 'when no block is given' do
60
- it 'should return @directories' do
61
- syncer.directories.should ==
62
- ['/some/directory', '/another/directory']
63
- end
64
- end
65
-
66
- context 'when a block is given' do
67
- it 'should evalute the block, allowing #add to add directories' do
68
- syncer.directories do
69
- add '/new/path'
70
- add '/another/new/path'
71
- end
72
- syncer.directories.should == [
73
- '/some/directory',
74
- '/another/directory',
75
- '/new/path',
76
- '/another/new/path'
77
- ]
78
- end
79
- end
80
- end # describe '#directories'
81
-
82
- describe '#add' do
83
- before do
84
- syncer.instance_variable_set(
85
- :@directories, ['/some/directory', '/another/directory']
86
- )
87
- end
88
-
89
- it 'should add the given path to @directories' do
90
- syncer.add '/my/path'
91
- syncer.directories.should ==
92
- ['/some/directory', '/another/directory', '/my/path']
93
- end
94
-
95
- # Note: Each Syncer should handle this as needed.
96
- # For example, expanding these here would break RSync::Pull
97
- it 'should not expand the given paths' do
98
- syncer.add 'relative/path'
99
- syncer.directories.should ==
100
- ['/some/directory', '/another/directory', 'relative/path']
101
- end
102
- end
103
-
104
- describe '#syncer_name' do
105
- it 'should return the class name with the Backup:: namespace removed' do
106
- syncer.send(:syncer_name).should == 'Syncer::Base'
107
- end
108
- end
109
- end
@@ -1,515 +0,0 @@
1
- # encoding: utf-8
2
- require File.expand_path('../../../spec_helper.rb', __FILE__)
3
-
4
- describe 'Backup::Syncer::Cloud::Base' do
5
- let(:syncer) { Backup::Syncer::Cloud::Base.new }
6
- let(:s) { sequence '' }
7
-
8
- it 'should be a subclass of Syncer::Base' do
9
- Backup::Syncer::Cloud::Base.
10
- superclass.should == Backup::Syncer::Base
11
- end
12
-
13
- it 'should establish a class constant for a Mutex' do
14
- Backup::Syncer::Cloud::Base::MUTEX.should be_an_instance_of Mutex
15
- end
16
-
17
- describe '#initialize' do
18
- after { Backup::Syncer::Cloud::Base.clear_defaults! }
19
-
20
- it 'should load pre-configured defaults through Syncer::Base' do
21
- Backup::Syncer::Cloud::Base.any_instance.expects(:load_defaults!)
22
- syncer
23
- end
24
-
25
- context 'when no pre-configured defaults have been set' do
26
- it 'should use default values if none are given' do
27
- syncer.path.should == 'backups'
28
- syncer.mirror.should == false
29
- syncer.concurrency_type.should == false
30
- syncer.concurrency_level.should == 2
31
- end
32
- end # context 'when no pre-configured defaults have been set'
33
-
34
- context 'when pre-configured defaults have been set' do
35
- before do
36
- Backup::Syncer::Cloud::Base.defaults do |cloud|
37
- cloud.concurrency_type = 'default_concurrency_type'
38
- cloud.concurrency_level = 'default_concurrency_level'
39
- end
40
- end
41
-
42
- it 'should use pre-configured defaults' do
43
- syncer.path.should == 'backups'
44
- syncer.mirror.should == false
45
- syncer.concurrency_type.should == 'default_concurrency_type'
46
- syncer.concurrency_level.should == 'default_concurrency_level'
47
- end
48
- end # context 'when pre-configured defaults have been set'
49
- end # describe '#initialize'
50
-
51
- describe '#perform' do
52
- let(:sync_context) { mock }
53
-
54
- before do
55
- syncer.stubs(:repository_object).returns(:a_repository_object)
56
-
57
- Backup::Logger.expects(:message).with(
58
- "Syncer::Cloud::Base started the syncing process:\n" +
59
- "\s\sConcurrency: false Level: 2"
60
- )
61
- Backup::Logger.expects(:message).with(
62
- 'Syncer::Cloud::Base Syncing Complete!'
63
- )
64
- end
65
-
66
- it 'should sync each directory' do
67
- syncer.directories do
68
- add '/dir/one'
69
- add '/dir/two'
70
- end
71
-
72
- Backup::Syncer::Cloud::Base::SyncContext.expects(:new).in_sequence(s).with(
73
- '/dir/one', :a_repository_object, 'backups'
74
- ).returns(sync_context)
75
- sync_context.expects(:sync!).in_sequence(s).with(
76
- false, false, 2
77
- )
78
- Backup::Syncer::Cloud::Base::SyncContext.expects(:new).in_sequence(s).with(
79
- '/dir/two', :a_repository_object, 'backups'
80
- ).returns(sync_context)
81
- sync_context.expects(:sync!).in_sequence(s).with(
82
- false, false, 2
83
- )
84
-
85
- syncer.perform!
86
- end
87
-
88
- it 'should ensure each directory path is expanded with no trailing slash' do
89
- syncer.directories do
90
- add '/dir/one/'
91
- add 'dir/two'
92
- end
93
-
94
- Backup::Syncer::Cloud::Base::SyncContext.expects(:new).with(
95
- '/dir/one', :a_repository_object, 'backups'
96
- ).returns(sync_context)
97
-
98
- Backup::Syncer::Cloud::Base::SyncContext.expects(:new).with(
99
- File.expand_path('dir/two'), :a_repository_object, 'backups'
100
- ).returns(sync_context)
101
-
102
- sync_context.stubs(:sync!)
103
-
104
- syncer.perform!
105
- end
106
- end # describe '#perform'
107
-
108
- describe 'Cloud::Base::SyncContext' do
109
- let(:bucket) { mock }
110
- let(:sync_context) do
111
- Backup::Syncer::Cloud::Base::SyncContext.new(
112
- '/dir/to/sync', bucket, 'backups'
113
- )
114
- end
115
-
116
- describe '#initialize' do
117
- it 'should set variables' do
118
- sync_context.directory.should == '/dir/to/sync'
119
- sync_context.bucket.should == bucket
120
- sync_context.path.should == 'backups'
121
- sync_context.remote_base.should == 'backups/sync'
122
- end
123
- end
124
-
125
- describe '#sync!' do
126
- let(:all_files_array) { mock }
127
-
128
- before do
129
- sync_context.stubs(:all_file_names).returns(all_files_array)
130
- end
131
-
132
- context 'when concurrency_type is set to `false`' do
133
- it 'syncs files without concurrency' do
134
- all_files_array.expects(:each).in_sequence(s).
135
- multiple_yields('foo.file', 'foo_dir/foo.file')
136
-
137
- sync_context.expects(:sync_file).in_sequence(s).
138
- with('foo.file', :mirror)
139
- sync_context.expects(:sync_file).in_sequence(s).
140
- with('foo_dir/foo.file', :mirror)
141
-
142
- sync_context.sync!(:mirror, false, :foo)
143
- end
144
- end
145
-
146
- context 'when concurrency_type is set to `:threads`' do
147
- it 'uses `concurrency_level` number of threads for concurrency' do
148
- Parallel.expects(:each).in_sequence(s).with(
149
- all_files_array, :in_threads => :num_of_threads
150
- ).multiple_yields('foo.file', 'foo_dir/foo.file')
151
-
152
- sync_context.expects(:sync_file).in_sequence(s).
153
- with('foo.file', :mirror)
154
- sync_context.expects(:sync_file).in_sequence(s).
155
- with('foo_dir/foo.file', :mirror)
156
-
157
- sync_context.sync!(:mirror, :threads, :num_of_threads)
158
- end
159
- end
160
-
161
- context 'when concurrency_type is set to `:processes`' do
162
- it 'uses `concurrency_level` number of processes for concurrency' do
163
- Parallel.expects(:each).in_sequence(s).with(
164
- all_files_array, :in_processes => :num_of_processes
165
- ).multiple_yields('foo.file', 'foo_dir/foo.file')
166
-
167
- sync_context.expects(:sync_file).in_sequence(s).
168
- with('foo.file', :mirror)
169
- sync_context.expects(:sync_file).in_sequence(s).
170
- with('foo_dir/foo.file', :mirror)
171
-
172
- sync_context.sync!(:mirror, :processes, :num_of_processes)
173
- end
174
- end
175
-
176
- context 'when concurrency_type setting is invalid' do
177
- it 'should raise an error' do
178
- expect do
179
- sync_context.sync!(:foo, 'unknown type', :foo)
180
- end.to raise_error(
181
- Backup::Errors::Syncer::Cloud::ConfigurationError,
182
- 'Syncer::Cloud::ConfigurationError: ' +
183
- "Unknown concurrency_type setting: \"unknown type\""
184
- )
185
- end
186
- end
187
- end # describe '#sync!'
188
-
189
- describe '#all_file_names' do
190
- let(:local_files_hash) do
191
- { 'file_b' => :foo, 'file_a' => :foo, 'dir_a/file_b' => :foo }
192
- end
193
- let(:remote_files_hash) do
194
- { 'file_c' => :foo, 'file_a' => :foo, 'dir_a/file_a' => :foo }
195
- end
196
- let(:local_remote_union_array) do
197
- ['dir_a/file_a', 'dir_a/file_b', 'file_a', 'file_b', 'file_c']
198
- end
199
-
200
- it 'returns and caches a sorted union of local and remote file names' do
201
- sync_context.expects(:local_files).once.returns(local_files_hash)
202
- sync_context.expects(:remote_files).once.returns(remote_files_hash)
203
-
204
- sync_context.send(:all_file_names).should == local_remote_union_array
205
- sync_context.instance_variable_get(:@all_file_names).
206
- should == local_remote_union_array
207
- sync_context.send(:all_file_names).should == local_remote_union_array
208
- end
209
- end # describe '#all_file_names'
210
-
211
- describe '#local_files' do
212
- let(:local_file_class) { Backup::Syncer::Cloud::Base::LocalFile }
213
- let(:local_hashes_data) { "line1\nline2\nbad\xFFline\nline3" }
214
-
215
- let(:local_file_a) { stub(:relative_path => 'file_a') }
216
- let(:local_file_b) { stub(:relative_path => 'file_b') }
217
- let(:local_file_c) { stub(:relative_path => 'file_c') }
218
- let(:local_files_hash) do
219
- { 'file_a' => local_file_a,
220
- 'file_b' => local_file_b,
221
- 'file_c' => local_file_c }
222
- end
223
-
224
- it 'should return and caches a hash of LocalFile objects' do
225
- sync_context.expects(:local_hashes).once.returns(local_hashes_data)
226
-
227
- local_file_class.expects(:new).once.with('/dir/to/sync', "line1\n").
228
- returns(local_file_a)
229
- local_file_class.expects(:new).once.with('/dir/to/sync', "line2\n").
230
- returns(local_file_b)
231
- local_file_class.expects(:new).once.with('/dir/to/sync', "bad\xFFline\n").
232
- returns(nil)
233
- local_file_class.expects(:new).once.with('/dir/to/sync', "line3").
234
- returns(local_file_c)
235
-
236
- sync_context.send(:local_files).should == local_files_hash
237
- sync_context.instance_variable_get(:@local_files).
238
- should == local_files_hash
239
- sync_context.send(:local_files).should == local_files_hash
240
- end
241
-
242
- # Note: don't use methods that validate encoding
243
- it 'will raise an Exception if String#split is used',
244
- :if => RUBY_VERSION >= '1.9' do
245
- expect do
246
- "line1\nbad\xFFline\nline3".split("\n")
247
- end.to raise_error(ArgumentError, 'invalid byte sequence in UTF-8')
248
- end
249
- end # describe '#local_files'
250
-
251
- describe '#local_hashes' do
252
- it 'should collect file paths and MD5 checksums for @directory' do
253
- Backup::Logger.expects(:message).with(
254
- "\s\sGenerating checksums for '/dir/to/sync'"
255
- )
256
- sync_context.expects(:`).with(
257
- "find '/dir/to/sync' -print0 | xargs -0 openssl md5 2> /dev/null"
258
- ).returns('MD5(tmp/foo)= 0123456789abcdefghijklmnopqrstuv')
259
-
260
- sync_context.send(:local_hashes).should ==
261
- 'MD5(tmp/foo)= 0123456789abcdefghijklmnopqrstuv'
262
- end
263
- end
264
-
265
- describe '#remote_files' do
266
- let(:repository_object) { mock }
267
- let(:repository_files) { mock }
268
- let(:file_objects) { mock }
269
- let(:file_obj_a) { stub(:key => 'file_a') }
270
- let(:file_obj_b) { stub(:key => 'file_b') }
271
- let(:file_obj_c) { stub(:key => 'dir/file_c') }
272
- let(:remote_files_hash) do
273
- { 'file_a' => file_obj_a,
274
- 'file_b' => file_obj_b,
275
- 'dir/file_c' => file_obj_c }
276
- end
277
-
278
- before do
279
- sync_context.instance_variable_set(:@bucket, repository_object)
280
-
281
- repository_object.expects(:files).once.returns(repository_files)
282
- repository_files.expects(:all).once.with(:prefix => 'backups/sync').
283
- returns(file_objects)
284
- file_objects.expects(:each).once.multiple_yields(
285
- file_obj_a, file_obj_b, file_obj_c
286
- )
287
-
288
- # this is to avoid: unexpected invocation: #<Mock>.to_a()
289
- # only 1.9.2 seems affected by this
290
- if RUBY_VERSION == '1.9.2'
291
- file_obj_a.stubs(:to_a)
292
- file_obj_b.stubs(:to_a)
293
- file_obj_c.stubs(:to_a)
294
- end
295
- end
296
-
297
- context 'when it returns and caches a hash of repository file objects' do
298
- it 'should remove the @remote_base from the path for the hash key' do
299
- sync_context.send(:remote_files).should == remote_files_hash
300
- sync_context.instance_variable_get(:@remote_files).
301
- should == remote_files_hash
302
- sync_context.send(:remote_files).should == remote_files_hash
303
- end
304
- end
305
- end # describe '#remote_files'
306
-
307
- describe '#sync_file' do
308
- let(:local_file) do
309
- stub(
310
- :path => '/dir/to/sync/sync.file',
311
- :md5 => '0123456789abcdefghijklmnopqrstuv')
312
- end
313
- let(:remote_file) do
314
- stub(:path => 'backups/sync/sync.file')
315
- end
316
- let(:file) { mock }
317
- let(:repository_object) { mock }
318
- let(:repository_files) { mock }
319
-
320
- before do
321
- sync_context.instance_variable_set(:@bucket, repository_object)
322
- repository_object.stubs(:files).returns(repository_files)
323
- end
324
-
325
- context 'when the requested file to sync exists locally' do
326
- before do
327
- sync_context.stubs(:local_files).returns(
328
- { 'sync.file' => local_file }
329
- )
330
- File.expects(:exist?).with('/dir/to/sync/sync.file').returns(true)
331
- end
332
-
333
- context 'when the MD5 checksum matches the remote file' do
334
- before do
335
- remote_file.stubs(:etag).returns('0123456789abcdefghijklmnopqrstuv')
336
- sync_context.stubs(:remote_files).returns(
337
- { 'sync.file' => remote_file }
338
- )
339
- end
340
-
341
- it 'should skip the file' do
342
- File.expects(:open).never
343
- Backup::Syncer::Cloud::Base::MUTEX.expects(:synchronize).yields
344
- Backup::Logger.expects(:message).with(
345
- "\s\s[skipping] 'backups/sync/sync.file'"
346
- )
347
-
348
- sync_context.send(:sync_file, 'sync.file', :foo)
349
- end
350
- end
351
-
352
- context 'when the MD5 checksum does not match the remote file' do
353
- before do
354
- remote_file.stubs(:etag).returns('vutsrqponmlkjihgfedcba9876543210')
355
- sync_context.stubs(:remote_files).returns(
356
- { 'sync.file' => remote_file }
357
- )
358
- end
359
-
360
- it 'should upload the file' do
361
- Backup::Syncer::Cloud::Base::MUTEX.expects(:synchronize).yields
362
- Backup::Logger.expects(:message).with(
363
- "\s\s[transferring] 'backups/sync/sync.file'"
364
- )
365
-
366
- File.expects(:open).with('/dir/to/sync/sync.file', 'r').yields(file)
367
- repository_files.expects(:create).with(
368
- :key => 'backups/sync/sync.file',
369
- :body => file
370
- )
371
-
372
- sync_context.send(:sync_file, 'sync.file', :foo)
373
- end
374
- end
375
-
376
- context 'when the requested file does not exist on the remote' do
377
- before do
378
- sync_context.stubs(:remote_files).returns({})
379
- end
380
-
381
- it 'should upload the file' do
382
- Backup::Syncer::Cloud::Base::MUTEX.expects(:synchronize).yields
383
- Backup::Logger.expects(:message).with(
384
- "\s\s[transferring] 'backups/sync/sync.file'"
385
- )
386
-
387
- File.expects(:open).with('/dir/to/sync/sync.file', 'r').yields(file)
388
- repository_files.expects(:create).with(
389
- :key => 'backups/sync/sync.file',
390
- :body => file
391
- )
392
-
393
- sync_context.send(:sync_file, 'sync.file', :foo)
394
- end
395
- end
396
- end
397
-
398
- context 'when the requested file does not exist locally' do
399
- before do
400
- sync_context.stubs(:remote_files).returns(
401
- { 'sync.file' => remote_file }
402
- )
403
- sync_context.stubs(:local_files).returns({})
404
- end
405
-
406
- context 'when the `mirror` option is set to true' do
407
- it 'should remove the file from the remote' do
408
- Backup::Syncer::Cloud::Base::MUTEX.expects(:synchronize).yields
409
- Backup::Logger.expects(:message).with(
410
- "\s\s[removing] 'backups/sync/sync.file'"
411
- )
412
-
413
- remote_file.expects(:destroy)
414
-
415
- sync_context.send(:sync_file, 'sync.file', true)
416
- end
417
- end
418
-
419
- context 'when the `mirror` option is set to false' do
420
- it 'should leave the file on the remote' do
421
- Backup::Syncer::Cloud::Base::MUTEX.expects(:synchronize).yields
422
- Backup::Logger.expects(:message).with(
423
- "\s\s[leaving] 'backups/sync/sync.file'"
424
- )
425
-
426
- remote_file.expects(:destroy).never
427
-
428
- sync_context.send(:sync_file, 'sync.file', false)
429
- end
430
- end
431
- end
432
- end # describe '#sync_file'
433
- end # describe 'Cloud::Base::SyncContext'
434
-
435
- describe 'Cloud::Base::LocalFile' do
436
- let(:local_file_class) { Backup::Syncer::Cloud::Base::LocalFile }
437
-
438
- describe '#new' do
439
- describe 'wrapping #initialize and using #sanitize to validate objects' do
440
- context 'when the path is valid UTF-8' do
441
- let(:local_file) do
442
- local_file_class.new(
443
- 'foo',
444
- 'MD5(foo)= 0123456789abcdefghijklmnopqrstuv'
445
- )
446
- end
447
-
448
- it 'should return the new object' do
449
- Backup::Logger.expects(:warn).never
450
-
451
- local_file.should be_an_instance_of local_file_class
452
- end
453
- end
454
-
455
- context 'when the path contains invalid UTF-8' do
456
- let(:local_file) do
457
- local_file_class.new(
458
- "/bad/pa\xFFth",
459
- "MD5(/bad/pa\xFFth/to/file)= 0123456789abcdefghijklmnopqrstuv"
460
- )
461
- end
462
- it 'should return nil and log a warning' do
463
- Backup::Logger.expects(:warn).with(
464
- "\s\s[skipping] /bad/pa\xEF\xBF\xBDth/to/file\n" +
465
- "\s\sPath Contains Invalid UTF-8 byte sequences"
466
- )
467
-
468
- local_file.should be_nil
469
- end
470
- end
471
- end
472
- end # describe '#new'
473
-
474
- describe '#initialize' do
475
- let(:local_file) do
476
- local_file_class.new(:directory, :line)
477
- end
478
-
479
- before do
480
- local_file_class.any_instance.expects(:sanitize).with(:directory).
481
- returns('/dir/to/sync')
482
- local_file_class.any_instance.expects(:sanitize).with(:line).
483
- returns("MD5(/dir/to/sync/subdir/sync.file)= 0123456789abcdefghijklmnopqrstuv\n")
484
- end
485
-
486
- it 'should determine @path, @relative_path and @md5' do
487
- local_file.path.should == '/dir/to/sync/subdir/sync.file'
488
- local_file.relative_path.should == 'subdir/sync.file'
489
- local_file.md5.should == '0123456789abcdefghijklmnopqrstuv'
490
- end
491
-
492
- it 'should return nil if the object is invalid' do
493
- local_file_class.any_instance.expects(:invalid?).returns(true)
494
- Backup::Logger.expects(:warn)
495
- local_file.should be_nil
496
- end
497
- end # describe '#initialize'
498
-
499
- describe '#sanitize' do
500
- let(:local_file) do
501
- local_file_class.new('foo', 'MD5(foo)= 0123456789abcdefghijklmnopqrstuv')
502
- end
503
-
504
- it 'should replace any invalid UTF-8 characters' do
505
- local_file.send(:sanitize, "/path/to/d\xFFir/subdir/sync\xFFfile").
506
- should == "/path/to/d\xEF\xBF\xBDir/subdir/sync\xEF\xBF\xBDfile"
507
- end
508
-
509
- it 'should flag the LocalFile object as invalid' do
510
- local_file.send(:sanitize, "/path/to/d\xFFir/subdir/sync\xFFfile")
511
- local_file.invalid?.should be_true
512
- end
513
- end # describe '#sanitize'
514
- end # describe 'Cloud::Base::LocalFile'
515
- end