cm-backup 1.0.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (133) hide show
  1. checksums.yaml +7 -0
  2. data/README.md +20 -0
  3. data/bin/backup +5 -0
  4. data/lib/backup.rb +144 -0
  5. data/lib/backup/archive.rb +170 -0
  6. data/lib/backup/binder.rb +22 -0
  7. data/lib/backup/cleaner.rb +116 -0
  8. data/lib/backup/cli.rb +374 -0
  9. data/lib/backup/cloud_io/base.rb +41 -0
  10. data/lib/backup/cloud_io/cloud_files.rb +298 -0
  11. data/lib/backup/cloud_io/s3.rb +260 -0
  12. data/lib/backup/compressor/base.rb +35 -0
  13. data/lib/backup/compressor/bzip2.rb +39 -0
  14. data/lib/backup/compressor/custom.rb +53 -0
  15. data/lib/backup/compressor/gzip.rb +74 -0
  16. data/lib/backup/config.rb +119 -0
  17. data/lib/backup/config/dsl.rb +103 -0
  18. data/lib/backup/config/helpers.rb +143 -0
  19. data/lib/backup/database/base.rb +85 -0
  20. data/lib/backup/database/mongodb.rb +187 -0
  21. data/lib/backup/database/mysql.rb +192 -0
  22. data/lib/backup/database/openldap.rb +95 -0
  23. data/lib/backup/database/postgresql.rb +133 -0
  24. data/lib/backup/database/redis.rb +179 -0
  25. data/lib/backup/database/riak.rb +82 -0
  26. data/lib/backup/database/sqlite.rb +57 -0
  27. data/lib/backup/encryptor/base.rb +29 -0
  28. data/lib/backup/encryptor/gpg.rb +747 -0
  29. data/lib/backup/encryptor/open_ssl.rb +77 -0
  30. data/lib/backup/errors.rb +58 -0
  31. data/lib/backup/logger.rb +199 -0
  32. data/lib/backup/logger/console.rb +51 -0
  33. data/lib/backup/logger/fog_adapter.rb +29 -0
  34. data/lib/backup/logger/logfile.rb +133 -0
  35. data/lib/backup/logger/syslog.rb +116 -0
  36. data/lib/backup/model.rb +479 -0
  37. data/lib/backup/notifier/base.rb +128 -0
  38. data/lib/backup/notifier/campfire.rb +63 -0
  39. data/lib/backup/notifier/command.rb +102 -0
  40. data/lib/backup/notifier/datadog.rb +107 -0
  41. data/lib/backup/notifier/flowdock.rb +103 -0
  42. data/lib/backup/notifier/hipchat.rb +118 -0
  43. data/lib/backup/notifier/http_post.rb +117 -0
  44. data/lib/backup/notifier/mail.rb +249 -0
  45. data/lib/backup/notifier/nagios.rb +69 -0
  46. data/lib/backup/notifier/pagerduty.rb +81 -0
  47. data/lib/backup/notifier/prowl.rb +68 -0
  48. data/lib/backup/notifier/pushover.rb +74 -0
  49. data/lib/backup/notifier/ses.rb +105 -0
  50. data/lib/backup/notifier/slack.rb +148 -0
  51. data/lib/backup/notifier/twitter.rb +58 -0
  52. data/lib/backup/notifier/zabbix.rb +63 -0
  53. data/lib/backup/package.rb +55 -0
  54. data/lib/backup/packager.rb +107 -0
  55. data/lib/backup/pipeline.rb +124 -0
  56. data/lib/backup/splitter.rb +76 -0
  57. data/lib/backup/storage/base.rb +69 -0
  58. data/lib/backup/storage/cloud_files.rb +158 -0
  59. data/lib/backup/storage/cycler.rb +75 -0
  60. data/lib/backup/storage/dropbox.rb +212 -0
  61. data/lib/backup/storage/ftp.rb +112 -0
  62. data/lib/backup/storage/local.rb +64 -0
  63. data/lib/backup/storage/qiniu.rb +65 -0
  64. data/lib/backup/storage/rsync.rb +248 -0
  65. data/lib/backup/storage/s3.rb +156 -0
  66. data/lib/backup/storage/scp.rb +67 -0
  67. data/lib/backup/storage/sftp.rb +82 -0
  68. data/lib/backup/syncer/base.rb +70 -0
  69. data/lib/backup/syncer/cloud/base.rb +179 -0
  70. data/lib/backup/syncer/cloud/cloud_files.rb +83 -0
  71. data/lib/backup/syncer/cloud/local_file.rb +100 -0
  72. data/lib/backup/syncer/cloud/s3.rb +110 -0
  73. data/lib/backup/syncer/rsync/base.rb +54 -0
  74. data/lib/backup/syncer/rsync/local.rb +31 -0
  75. data/lib/backup/syncer/rsync/pull.rb +51 -0
  76. data/lib/backup/syncer/rsync/push.rb +205 -0
  77. data/lib/backup/template.rb +46 -0
  78. data/lib/backup/utilities.rb +224 -0
  79. data/lib/backup/version.rb +5 -0
  80. data/templates/cli/archive +28 -0
  81. data/templates/cli/compressor/bzip2 +4 -0
  82. data/templates/cli/compressor/custom +7 -0
  83. data/templates/cli/compressor/gzip +4 -0
  84. data/templates/cli/config +123 -0
  85. data/templates/cli/databases/mongodb +15 -0
  86. data/templates/cli/databases/mysql +18 -0
  87. data/templates/cli/databases/openldap +24 -0
  88. data/templates/cli/databases/postgresql +16 -0
  89. data/templates/cli/databases/redis +16 -0
  90. data/templates/cli/databases/riak +17 -0
  91. data/templates/cli/databases/sqlite +11 -0
  92. data/templates/cli/encryptor/gpg +27 -0
  93. data/templates/cli/encryptor/openssl +9 -0
  94. data/templates/cli/model +26 -0
  95. data/templates/cli/notifier/zabbix +15 -0
  96. data/templates/cli/notifiers/campfire +12 -0
  97. data/templates/cli/notifiers/command +32 -0
  98. data/templates/cli/notifiers/datadog +57 -0
  99. data/templates/cli/notifiers/flowdock +16 -0
  100. data/templates/cli/notifiers/hipchat +16 -0
  101. data/templates/cli/notifiers/http_post +32 -0
  102. data/templates/cli/notifiers/mail +24 -0
  103. data/templates/cli/notifiers/nagios +13 -0
  104. data/templates/cli/notifiers/pagerduty +12 -0
  105. data/templates/cli/notifiers/prowl +11 -0
  106. data/templates/cli/notifiers/pushover +11 -0
  107. data/templates/cli/notifiers/ses +15 -0
  108. data/templates/cli/notifiers/slack +22 -0
  109. data/templates/cli/notifiers/twitter +13 -0
  110. data/templates/cli/splitter +7 -0
  111. data/templates/cli/storages/cloud_files +11 -0
  112. data/templates/cli/storages/dropbox +20 -0
  113. data/templates/cli/storages/ftp +13 -0
  114. data/templates/cli/storages/local +8 -0
  115. data/templates/cli/storages/qiniu +12 -0
  116. data/templates/cli/storages/rsync +17 -0
  117. data/templates/cli/storages/s3 +16 -0
  118. data/templates/cli/storages/scp +15 -0
  119. data/templates/cli/storages/sftp +15 -0
  120. data/templates/cli/syncers/cloud_files +22 -0
  121. data/templates/cli/syncers/rsync_local +20 -0
  122. data/templates/cli/syncers/rsync_pull +28 -0
  123. data/templates/cli/syncers/rsync_push +28 -0
  124. data/templates/cli/syncers/s3 +27 -0
  125. data/templates/general/links +3 -0
  126. data/templates/general/version.erb +2 -0
  127. data/templates/notifier/mail/failure.erb +16 -0
  128. data/templates/notifier/mail/success.erb +16 -0
  129. data/templates/notifier/mail/warning.erb +16 -0
  130. data/templates/storage/dropbox/authorization_url.erb +6 -0
  131. data/templates/storage/dropbox/authorized.erb +4 -0
  132. data/templates/storage/dropbox/cache_file_written.erb +10 -0
  133. metadata +1077 -0
@@ -0,0 +1,374 @@
1
+ # encoding: utf-8
2
+
3
+ ##
4
+ # Build the Backup Command Line Interface using Thor
5
+ module Backup
6
+ class CLI < Thor
7
+ class Error < Backup::Error; end
8
+ class FatalError < Backup::FatalError; end
9
+
10
+ ##
11
+ # [Perform]
12
+ #
13
+ # The only required option is the --trigger [-t].
14
+ # If --config-file, --data-path, --tmp-path or --log-path
15
+ # aren't specified they will fallback to defaults.
16
+ # If --root-path is given, it will be used as the base path for our defaults,
17
+ # as well as the base path for any option specified as a relative path.
18
+ # Any option given as an absolute path will be used "as-is".
19
+ #
20
+ # This command will exit with one of the following status codes:
21
+ #
22
+ # 0: All triggers were successful and no warnings were issued.
23
+ # 1: All triggers were successful, but some had warnings.
24
+ # 2: All triggers were processed, but some failed.
25
+ # 3: A fatal error caused Backup to exit.
26
+ # Some triggers may not have been processed.
27
+ #
28
+ # If the --check option is given, `backup check` will be run
29
+ # and no triggers will be performed.
30
+ desc 'perform', "Performs the backup for the specified trigger(s)."
31
+
32
+ long_desc <<-EOS.gsub(/^ +/, '')
33
+ Performs the backup for the specified trigger(s).
34
+
35
+ You may perform multiple backups by providing multiple triggers,
36
+ separated by commas. Each will run in the order specified.
37
+
38
+ $ backup perform --triggers backup1,backup2,backup3,backup4
39
+
40
+ --root-path may be an absolute path or relative to the current directory.
41
+
42
+ To use the current directory, use: `--root-path .`
43
+
44
+ Relative paths given for --config-file, --data-path, --tmp-path,
45
+ and --log-path will be relative to --root-path.
46
+
47
+ Console log output may be forced using --no-quiet.
48
+
49
+ Logging to file or syslog may be disabled using --no-logfile or --no-syslog
50
+ respectively. This will override logging options set in `config.rb`.
51
+ EOS
52
+
53
+ method_option :trigger,
54
+ :aliases => ['-t', '--triggers'],
55
+ :required => true,
56
+ :type => :string,
57
+ :desc => "Triggers to perform. e.g. 'trigger_a,trigger_b'"
58
+
59
+ method_option :config_file,
60
+ :aliases => '-c',
61
+ :type => :string,
62
+ :default => '',
63
+ :desc => 'Path to your config.rb file.'
64
+
65
+ method_option :root_path,
66
+ :aliases => '-r',
67
+ :type => :string,
68
+ :default => '',
69
+ :desc => 'Root path to base all relative path on.'
70
+
71
+ method_option :data_path,
72
+ :aliases => '-d',
73
+ :type => :string,
74
+ :default => '',
75
+ :desc => 'Path to store storage cycling data.'
76
+
77
+ method_option :log_path,
78
+ :aliases => '-l',
79
+ :type => :string,
80
+ :default => '',
81
+ :desc => "Path to store Backup's log file."
82
+
83
+ method_option :tmp_path,
84
+ :type => :string,
85
+ :default => '',
86
+ :desc => 'Path to store temporary data during the backup.'
87
+
88
+ # Note that :quiet, :syslog and :logfile are specified as :string types,
89
+ # so the --no-<option> usage will set the value to nil instead of false.
90
+ method_option :quiet,
91
+ :aliases => '-q',
92
+ :type => :string,
93
+ :default => false,
94
+ :banner => '',
95
+ :desc => 'Disable console log output.'
96
+
97
+ method_option :syslog,
98
+ :type => :string,
99
+ :default => false,
100
+ :banner => '',
101
+ :desc => 'Enable logging to syslog.'
102
+
103
+ method_option :logfile,
104
+ :type => :string,
105
+ :default => true,
106
+ :banner => '',
107
+ :desc => "Enable Backup's log file."
108
+
109
+ method_option :check,
110
+ :type => :boolean,
111
+ :default => false,
112
+ :desc => 'Check configuration for errors or warnings.'
113
+
114
+ def perform
115
+ check if options[:check] # this will exit()
116
+
117
+ models = nil
118
+ begin
119
+ # Set logger options
120
+ opts = options
121
+ Logger.configure do
122
+ console.quiet = opts[:quiet]
123
+ logfile.enabled = opts[:logfile]
124
+ logfile.log_path = opts[:log_path]
125
+ syslog.enabled = opts[:syslog]
126
+ end
127
+
128
+ # Load the user's +config.rb+ file and all their Models
129
+ Config.load(options)
130
+
131
+ # Identify all Models to be run for the given +triggers+.
132
+ triggers = options[:trigger].split(',').map(&:strip)
133
+ models = triggers.map {|trigger|
134
+ Model.find_by_trigger(trigger)
135
+ }.flatten.uniq
136
+
137
+ raise Error, "No Models found for trigger(s) " +
138
+ "'#{ triggers.join(',') }'." if models.empty?
139
+
140
+ # Finalize Logger and begin real-time logging.
141
+ Logger.start!
142
+
143
+ rescue Exception => err
144
+ Logger.error Error.wrap(err)
145
+ unless Helpers.is_backup_error? err
146
+ Logger.error err.backtrace.join("\n")
147
+ end
148
+ # Logger configuration will be ignored
149
+ # and messages will be output to the console only.
150
+ Logger.abort!
151
+ exit(3)
152
+ end
153
+
154
+ until models.empty?
155
+ model = models.shift
156
+ model.perform!
157
+
158
+ case model.exit_status
159
+ when 1
160
+ warnings = true
161
+ when 2
162
+ errors = true
163
+ unless models.empty?
164
+ Logger.info Error.new(<<-EOS)
165
+ Backup will now continue...
166
+ The following triggers will now be processed:
167
+ (#{ models.map {|m| m.trigger }.join(', ') })
168
+ EOS
169
+ end
170
+ when 3
171
+ fatal = true
172
+ unless models.empty?
173
+ Logger.error FatalError.new(<<-EOS)
174
+ Backup will now exit.
175
+ The following triggers will not be processed:
176
+ (#{ models.map {|m| m.trigger }.join(', ') })
177
+ EOS
178
+ end
179
+ end
180
+
181
+ model.notifiers.each(&:perform!)
182
+ exit(3) if fatal
183
+ Logger.clear!
184
+ end
185
+
186
+ exit(errors ? 2 : 1) if errors || warnings
187
+ end
188
+
189
+ ##
190
+ # [Check]
191
+ #
192
+ # Loads the user's `config.rb` (and all Model files) and reports any Errors
193
+ # or Warnings. This is primarily for checking for syntax errors, missing
194
+ # dependencies and deprecation warnings.
195
+ #
196
+ # This may also be invoked using the `--check` option to `backup perform`.
197
+ #
198
+ # This command only requires `Config.config_file` to be correct.
199
+ # All other Config paths are irrelevant.
200
+ #
201
+ # All output will be sent to the console only.
202
+ # Logger options will be ignored.
203
+ #
204
+ # If successful, this method with exit(0).
205
+ # If there are Errors or Warnings, it will exit(1).
206
+ desc 'check', 'Check for configuration errors or warnings'
207
+
208
+ long_desc <<-EOS.gsub(/^ +/, '')
209
+ Loads your 'config.rb' file and all models and reports any
210
+ errors or warnings with your configuration, including missing
211
+ dependencies and the use of any deprecated settings.
212
+ EOS
213
+
214
+ method_option :config_file,
215
+ :aliases => '-c',
216
+ :type => :string,
217
+ :default => '',
218
+ :desc => "Path to your config.rb file."
219
+
220
+ def check
221
+ begin
222
+ Config.load(options)
223
+ rescue Exception => err
224
+ Logger.error Error.wrap(err)
225
+ unless Helpers.is_backup_error? err
226
+ Logger.error err.backtrace.join("\n")
227
+ end
228
+ end
229
+
230
+ if Logger.has_warnings? || Logger.has_errors?
231
+ Logger.error 'Configuration Check Failed.'
232
+ exit_code = 1
233
+ else
234
+ Logger.info 'Configuration Check Succeeded.'
235
+ exit_code = 0
236
+ end
237
+
238
+ Logger.abort!
239
+ exit(exit_code)
240
+ end
241
+
242
+ ##
243
+ # [Generate:Model]
244
+ # Generates a model configuration file based on the arguments passed in.
245
+ # For example:
246
+ # $ backup generate:model --trigger my_backup --databases='mongodb'
247
+ # will generate a pre-populated model with a base MongoDB setup
248
+ desc 'generate:model', "Generates a Backup model file."
249
+
250
+ long_desc <<-EOS.gsub(/^ +/, '')
251
+ Generates a Backup model file.
252
+
253
+ If your configuration file is not in the default location at
254
+ #{ Config.config_file }
255
+ you must specify it's location using '--config-file'.
256
+ If no configuration file exists at this location, one will be created.
257
+
258
+ The model file will be created as '<config_path>/models/<trigger>.rb'
259
+ Your model file will be created in a 'models/' sub-directory
260
+ where your config file is located. The default location would be:
261
+ #{ Config.root_path }/models/<trigger>.rb
262
+ EOS
263
+
264
+ method_option :trigger,
265
+ :aliases => '-t',
266
+ :required => true,
267
+ :type => :string,
268
+ :desc => 'Trigger name for the Backup model'
269
+
270
+ method_option :config_file,
271
+ :type => :string,
272
+ :desc => 'Path to your Backup configuration file'
273
+
274
+ # options with their available values
275
+ %w{ databases storages syncers encryptor compressor notifiers }.each do |name|
276
+ path = File.join(Backup::TEMPLATE_PATH, 'cli', name)
277
+ opts = Dir[path + '/*'].sort.map {|p| File.basename(p) }.join(', ')
278
+ method_option name, :type => :string, :desc => "(#{ opts })"
279
+ end
280
+
281
+ method_option :archives,
282
+ :type => :boolean,
283
+ :desc => 'Model will include tar archives.'
284
+
285
+ method_option :splitter,
286
+ :type => :boolean,
287
+ :default => false,
288
+ :desc => 'Add Splitter to the model'
289
+
290
+ define_method 'generate:model' do
291
+ opts = options.merge(:trigger => options[:trigger].gsub(/\W/, '_'))
292
+ config_file = opts[:config_file] ?
293
+ File.expand_path(opts.delete(:config_file)) : Config.config_file
294
+ models_path = File.join(File.dirname(config_file), 'models')
295
+ model_file = File.join(models_path, "#{ opts[:trigger] }.rb")
296
+
297
+ unless File.exist?(config_file)
298
+ invoke 'generate:config', [], :config_file => config_file
299
+ end
300
+
301
+ FileUtils.mkdir_p(models_path)
302
+ if Helpers.overwrite?(model_file)
303
+ File.open(model_file, 'w') do |file|
304
+ file.write(Backup::Template.new({:options => opts}).result('cli/model'))
305
+ end
306
+ puts "Generated model file: '#{ model_file }'."
307
+ end
308
+ end
309
+
310
+ ##
311
+ # [Generate:Config]
312
+ # Generates the main configuration file
313
+ desc 'generate:config', 'Generates the main Backup configuration file'
314
+
315
+ long_desc <<-EOS.gsub(/^ +/, '')
316
+ Path to the Backup configuration file to generate.
317
+
318
+ Defaults to:
319
+
320
+ #{ Config.config_file }
321
+ EOS
322
+
323
+ method_option :config_file,
324
+ :type => :string,
325
+ :desc => 'Path to the Backup configuration file to generate.'
326
+
327
+ define_method 'generate:config' do
328
+ config_file = options[:config_file] ?
329
+ File.expand_path(options[:config_file]) : Config.config_file
330
+
331
+ FileUtils.mkdir_p(File.dirname(config_file))
332
+ if Helpers.overwrite?(config_file)
333
+ File.open(config_file, 'w') do |file|
334
+ file.write(Backup::Template.new.result('cli/config'))
335
+ end
336
+ puts "Generated configuration file: '#{ config_file }'."
337
+ end
338
+ end
339
+
340
+ ##
341
+ # [Version]
342
+ # Returns the current version of the Backup gem
343
+ map '-v' => :version
344
+ desc 'version', 'Display installed Backup version'
345
+ def version
346
+ puts "Backup #{ Backup::VERSION }"
347
+ end
348
+
349
+ # This is to avoid Thor's warnings when stubbing methods on the Thor class.
350
+ module Helpers
351
+ class << self
352
+
353
+ def overwrite?(path)
354
+ return true unless File.exist?(path)
355
+
356
+ $stderr.print "A file already exists at '#{ path }'.\n" +
357
+ "Do you want to overwrite? [y/n] "
358
+ /^[Yy]/ =~ $stdin.gets
359
+ end
360
+
361
+ def exec!(cmd)
362
+ puts "Launching: #{ cmd }"
363
+ exec(cmd)
364
+ end
365
+
366
+ def is_backup_error?(error)
367
+ error.class.ancestors.include? Backup::Error
368
+ end
369
+
370
+ end
371
+ end
372
+
373
+ end
374
+ end
@@ -0,0 +1,41 @@
1
+ # encoding: utf-8
2
+
3
+ module Backup
4
+ module CloudIO
5
+ class Error < Backup::Error; end
6
+ class FileSizeError < Backup::Error; end
7
+
8
+ class Base
9
+ attr_reader :max_retries, :retry_waitsec
10
+
11
+ def initialize(options = {})
12
+ @max_retries = options[:max_retries]
13
+ @retry_waitsec = options[:retry_waitsec]
14
+ end
15
+
16
+ private
17
+
18
+ def with_retries(operation)
19
+ retries = 0
20
+ begin
21
+ yield
22
+ rescue => err
23
+ retries += 1
24
+ raise Error.wrap(err, <<-EOS) if retries > max_retries
25
+ Max Retries (#{ max_retries }) Exceeded!
26
+ Operation: #{ operation }
27
+ Be sure to check the log messages for each retry attempt.
28
+ EOS
29
+
30
+ Logger.info Error.wrap(err, <<-EOS)
31
+ Retry ##{ retries } of #{ max_retries }
32
+ Operation: #{ operation }
33
+ EOS
34
+ sleep(retry_waitsec)
35
+ retry
36
+ end
37
+ end
38
+
39
+ end
40
+ end
41
+ end
@@ -0,0 +1,298 @@
1
+ # encoding: utf-8
2
+ require 'backup/cloud_io/base'
3
+ require 'fog'
4
+ require 'digest/md5'
5
+
6
+ module Backup
7
+ module CloudIO
8
+ class CloudFiles < Base
9
+ class Error < Backup::Error; end
10
+
11
+ MAX_FILE_SIZE = 1024**3 * 5 # 5 GiB
12
+ MAX_SLO_SIZE = 1024**3 * 5000 # 1000 segments @ 5 GiB
13
+ SEGMENT_BUFFER = 1024**2 # 1 MiB
14
+
15
+ attr_reader :username, :api_key, :auth_url, :region, :servicenet,
16
+ :container, :segments_container, :segment_size, :days_to_keep,
17
+ :fog_options
18
+
19
+ def initialize(options = {})
20
+ super
21
+
22
+ @username = options[:username]
23
+ @api_key = options[:api_key]
24
+ @auth_url = options[:auth_url]
25
+ @region = options[:region]
26
+ @servicenet = options[:servicenet]
27
+ @container = options[:container]
28
+ @segments_container = options[:segments_container]
29
+ @segment_size = options[:segment_size]
30
+ @days_to_keep = options[:days_to_keep]
31
+ @fog_options = options[:fog_options]
32
+ end
33
+
34
+ # The Syncer may call this method in multiple threads,
35
+ # but #objects is always called before this occurs.
36
+ def upload(src, dest)
37
+ create_containers
38
+
39
+ file_size = File.size(src)
40
+ segment_bytes = segment_size * 1024**2
41
+ if segment_bytes > 0 && file_size > segment_bytes
42
+ raise FileSizeError, <<-EOS if file_size > MAX_SLO_SIZE
43
+ File Too Large
44
+ File: #{ src }
45
+ Size: #{ file_size }
46
+ Max SLO Size is #{ MAX_SLO_SIZE } (5 GiB * 1000 segments)
47
+ EOS
48
+
49
+ segment_bytes = adjusted_segment_bytes(segment_bytes, file_size)
50
+ segments = upload_segments(src, dest, segment_bytes, file_size)
51
+ upload_manifest(dest, segments)
52
+ else
53
+ raise FileSizeError, <<-EOS if file_size > MAX_FILE_SIZE
54
+ File Too Large
55
+ File: #{ src }
56
+ Size: #{ file_size }
57
+ Max File Size is #{ MAX_FILE_SIZE } (5 GiB)
58
+ EOS
59
+
60
+ put_object(src, dest)
61
+ end
62
+ end
63
+
64
+ # Returns all objects in the container with the given prefix.
65
+ #
66
+ # - #get_container returns a max of 10000 objects per request.
67
+ # - Returns objects sorted using a sqlite binary collating function.
68
+ # - If marker is given, only objects after the marker are in the response.
69
+ def objects(prefix)
70
+ objects = []
71
+ resp = nil
72
+ prefix = prefix.chomp('/')
73
+ opts = { :prefix => prefix + '/' }
74
+
75
+ create_containers
76
+
77
+ while resp.nil? || resp.body.count == 10000
78
+ opts.merge!(:marker => objects.last.name) unless objects.empty?
79
+ with_retries("GET '#{ container }/#{ prefix }/*'") do
80
+ resp = connection.get_container(container, opts)
81
+ end
82
+ resp.body.each do |obj_data|
83
+ objects << Object.new(self, obj_data)
84
+ end
85
+ end
86
+
87
+ objects
88
+ end
89
+
90
+ # Used by Object to fetch metadata if needed.
91
+ def head_object(object)
92
+ resp = nil
93
+ with_retries("HEAD '#{ container }/#{ object.name }'") do
94
+ resp = connection.head_object(container, object.name)
95
+ end
96
+ resp
97
+ end
98
+
99
+ # Delete non-SLO object(s) from the container.
100
+ #
101
+ # - Called by the Storage (with objects) and the Syncer (with names)
102
+ # - Deletes 10,000 objects per request.
103
+ # - Missing objects will be ignored.
104
+ def delete(objects_or_names)
105
+ names = Array(objects_or_names).dup
106
+ names.map!(&:name) if names.first.is_a?(Object)
107
+
108
+ until names.empty?
109
+ _names = names.slice!(0, 10000)
110
+ with_retries('DELETE Multiple Objects') do
111
+ resp = connection.delete_multiple_objects(container, _names)
112
+ resp_status = resp.body['Response Status']
113
+ raise Error, <<-EOS unless resp_status == '200 OK'
114
+ #{ resp_status }
115
+ The server returned the following:
116
+ #{ resp.body.inspect }
117
+ EOS
118
+ end
119
+ end
120
+ end
121
+
122
+ # Delete an SLO object(s) from the container.
123
+ #
124
+ # - Used only by the Storage. The Syncer cannot use SLOs.
125
+ # - Removes the SLO manifest object and all associated segments.
126
+ # - Missing segments will be ignored.
127
+ def delete_slo(objects)
128
+ Array(objects).each do |object|
129
+ with_retries("DELETE SLO Manifest '#{ container }/#{ object.name }'") do
130
+ resp = connection.delete_static_large_object(container, object.name)
131
+ resp_status = resp.body['Response Status']
132
+ raise Error, <<-EOS unless resp_status == '200 OK'
133
+ #{ resp_status }
134
+ The server returned the following:
135
+ #{ resp.body.inspect }
136
+ EOS
137
+ end
138
+ end
139
+ end
140
+
141
+ private
142
+
143
+ def connection
144
+ @connection ||= Fog::Storage.new({
145
+ :provider => 'Rackspace',
146
+ :rackspace_username => username,
147
+ :rackspace_api_key => api_key,
148
+ :rackspace_auth_url => auth_url,
149
+ :rackspace_region => region,
150
+ :rackspace_servicenet => servicenet
151
+ }.merge(fog_options || {}))
152
+ end
153
+
154
+ def create_containers
155
+ return if @containers_created
156
+ @containers_created = true
157
+
158
+ with_retries('Create Containers') do
159
+ connection.put_container(container)
160
+ connection.put_container(segments_container) if segments_container
161
+ end
162
+ end
163
+
164
+ def put_object(src, dest)
165
+ opts = headers.merge('ETag' => Digest::MD5.file(src).hexdigest)
166
+ with_retries("PUT '#{ container }/#{ dest }'") do
167
+ File.open(src, 'r') do |file|
168
+ connection.put_object(container, dest, file, opts)
169
+ end
170
+ end
171
+ end
172
+
173
+ # Each segment is uploaded using chunked transfer encoding using
174
+ # SEGMENT_BUFFER, and each segment's MD5 is sent to verify the transfer.
175
+ # Each segment's MD5 and byte_size will also be verified when the
176
+ # SLO manifest object is uploaded.
177
+ def upload_segments(src, dest, segment_bytes, file_size)
178
+ total_segments = (file_size / segment_bytes.to_f).ceil
179
+ progress = (0.1..0.9).step(0.1).map {|n| (total_segments * n).floor }
180
+ Logger.info "\s\sUploading #{ total_segments } SLO Segments..."
181
+
182
+ segments = []
183
+ File.open(src, 'r') do |file|
184
+ segment_number = 0
185
+ until file.eof?
186
+ segment_number += 1
187
+ object = "#{ dest }/#{ segment_number.to_s.rjust(4, '0') }"
188
+ pos = file.pos
189
+ md5 = segment_md5(file, segment_bytes)
190
+ opts = headers.merge('ETag' => md5)
191
+
192
+ with_retries("PUT '#{ segments_container }/#{ object }'") do
193
+ file.seek(pos)
194
+ offset = 0
195
+ connection.put_object(segments_container, object, nil, opts) do
196
+ # block is called to stream data until it returns ''
197
+ data = ''
198
+ if offset <= segment_bytes - SEGMENT_BUFFER
199
+ data = file.read(SEGMENT_BUFFER).to_s # nil => ''
200
+ offset += data.size
201
+ end
202
+ data
203
+ end
204
+ end
205
+
206
+ segments << {
207
+ :path => "#{ segments_container }/#{ object }",
208
+ :etag => md5,
209
+ :size_bytes => file.pos - pos
210
+ }
211
+
212
+ if i = progress.rindex(segment_number)
213
+ Logger.info "\s\s...#{ i + 1 }0% Complete..."
214
+ end
215
+ end
216
+ end
217
+ segments
218
+ end
219
+
220
+ def segment_md5(file, segment_bytes)
221
+ md5 = Digest::MD5.new
222
+ offset = 0
223
+ while offset <= segment_bytes - SEGMENT_BUFFER
224
+ data = file.read(SEGMENT_BUFFER)
225
+ break unless data
226
+ offset += data.size
227
+ md5 << data
228
+ end
229
+ md5.hexdigest
230
+ end
231
+
232
+ # Each segment's ETag and byte_size will be verified once uploaded.
233
+ # Request will raise an exception if verification fails or segments
234
+ # are not found. However, each segment's ETag was verified when we
235
+ # uploaded the segments, so this should only retry failed requests.
236
+ def upload_manifest(dest, segments)
237
+ Logger.info "\s\sStoring SLO Manifest '#{ container }/#{ dest }'"
238
+
239
+ with_retries("PUT SLO Manifest '#{ container }/#{ dest }'") do
240
+ connection.put_static_obj_manifest(container, dest, segments, headers)
241
+ end
242
+ end
243
+
244
+ # If :days_to_keep was set, each object will be scheduled for deletion.
245
+ # This includes non-SLO objects, the SLO manifest and all segments.
246
+ def headers
247
+ headers = {}
248
+ headers.merge!('X-Delete-At' => delete_at) if delete_at
249
+ headers
250
+ end
251
+
252
+ def delete_at
253
+ return unless days_to_keep
254
+ @delete_at ||= (Time.now.utc + days_to_keep * 60**2 * 24).to_i
255
+ end
256
+
257
+ def adjusted_segment_bytes(segment_bytes, file_size)
258
+ return segment_bytes if file_size / segment_bytes.to_f <= 1000
259
+
260
+ mb = orig_mb = segment_bytes / 1024**2
261
+ mb += 1 until file_size / (1024**2 * mb).to_f <= 1000
262
+ Logger.warn Error.new(<<-EOS)
263
+ Segment Size Adjusted
264
+ Your original #segment_size of #{ orig_mb } MiB has been adjusted
265
+ to #{ mb } MiB in order to satisfy the limit of 1000 segments.
266
+ To enforce your chosen #segment_size, you should use the Splitter.
267
+ e.g. split_into_chunks_of #{ mb * 1000 } (#segment_size * 1000)
268
+ EOS
269
+ 1024**2 * mb
270
+ end
271
+
272
+ class Object
273
+ attr_reader :name, :hash
274
+
275
+ def initialize(cloud_io, data)
276
+ @cloud_io = cloud_io
277
+ @name = data['name']
278
+ @hash = data['hash']
279
+ end
280
+
281
+ def slo?
282
+ !!metadata['X-Static-Large-Object']
283
+ end
284
+
285
+ def marked_for_deletion?
286
+ !!metadata['X-Delete-At']
287
+ end
288
+
289
+ private
290
+
291
+ def metadata
292
+ @metadata ||= @cloud_io.head_object(self).headers
293
+ end
294
+ end
295
+
296
+ end
297
+ end
298
+ end