alg-backup 3.0.10

Sign up to get free protection for your applications and to get access to all the features.
Files changed (137) hide show
  1. data/.gitignore +2 -0
  2. data/.infinity_test +7 -0
  3. data/.rspec +3 -0
  4. data/Gemfile +25 -0
  5. data/Gemfile.lock +101 -0
  6. data/LICENSE.md +24 -0
  7. data/README.md +276 -0
  8. data/backup.gemspec +39 -0
  9. data/bin/backup +260 -0
  10. data/lib/backup.rb +168 -0
  11. data/lib/backup/archive.rb +73 -0
  12. data/lib/backup/cli.rb +50 -0
  13. data/lib/backup/compressor/base.rb +17 -0
  14. data/lib/backup/compressor/gzip.rb +61 -0
  15. data/lib/backup/configuration/base.rb +15 -0
  16. data/lib/backup/configuration/compressor/base.rb +10 -0
  17. data/lib/backup/configuration/compressor/gzip.rb +23 -0
  18. data/lib/backup/configuration/database/base.rb +18 -0
  19. data/lib/backup/configuration/database/mongodb.rb +37 -0
  20. data/lib/backup/configuration/database/mysql.rb +37 -0
  21. data/lib/backup/configuration/database/postgresql.rb +37 -0
  22. data/lib/backup/configuration/database/redis.rb +35 -0
  23. data/lib/backup/configuration/encryptor/base.rb +10 -0
  24. data/lib/backup/configuration/encryptor/gpg.rb +17 -0
  25. data/lib/backup/configuration/encryptor/open_ssl.rb +26 -0
  26. data/lib/backup/configuration/helpers.rb +54 -0
  27. data/lib/backup/configuration/notifier/base.rb +39 -0
  28. data/lib/backup/configuration/notifier/mail.rb +52 -0
  29. data/lib/backup/configuration/notifier/twitter.rb +21 -0
  30. data/lib/backup/configuration/storage/base.rb +18 -0
  31. data/lib/backup/configuration/storage/cloudfiles.rb +21 -0
  32. data/lib/backup/configuration/storage/dropbox.rb +29 -0
  33. data/lib/backup/configuration/storage/ftp.rb +25 -0
  34. data/lib/backup/configuration/storage/rsync.rb +25 -0
  35. data/lib/backup/configuration/storage/s3.rb +25 -0
  36. data/lib/backup/configuration/storage/scp.rb +25 -0
  37. data/lib/backup/configuration/storage/sftp.rb +25 -0
  38. data/lib/backup/configuration/syncer/rsync.rb +45 -0
  39. data/lib/backup/configuration/syncer/s3.rb +33 -0
  40. data/lib/backup/database/base.rb +33 -0
  41. data/lib/backup/database/mongodb.rb +137 -0
  42. data/lib/backup/database/mysql.rb +104 -0
  43. data/lib/backup/database/postgresql.rb +111 -0
  44. data/lib/backup/database/redis.rb +105 -0
  45. data/lib/backup/dependency.rb +84 -0
  46. data/lib/backup/encryptor/base.rb +17 -0
  47. data/lib/backup/encryptor/gpg.rb +78 -0
  48. data/lib/backup/encryptor/open_ssl.rb +67 -0
  49. data/lib/backup/finder.rb +39 -0
  50. data/lib/backup/logger.rb +86 -0
  51. data/lib/backup/model.rb +272 -0
  52. data/lib/backup/notifier/base.rb +29 -0
  53. data/lib/backup/notifier/binder.rb +32 -0
  54. data/lib/backup/notifier/mail.rb +141 -0
  55. data/lib/backup/notifier/templates/notify_failure.erb +31 -0
  56. data/lib/backup/notifier/templates/notify_success.erb +16 -0
  57. data/lib/backup/notifier/twitter.rb +87 -0
  58. data/lib/backup/storage/base.rb +67 -0
  59. data/lib/backup/storage/cloudfiles.rb +95 -0
  60. data/lib/backup/storage/dropbox.rb +87 -0
  61. data/lib/backup/storage/ftp.rb +114 -0
  62. data/lib/backup/storage/object.rb +45 -0
  63. data/lib/backup/storage/rsync.rb +99 -0
  64. data/lib/backup/storage/s3.rb +108 -0
  65. data/lib/backup/storage/scp.rb +106 -0
  66. data/lib/backup/storage/sftp.rb +106 -0
  67. data/lib/backup/syncer/base.rb +10 -0
  68. data/lib/backup/syncer/rsync.rb +117 -0
  69. data/lib/backup/syncer/s3.rb +116 -0
  70. data/lib/backup/version.rb +43 -0
  71. data/lib/templates/archive +4 -0
  72. data/lib/templates/compressor/gzip +4 -0
  73. data/lib/templates/database/mongodb +10 -0
  74. data/lib/templates/database/mysql +11 -0
  75. data/lib/templates/database/postgresql +11 -0
  76. data/lib/templates/database/redis +10 -0
  77. data/lib/templates/encryptor/gpg +9 -0
  78. data/lib/templates/encryptor/openssl +5 -0
  79. data/lib/templates/notifier/mail +14 -0
  80. data/lib/templates/notifier/twitter +9 -0
  81. data/lib/templates/readme +15 -0
  82. data/lib/templates/storage/cloudfiles +7 -0
  83. data/lib/templates/storage/dropbox +9 -0
  84. data/lib/templates/storage/ftp +8 -0
  85. data/lib/templates/storage/rsync +7 -0
  86. data/lib/templates/storage/s3 +8 -0
  87. data/lib/templates/storage/scp +8 -0
  88. data/lib/templates/storage/sftp +8 -0
  89. data/lib/templates/syncer/rsync +14 -0
  90. data/lib/templates/syncer/s3 +12 -0
  91. data/spec/archive_spec.rb +90 -0
  92. data/spec/backup_spec.rb +11 -0
  93. data/spec/compressor/gzip_spec.rb +59 -0
  94. data/spec/configuration/base_spec.rb +35 -0
  95. data/spec/configuration/compressor/gzip_spec.rb +28 -0
  96. data/spec/configuration/database/base_spec.rb +16 -0
  97. data/spec/configuration/database/mongodb_spec.rb +30 -0
  98. data/spec/configuration/database/mysql_spec.rb +32 -0
  99. data/spec/configuration/database/postgresql_spec.rb +32 -0
  100. data/spec/configuration/database/redis_spec.rb +30 -0
  101. data/spec/configuration/encryptor/gpg_spec.rb +25 -0
  102. data/spec/configuration/encryptor/open_ssl_spec.rb +31 -0
  103. data/spec/configuration/notifier/mail_spec.rb +32 -0
  104. data/spec/configuration/storage/cloudfiles_spec.rb +34 -0
  105. data/spec/configuration/storage/dropbox_spec.rb +43 -0
  106. data/spec/configuration/storage/ftp_spec.rb +40 -0
  107. data/spec/configuration/storage/rsync_spec.rb +37 -0
  108. data/spec/configuration/storage/s3_spec.rb +37 -0
  109. data/spec/configuration/storage/scp_spec.rb +40 -0
  110. data/spec/configuration/storage/sftp_spec.rb +40 -0
  111. data/spec/configuration/syncer/rsync_spec.rb +46 -0
  112. data/spec/configuration/syncer/s3_spec.rb +43 -0
  113. data/spec/database/base_spec.rb +30 -0
  114. data/spec/database/mongodb_spec.rb +144 -0
  115. data/spec/database/mysql_spec.rb +150 -0
  116. data/spec/database/postgresql_spec.rb +164 -0
  117. data/spec/database/redis_spec.rb +122 -0
  118. data/spec/encryptor/gpg_spec.rb +57 -0
  119. data/spec/encryptor/open_ssl_spec.rb +102 -0
  120. data/spec/logger_spec.rb +46 -0
  121. data/spec/model_spec.rb +236 -0
  122. data/spec/notifier/mail_spec.rb +97 -0
  123. data/spec/notifier/twitter_spec.rb +86 -0
  124. data/spec/spec_helper.rb +21 -0
  125. data/spec/storage/base_spec.rb +33 -0
  126. data/spec/storage/cloudfiles_spec.rb +102 -0
  127. data/spec/storage/dropbox_spec.rb +105 -0
  128. data/spec/storage/ftp_spec.rb +133 -0
  129. data/spec/storage/object_spec.rb +74 -0
  130. data/spec/storage/rsync_spec.rb +115 -0
  131. data/spec/storage/s3_spec.rb +110 -0
  132. data/spec/storage/scp_spec.rb +129 -0
  133. data/spec/storage/sftp_spec.rb +125 -0
  134. data/spec/syncer/rsync_spec.rb +156 -0
  135. data/spec/syncer/s3_spec.rb +139 -0
  136. data/spec/version_spec.rb +21 -0
  137. metadata +217 -0
@@ -0,0 +1,67 @@
1
+ # encoding: utf-8
2
+
3
+ module Backup
4
+ module Encryptor
5
+ class OpenSSL < Base
6
+
7
+ ##
8
+ # The password that'll be used to encrypt the backup. This
9
+ # password will be required to decrypt the backup later on.
10
+ attr_accessor :password
11
+
12
+ ##
13
+ # Determines whether the 'base64' should be used or not
14
+ attr_writer :base64
15
+
16
+ ##
17
+ # Determines whether the 'salt' flag should be used
18
+ attr_writer :salt
19
+
20
+ ##
21
+ # Creates a new instance of Backup::Encryptor::OpenSSL and
22
+ # sets the password attribute to what was provided
23
+ def initialize(&block)
24
+ load_defaults!
25
+
26
+ @base64 ||= false
27
+ @salt ||= false
28
+
29
+ instance_eval(&block) if block_given?
30
+ end
31
+
32
+ ##
33
+ # Performs the encryption of the backup file
34
+ def perform!
35
+ log!
36
+ run("#{ utility(:openssl) } #{ options } -in '#{ Backup::Model.file }' -out '#{ Backup::Model.file }.enc' -k '#{ password }'")
37
+ rm(Backup::Model.file)
38
+ Backup::Model.extension += '.enc'
39
+ end
40
+
41
+ private
42
+
43
+ ##
44
+ # Backup::Encryptor::OpenSSL uses the 256bit AES encryption cipher.
45
+ # 256bit AES is what the US Government uses to encrypt information at the "Top Secret" level.
46
+ def options
47
+ (['aes-256-cbc'] + base64 + salt).join("\s")
48
+ end
49
+
50
+ ##
51
+ # Returns '-a' if @base64 is set to 'true'.
52
+ # This option will make the encrypted output base64 encoded,
53
+ # this makes the encrypted file readable using text editors
54
+ def base64
55
+ return ['-base64'] if @base64; []
56
+ end
57
+
58
+ ##
59
+ # Returns '-salt' if @salt is set to 'true'.
60
+ # This options adds strength to the encryption
61
+ def salt
62
+ return ['-salt'] if @salt; []
63
+ end
64
+
65
+ end
66
+ end
67
+ end
@@ -0,0 +1,39 @@
1
+ # encoding: utf-8
2
+
3
+ module Backup
4
+ class Finder
5
+ attr_accessor :trigger, :config
6
+
7
+ ##
8
+ # Initializes a new Backup::Finder object
9
+ # and stores the path to the configuration file
10
+ def initialize(trigger, config)
11
+ @trigger = trigger.to_sym
12
+ @config = config
13
+ end
14
+
15
+ ##
16
+ # Tries to find and load the configuration file and return the proper
17
+ # backup model configuration (specified by the 'trigger')
18
+ def find
19
+ unless File.exist?(config)
20
+ puts "Could not find a configuration file in '#{config}'."; exit
21
+ end
22
+
23
+ ##
24
+ # Loads the backup configuration file
25
+ instance_eval(File.read(config))
26
+
27
+ ##
28
+ # Iterates through all the instantiated backup models and returns
29
+ # the one that matches the specified 'trigger'
30
+ Backup::Model.all.each do |model|
31
+ if model.trigger.eql?(trigger)
32
+ return Backup::Model.current = model
33
+ end
34
+ end
35
+
36
+ puts "Could not find trigger '#{trigger}' in '#{config}'."; exit
37
+ end
38
+ end
39
+ end
@@ -0,0 +1,86 @@
1
+ # encoding: utf-8
2
+
3
+ module Backup
4
+ class Logger
5
+
6
+ ##
7
+ # Outputs a messages to the console and writes it to the backup.log
8
+ def self.message(string)
9
+ puts loggify(:message, string, :green)
10
+ to_file loggify(:message, string)
11
+ end
12
+
13
+ ##
14
+ # Outputs an error to the console and writes it to the backup.log
15
+ def self.error(string)
16
+ puts loggify(:error, string, :red)
17
+ to_file loggify(:error, string)
18
+ end
19
+
20
+ ##
21
+ # Outputs a notice to the console and writes it to the backup.log
22
+ def self.warn(string)
23
+ puts loggify(:warning, string, :yellow)
24
+ to_file loggify(:warning, string)
25
+ end
26
+
27
+ ##
28
+ # Silently logs data to the log file
29
+ def self.silent(string)
30
+ to_file loggify(:silent, string)
31
+ end
32
+
33
+ ##
34
+ # Returns the time in [YYYY/MM/DD HH:MM:SS] format
35
+ def self.time
36
+ Time.now.strftime("%Y/%m/%d %H:%M:%S")
37
+ end
38
+
39
+ ##
40
+ # Builds the string in a log format with the date/time, the type (colorized)
41
+ # based on whether it's a message, notice or error, and the message itself.
42
+ # ANSI color codes are only used in the console, and are not written to the log
43
+ # since it doesn't do anything and just adds more unnecessary bloat to the log file
44
+ def self.loggify(type, string, color = false)
45
+ return "[#{time}][#{type}] #{string}" unless color
46
+ "[#{time}][#{send(color, type)}] #{string}"
47
+ end
48
+
49
+ ##
50
+ # Writes (appends) a string to the backup.log file
51
+ def self.to_file(string)
52
+ File.open(File.join(LOG_PATH, 'backup.log'), 'a') do |file|
53
+ file.write("#{string}\n")
54
+ end
55
+ end
56
+
57
+ ##
58
+ # Invokes the #colorize method with the provided string
59
+ # and the color code "32" (for green)
60
+ def self.green(string)
61
+ colorize(string, 32)
62
+ end
63
+
64
+ ##
65
+ # Invokes the #colorize method with the provided string
66
+ # and the color code "33" (for yellow)
67
+ def self.yellow(string)
68
+ colorize(string, 33)
69
+ end
70
+
71
+ ##
72
+ # Invokes the #colorize method the with provided string
73
+ # and the color code "31" (for red)
74
+ def self.red(string)
75
+ colorize(string, 31)
76
+ end
77
+
78
+ ##
79
+ # Wraps the provided string in colorizing tags to provide
80
+ # easier to view output to the client
81
+ def self.colorize(string, code)
82
+ "\e[#{code}m#{string}\e[0m"
83
+ end
84
+
85
+ end
86
+ end
@@ -0,0 +1,272 @@
1
+ # encoding: utf-8
2
+
3
+ module Backup
4
+ class Model
5
+ include Backup::CLI
6
+
7
+ ##
8
+ # The trigger is used as an identifier for
9
+ # initializing the backup process
10
+ attr_accessor :trigger
11
+
12
+ ##
13
+ # The label is used for a more friendly user output
14
+ attr_accessor :label
15
+
16
+ ##
17
+ # The databases attribute holds an array of database objects
18
+ attr_accessor :databases
19
+
20
+ ##
21
+ # The archives attr_accessor holds an array of archive objects
22
+ attr_accessor :archives
23
+
24
+ ##
25
+ # The encryptors attr_accessor holds an array of encryptor objects
26
+ attr_accessor :encryptors
27
+
28
+ ##
29
+ # The compressors attr_accessor holds an array of compressor objects
30
+ attr_accessor :compressors
31
+
32
+ ##
33
+ # The notifiers attr_accessor holds an array of notifier objects
34
+ attr_accessor :notifiers
35
+
36
+ ##
37
+ # The storages attribute holds an array of storage objects
38
+ attr_accessor :storages
39
+
40
+ ##
41
+ # The syncers attribute holds an array of syncer objects
42
+ attr_accessor :syncers
43
+
44
+ ##
45
+ # The time when the backup initiated (in format: 2011.02.20.03.29.59)
46
+ attr_accessor :time
47
+
48
+ class << self
49
+ ##
50
+ # The Backup::Model.all class method keeps track of all the models
51
+ # that have been instantiated. It returns the @all class variable,
52
+ # which contains an array of all the models
53
+ attr_accessor :all
54
+
55
+ ##
56
+ # Contains the current file extension (this changes from time to time after a file
57
+ # gets compressed or encrypted so we can keep track of the correct file when new
58
+ # extensions get appended to the current file name)
59
+ attr_accessor :extension
60
+
61
+ ##
62
+ # Contains the currently-in-use model. This attribute should get set by Backup::Finder.
63
+ # Use Backup::Model.current to retrieve the actual data of the model
64
+ attr_accessor :current
65
+
66
+ ##
67
+ # Returns the full path to the current file (including the current extension).
68
+ # To just return the filename and extension without the path, use File.basename(Backup::Model.file)
69
+ def file
70
+ File.join(TMP_PATH, "#{ TIME }.#{ TRIGGER }.#{ Backup::Model.extension }")
71
+ end
72
+
73
+ ##
74
+ # Returns the temporary trigger path of the current model
75
+ # e.g. /Users/Michael/tmp/backup/my_trigger
76
+ def tmp_path
77
+ File.join(TMP_PATH, TRIGGER)
78
+ end
79
+ end
80
+
81
+ ##
82
+ # Accessible through "Backup::Model.all", it stores an array of Backup::Model instances.
83
+ # Everytime a new Backup::Model gets instantiated it gets pushed into this array
84
+ @all = Array.new
85
+
86
+ ##
87
+ # Contains the current file extension (should change after each compression or encryption)
88
+ @extension = 'tar'
89
+
90
+ ##
91
+ # Takes a trigger, label and the configuration block and instantiates the model.
92
+ # The TIME (time of execution) gets stored in the @time attribute.
93
+ # After the instance has evaluated the configuration block and properly set the
94
+ # configuration for the model, it will append the newly created "model" instance
95
+ # to the @all class variable (Array) so it can be accessed by Backup::Finder
96
+ # and any other location
97
+ def initialize(trigger, label, &block)
98
+ @trigger = trigger
99
+ @label = label
100
+ @time = TIME
101
+
102
+ @databases = Array.new
103
+ @archives = Array.new
104
+ @encryptors = Array.new
105
+ @compressors = Array.new
106
+ @storages = Array.new
107
+ @notifiers = Array.new
108
+ @syncers = Array.new
109
+
110
+ instance_eval(&block)
111
+ Backup::Model.all << self
112
+ end
113
+
114
+ ##
115
+ # Adds a database to the array of databases
116
+ # to dump during the backup process
117
+ def database(database, &block)
118
+ @databases << Backup::Database.const_get(
119
+ last_constant(database)
120
+ ).new(&block)
121
+ end
122
+
123
+ ##
124
+ # Adds an archive to the array of archives
125
+ # to store during the backup process
126
+ def archive(name, &block)
127
+ @archives << Backup::Archive.new(name, &block)
128
+ end
129
+
130
+ ##
131
+ # Adds an encryptor to the array of encryptors
132
+ # to use during the backup process
133
+ def encrypt_with(name, &block)
134
+ @encryptors << Backup::Encryptor.const_get(
135
+ last_constant(name)
136
+ ).new(&block)
137
+ end
138
+
139
+ ##
140
+ # Adds a compressor to the array of compressors
141
+ # to use during the backup process
142
+ def compress_with(name, &block)
143
+ @compressors << Backup::Compressor.const_get(
144
+ last_constant(name)
145
+ ).new(&block)
146
+ end
147
+
148
+ ##
149
+ # Adds a notifier to the array of notifiers
150
+ # to use during the backup process
151
+ def notify_by(name, &block)
152
+ @notifiers << Backup::Notifier.const_get(
153
+ last_constant(name)
154
+ ).new(&block)
155
+ end
156
+
157
+ ##
158
+ # Adds a storage method to the array of storage
159
+ # methods to use during the backup process
160
+ def store_with(storage, &block)
161
+ @storages << Backup::Storage.const_get(
162
+ last_constant(storage)
163
+ ).new(&block)
164
+ end
165
+
166
+ ##
167
+ # Adds a syncer method to the array of syncer
168
+ # methods to use during the backup process
169
+ def sync_with(syncer, &block)
170
+ @syncers << Backup::Syncer.const_get(
171
+ last_constant(syncer)
172
+ ).new(&block)
173
+ end
174
+
175
+ ##
176
+ # Performs the backup process
177
+ ##
178
+ # [Databases]
179
+ # Runs all (if any) database objects to dump the databases
180
+ ##
181
+ # [Archives]
182
+ # Runs all (if any) archive objects to package all their
183
+ # paths in to a single tar file and places it in the backup folder
184
+ ##
185
+ # [Package]
186
+ # After all the database dumps and archives are placed inside
187
+ # the folder, it'll make a single .tar package (archive) out of it
188
+ ##
189
+ # [Encryption]
190
+ # Optionally encrypts the packaged file with one or more encryptors
191
+ ##
192
+ # [Compression]
193
+ # Optionally compresses the packaged file with one or more compressors
194
+ ##
195
+ # [Storages]
196
+ # Runs all (if any) storage objects to store the backups to remote locations
197
+ # and (if configured) it'll cycle the files on the remote location to limit the
198
+ # amount of backups stored on each individual location
199
+ ##
200
+ # [Syncers]
201
+ # Runs all (if any) sync objects to store the backups to remote locations.
202
+ # A Syncer does not go through the process of packaging, compressing, encrypting backups.
203
+ # A Syncer directly transfers data from the filesystem to the remote location
204
+ ##
205
+ # [Notifiers]
206
+ # Runs all (if any) notifier objects when a backup proces finished with or without
207
+ # any errors.
208
+ ##
209
+ # [Cleaning]
210
+ # After the whole backup process finishes, it'll go ahead and remove any temporary
211
+ # file that it produced. If an exception(error) is raised during this process which
212
+ # breaks the process, it'll always ensure it removes the temporary files regardless
213
+ # to avoid mass consumption of storage space on the machine
214
+ def perform!
215
+ begin
216
+ if databases.any? or archives.any?
217
+ databases.each { |d| d.perform! }
218
+ archives.each { |a| a.perform! }
219
+ package!
220
+ compressors.each { |c| c.perform! }
221
+ encryptors.each { |e| e.perform! }
222
+ storages.each { |s| s.perform! }
223
+ clean!
224
+ end
225
+
226
+ syncers.each { |s| s.perform! }
227
+ notifiers.each { |n| n.perform!(self) }
228
+ rescue Exception => exception
229
+ clean!
230
+ notifiers.each { |n| n.perform!(self, exception) }
231
+ show_exception!(exception)
232
+ end
233
+ end
234
+
235
+ private
236
+
237
+ ##
238
+ # After all the databases and archives have been dumped and sorted,
239
+ # these files will be bundled in to a .tar archive (uncompressed) so it
240
+ # becomes a single (transferrable) packaged file.
241
+ def package!
242
+ Logger.message "Backup started packaging everything to a single archive file."
243
+ run("#{ utility(:tar) } -c -C '#{ TMP_PATH }' '#{ TRIGGER }' > '#{ File.join(TMP_PATH, "#{TIME}.#{TRIGGER}.tar") }'")
244
+ end
245
+
246
+ ##
247
+ # Cleans up the temporary files that were created after the backup process finishes
248
+ def clean!
249
+ Logger.message "Backup started cleaning up the temporary files."
250
+ run("#{ utility(:rm) } -rf '#{ File.join(TMP_PATH, TRIGGER) }' '#{ File.join(TMP_PATH, "#{TIME}.#{TRIGGER}.#{Backup::Model.extension}") }'")
251
+ end
252
+
253
+ ##
254
+ # Returns the string representation of the last value of a nested constant
255
+ # example:
256
+ # Backup::Model::MySQL
257
+ # becomes and returns:
258
+ # "MySQL"
259
+ def last_constant(constant)
260
+ constant.to_s.split("::").last
261
+ end
262
+
263
+ ##
264
+ # Formats an exception
265
+ def show_exception!(exception)
266
+ puts ("=" * 75) + "\nException that got raised:\n#{exception}\n" + ("=" * 75) + "\n" + exception.backtrace.join("\n")
267
+ puts ("=" * 75) + "\n\nYou are running Backup version \"#{Backup::Version.current}\" and Ruby version \"#{ENV['RUBY_VERSION']}\".\n"
268
+ puts "If you've setup a \"Notification\" in your configuration file, the above error will have been sent."
269
+ end
270
+
271
+ end
272
+ end