alg-backup 3.0.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (137) hide show
  1. data/.gitignore +2 -0
  2. data/.infinity_test +7 -0
  3. data/.rspec +3 -0
  4. data/Gemfile +25 -0
  5. data/Gemfile.lock +101 -0
  6. data/LICENSE.md +24 -0
  7. data/README.md +276 -0
  8. data/backup.gemspec +39 -0
  9. data/bin/backup +260 -0
  10. data/lib/backup.rb +168 -0
  11. data/lib/backup/archive.rb +73 -0
  12. data/lib/backup/cli.rb +50 -0
  13. data/lib/backup/compressor/base.rb +17 -0
  14. data/lib/backup/compressor/gzip.rb +61 -0
  15. data/lib/backup/configuration/base.rb +15 -0
  16. data/lib/backup/configuration/compressor/base.rb +10 -0
  17. data/lib/backup/configuration/compressor/gzip.rb +23 -0
  18. data/lib/backup/configuration/database/base.rb +18 -0
  19. data/lib/backup/configuration/database/mongodb.rb +37 -0
  20. data/lib/backup/configuration/database/mysql.rb +37 -0
  21. data/lib/backup/configuration/database/postgresql.rb +37 -0
  22. data/lib/backup/configuration/database/redis.rb +35 -0
  23. data/lib/backup/configuration/encryptor/base.rb +10 -0
  24. data/lib/backup/configuration/encryptor/gpg.rb +17 -0
  25. data/lib/backup/configuration/encryptor/open_ssl.rb +26 -0
  26. data/lib/backup/configuration/helpers.rb +54 -0
  27. data/lib/backup/configuration/notifier/base.rb +39 -0
  28. data/lib/backup/configuration/notifier/mail.rb +52 -0
  29. data/lib/backup/configuration/notifier/twitter.rb +21 -0
  30. data/lib/backup/configuration/storage/base.rb +18 -0
  31. data/lib/backup/configuration/storage/cloudfiles.rb +21 -0
  32. data/lib/backup/configuration/storage/dropbox.rb +29 -0
  33. data/lib/backup/configuration/storage/ftp.rb +25 -0
  34. data/lib/backup/configuration/storage/rsync.rb +25 -0
  35. data/lib/backup/configuration/storage/s3.rb +25 -0
  36. data/lib/backup/configuration/storage/scp.rb +25 -0
  37. data/lib/backup/configuration/storage/sftp.rb +25 -0
  38. data/lib/backup/configuration/syncer/rsync.rb +45 -0
  39. data/lib/backup/configuration/syncer/s3.rb +33 -0
  40. data/lib/backup/database/base.rb +33 -0
  41. data/lib/backup/database/mongodb.rb +137 -0
  42. data/lib/backup/database/mysql.rb +104 -0
  43. data/lib/backup/database/postgresql.rb +111 -0
  44. data/lib/backup/database/redis.rb +105 -0
  45. data/lib/backup/dependency.rb +84 -0
  46. data/lib/backup/encryptor/base.rb +17 -0
  47. data/lib/backup/encryptor/gpg.rb +78 -0
  48. data/lib/backup/encryptor/open_ssl.rb +67 -0
  49. data/lib/backup/finder.rb +39 -0
  50. data/lib/backup/logger.rb +86 -0
  51. data/lib/backup/model.rb +272 -0
  52. data/lib/backup/notifier/base.rb +29 -0
  53. data/lib/backup/notifier/binder.rb +32 -0
  54. data/lib/backup/notifier/mail.rb +141 -0
  55. data/lib/backup/notifier/templates/notify_failure.erb +31 -0
  56. data/lib/backup/notifier/templates/notify_success.erb +16 -0
  57. data/lib/backup/notifier/twitter.rb +87 -0
  58. data/lib/backup/storage/base.rb +67 -0
  59. data/lib/backup/storage/cloudfiles.rb +95 -0
  60. data/lib/backup/storage/dropbox.rb +87 -0
  61. data/lib/backup/storage/ftp.rb +114 -0
  62. data/lib/backup/storage/object.rb +45 -0
  63. data/lib/backup/storage/rsync.rb +99 -0
  64. data/lib/backup/storage/s3.rb +108 -0
  65. data/lib/backup/storage/scp.rb +106 -0
  66. data/lib/backup/storage/sftp.rb +106 -0
  67. data/lib/backup/syncer/base.rb +10 -0
  68. data/lib/backup/syncer/rsync.rb +117 -0
  69. data/lib/backup/syncer/s3.rb +116 -0
  70. data/lib/backup/version.rb +43 -0
  71. data/lib/templates/archive +4 -0
  72. data/lib/templates/compressor/gzip +4 -0
  73. data/lib/templates/database/mongodb +10 -0
  74. data/lib/templates/database/mysql +11 -0
  75. data/lib/templates/database/postgresql +11 -0
  76. data/lib/templates/database/redis +10 -0
  77. data/lib/templates/encryptor/gpg +9 -0
  78. data/lib/templates/encryptor/openssl +5 -0
  79. data/lib/templates/notifier/mail +14 -0
  80. data/lib/templates/notifier/twitter +9 -0
  81. data/lib/templates/readme +15 -0
  82. data/lib/templates/storage/cloudfiles +7 -0
  83. data/lib/templates/storage/dropbox +9 -0
  84. data/lib/templates/storage/ftp +8 -0
  85. data/lib/templates/storage/rsync +7 -0
  86. data/lib/templates/storage/s3 +8 -0
  87. data/lib/templates/storage/scp +8 -0
  88. data/lib/templates/storage/sftp +8 -0
  89. data/lib/templates/syncer/rsync +14 -0
  90. data/lib/templates/syncer/s3 +12 -0
  91. data/spec/archive_spec.rb +90 -0
  92. data/spec/backup_spec.rb +11 -0
  93. data/spec/compressor/gzip_spec.rb +59 -0
  94. data/spec/configuration/base_spec.rb +35 -0
  95. data/spec/configuration/compressor/gzip_spec.rb +28 -0
  96. data/spec/configuration/database/base_spec.rb +16 -0
  97. data/spec/configuration/database/mongodb_spec.rb +30 -0
  98. data/spec/configuration/database/mysql_spec.rb +32 -0
  99. data/spec/configuration/database/postgresql_spec.rb +32 -0
  100. data/spec/configuration/database/redis_spec.rb +30 -0
  101. data/spec/configuration/encryptor/gpg_spec.rb +25 -0
  102. data/spec/configuration/encryptor/open_ssl_spec.rb +31 -0
  103. data/spec/configuration/notifier/mail_spec.rb +32 -0
  104. data/spec/configuration/storage/cloudfiles_spec.rb +34 -0
  105. data/spec/configuration/storage/dropbox_spec.rb +43 -0
  106. data/spec/configuration/storage/ftp_spec.rb +40 -0
  107. data/spec/configuration/storage/rsync_spec.rb +37 -0
  108. data/spec/configuration/storage/s3_spec.rb +37 -0
  109. data/spec/configuration/storage/scp_spec.rb +40 -0
  110. data/spec/configuration/storage/sftp_spec.rb +40 -0
  111. data/spec/configuration/syncer/rsync_spec.rb +46 -0
  112. data/spec/configuration/syncer/s3_spec.rb +43 -0
  113. data/spec/database/base_spec.rb +30 -0
  114. data/spec/database/mongodb_spec.rb +144 -0
  115. data/spec/database/mysql_spec.rb +150 -0
  116. data/spec/database/postgresql_spec.rb +164 -0
  117. data/spec/database/redis_spec.rb +122 -0
  118. data/spec/encryptor/gpg_spec.rb +57 -0
  119. data/spec/encryptor/open_ssl_spec.rb +102 -0
  120. data/spec/logger_spec.rb +46 -0
  121. data/spec/model_spec.rb +236 -0
  122. data/spec/notifier/mail_spec.rb +97 -0
  123. data/spec/notifier/twitter_spec.rb +86 -0
  124. data/spec/spec_helper.rb +21 -0
  125. data/spec/storage/base_spec.rb +33 -0
  126. data/spec/storage/cloudfiles_spec.rb +102 -0
  127. data/spec/storage/dropbox_spec.rb +105 -0
  128. data/spec/storage/ftp_spec.rb +133 -0
  129. data/spec/storage/object_spec.rb +74 -0
  130. data/spec/storage/rsync_spec.rb +115 -0
  131. data/spec/storage/s3_spec.rb +110 -0
  132. data/spec/storage/scp_spec.rb +129 -0
  133. data/spec/storage/sftp_spec.rb +125 -0
  134. data/spec/syncer/rsync_spec.rb +156 -0
  135. data/spec/syncer/s3_spec.rb +139 -0
  136. data/spec/version_spec.rb +21 -0
  137. metadata +217 -0
@@ -0,0 +1,67 @@
1
+ # encoding: utf-8
2
+
3
+ module Backup
4
+ module Encryptor
5
+ class OpenSSL < Base
6
+
7
+ ##
8
+ # The password that'll be used to encrypt the backup. This
9
+ # password will be required to decrypt the backup later on.
10
+ attr_accessor :password
11
+
12
+ ##
13
+ # Determines whether the 'base64' should be used or not
14
+ attr_writer :base64
15
+
16
+ ##
17
+ # Determines whether the 'salt' flag should be used
18
+ attr_writer :salt
19
+
20
+ ##
21
+ # Creates a new instance of Backup::Encryptor::OpenSSL and
22
+ # sets the password attribute to what was provided
23
+ def initialize(&block)
24
+ load_defaults!
25
+
26
+ @base64 ||= false
27
+ @salt ||= false
28
+
29
+ instance_eval(&block) if block_given?
30
+ end
31
+
32
+ ##
33
+ # Performs the encryption of the backup file
34
+ def perform!
35
+ log!
36
+ run("#{ utility(:openssl) } #{ options } -in '#{ Backup::Model.file }' -out '#{ Backup::Model.file }.enc' -k '#{ password }'")
37
+ rm(Backup::Model.file)
38
+ Backup::Model.extension += '.enc'
39
+ end
40
+
41
+ private
42
+
43
+ ##
44
+ # Backup::Encryptor::OpenSSL uses the 256bit AES encryption cipher.
45
+ # 256bit AES is what the US Government uses to encrypt information at the "Top Secret" level.
46
+ def options
47
+ (['aes-256-cbc'] + base64 + salt).join("\s")
48
+ end
49
+
50
+ ##
51
+ # Returns '-a' if @base64 is set to 'true'.
52
+ # This option will make the encrypted output base64 encoded,
53
+ # this makes the encrypted file readable using text editors
54
+ def base64
55
+ return ['-base64'] if @base64; []
56
+ end
57
+
58
+ ##
59
+ # Returns '-salt' if @salt is set to 'true'.
60
+ # This options adds strength to the encryption
61
+ def salt
62
+ return ['-salt'] if @salt; []
63
+ end
64
+
65
+ end
66
+ end
67
+ end
@@ -0,0 +1,39 @@
1
+ # encoding: utf-8
2
+
3
+ module Backup
4
+ class Finder
5
+ attr_accessor :trigger, :config
6
+
7
+ ##
8
+ # Initializes a new Backup::Finder object
9
+ # and stores the path to the configuration file
10
+ def initialize(trigger, config)
11
+ @trigger = trigger.to_sym
12
+ @config = config
13
+ end
14
+
15
+ ##
16
+ # Tries to find and load the configuration file and return the proper
17
+ # backup model configuration (specified by the 'trigger')
18
+ def find
19
+ unless File.exist?(config)
20
+ puts "Could not find a configuration file in '#{config}'."; exit
21
+ end
22
+
23
+ ##
24
+ # Loads the backup configuration file
25
+ instance_eval(File.read(config))
26
+
27
+ ##
28
+ # Iterates through all the instantiated backup models and returns
29
+ # the one that matches the specified 'trigger'
30
+ Backup::Model.all.each do |model|
31
+ if model.trigger.eql?(trigger)
32
+ return Backup::Model.current = model
33
+ end
34
+ end
35
+
36
+ puts "Could not find trigger '#{trigger}' in '#{config}'."; exit
37
+ end
38
+ end
39
+ end
@@ -0,0 +1,86 @@
1
+ # encoding: utf-8
2
+
3
+ module Backup
4
+ class Logger
5
+
6
+ ##
7
+ # Outputs a messages to the console and writes it to the backup.log
8
+ def self.message(string)
9
+ puts loggify(:message, string, :green)
10
+ to_file loggify(:message, string)
11
+ end
12
+
13
+ ##
14
+ # Outputs an error to the console and writes it to the backup.log
15
+ def self.error(string)
16
+ puts loggify(:error, string, :red)
17
+ to_file loggify(:error, string)
18
+ end
19
+
20
+ ##
21
+ # Outputs a notice to the console and writes it to the backup.log
22
+ def self.warn(string)
23
+ puts loggify(:warning, string, :yellow)
24
+ to_file loggify(:warning, string)
25
+ end
26
+
27
+ ##
28
+ # Silently logs data to the log file
29
+ def self.silent(string)
30
+ to_file loggify(:silent, string)
31
+ end
32
+
33
+ ##
34
+ # Returns the time in [YYYY/MM/DD HH:MM:SS] format
35
+ def self.time
36
+ Time.now.strftime("%Y/%m/%d %H:%M:%S")
37
+ end
38
+
39
+ ##
40
+ # Builds the string in a log format with the date/time, the type (colorized)
41
+ # based on whether it's a message, notice or error, and the message itself.
42
+ # ANSI color codes are only used in the console, and are not written to the log
43
+ # since it doesn't do anything and just adds more unnecessary bloat to the log file
44
+ def self.loggify(type, string, color = false)
45
+ return "[#{time}][#{type}] #{string}" unless color
46
+ "[#{time}][#{send(color, type)}] #{string}"
47
+ end
48
+
49
+ ##
50
+ # Writes (appends) a string to the backup.log file
51
+ def self.to_file(string)
52
+ File.open(File.join(LOG_PATH, 'backup.log'), 'a') do |file|
53
+ file.write("#{string}\n")
54
+ end
55
+ end
56
+
57
+ ##
58
+ # Invokes the #colorize method with the provided string
59
+ # and the color code "32" (for green)
60
+ def self.green(string)
61
+ colorize(string, 32)
62
+ end
63
+
64
+ ##
65
+ # Invokes the #colorize method with the provided string
66
+ # and the color code "33" (for yellow)
67
+ def self.yellow(string)
68
+ colorize(string, 33)
69
+ end
70
+
71
+ ##
72
+ # Invokes the #colorize method the with provided string
73
+ # and the color code "31" (for red)
74
+ def self.red(string)
75
+ colorize(string, 31)
76
+ end
77
+
78
+ ##
79
+ # Wraps the provided string in colorizing tags to provide
80
+ # easier to view output to the client
81
+ def self.colorize(string, code)
82
+ "\e[#{code}m#{string}\e[0m"
83
+ end
84
+
85
+ end
86
+ end
@@ -0,0 +1,272 @@
1
+ # encoding: utf-8
2
+
3
+ module Backup
4
+ class Model
5
+ include Backup::CLI
6
+
7
+ ##
8
+ # The trigger is used as an identifier for
9
+ # initializing the backup process
10
+ attr_accessor :trigger
11
+
12
+ ##
13
+ # The label is used for a more friendly user output
14
+ attr_accessor :label
15
+
16
+ ##
17
+ # The databases attribute holds an array of database objects
18
+ attr_accessor :databases
19
+
20
+ ##
21
+ # The archives attr_accessor holds an array of archive objects
22
+ attr_accessor :archives
23
+
24
+ ##
25
+ # The encryptors attr_accessor holds an array of encryptor objects
26
+ attr_accessor :encryptors
27
+
28
+ ##
29
+ # The compressors attr_accessor holds an array of compressor objects
30
+ attr_accessor :compressors
31
+
32
+ ##
33
+ # The notifiers attr_accessor holds an array of notifier objects
34
+ attr_accessor :notifiers
35
+
36
+ ##
37
+ # The storages attribute holds an array of storage objects
38
+ attr_accessor :storages
39
+
40
+ ##
41
+ # The syncers attribute holds an array of syncer objects
42
+ attr_accessor :syncers
43
+
44
+ ##
45
+ # The time when the backup initiated (in format: 2011.02.20.03.29.59)
46
+ attr_accessor :time
47
+
48
+ class << self
49
+ ##
50
+ # The Backup::Model.all class method keeps track of all the models
51
+ # that have been instantiated. It returns the @all class variable,
52
+ # which contains an array of all the models
53
+ attr_accessor :all
54
+
55
+ ##
56
+ # Contains the current file extension (this changes from time to time after a file
57
+ # gets compressed or encrypted so we can keep track of the correct file when new
58
+ # extensions get appended to the current file name)
59
+ attr_accessor :extension
60
+
61
+ ##
62
+ # Contains the currently-in-use model. This attribute should get set by Backup::Finder.
63
+ # Use Backup::Model.current to retrieve the actual data of the model
64
+ attr_accessor :current
65
+
66
+ ##
67
+ # Returns the full path to the current file (including the current extension).
68
+ # To just return the filename and extension without the path, use File.basename(Backup::Model.file)
69
+ def file
70
+ File.join(TMP_PATH, "#{ TIME }.#{ TRIGGER }.#{ Backup::Model.extension }")
71
+ end
72
+
73
+ ##
74
+ # Returns the temporary trigger path of the current model
75
+ # e.g. /Users/Michael/tmp/backup/my_trigger
76
+ def tmp_path
77
+ File.join(TMP_PATH, TRIGGER)
78
+ end
79
+ end
80
+
81
+ ##
82
+ # Accessible through "Backup::Model.all", it stores an array of Backup::Model instances.
83
+ # Everytime a new Backup::Model gets instantiated it gets pushed into this array
84
+ @all = Array.new
85
+
86
+ ##
87
+ # Contains the current file extension (should change after each compression or encryption)
88
+ @extension = 'tar'
89
+
90
+ ##
91
+ # Takes a trigger, label and the configuration block and instantiates the model.
92
+ # The TIME (time of execution) gets stored in the @time attribute.
93
+ # After the instance has evaluated the configuration block and properly set the
94
+ # configuration for the model, it will append the newly created "model" instance
95
+ # to the @all class variable (Array) so it can be accessed by Backup::Finder
96
+ # and any other location
97
+ def initialize(trigger, label, &block)
98
+ @trigger = trigger
99
+ @label = label
100
+ @time = TIME
101
+
102
+ @databases = Array.new
103
+ @archives = Array.new
104
+ @encryptors = Array.new
105
+ @compressors = Array.new
106
+ @storages = Array.new
107
+ @notifiers = Array.new
108
+ @syncers = Array.new
109
+
110
+ instance_eval(&block)
111
+ Backup::Model.all << self
112
+ end
113
+
114
+ ##
115
+ # Adds a database to the array of databases
116
+ # to dump during the backup process
117
+ def database(database, &block)
118
+ @databases << Backup::Database.const_get(
119
+ last_constant(database)
120
+ ).new(&block)
121
+ end
122
+
123
+ ##
124
+ # Adds an archive to the array of archives
125
+ # to store during the backup process
126
+ def archive(name, &block)
127
+ @archives << Backup::Archive.new(name, &block)
128
+ end
129
+
130
+ ##
131
+ # Adds an encryptor to the array of encryptors
132
+ # to use during the backup process
133
+ def encrypt_with(name, &block)
134
+ @encryptors << Backup::Encryptor.const_get(
135
+ last_constant(name)
136
+ ).new(&block)
137
+ end
138
+
139
+ ##
140
+ # Adds a compressor to the array of compressors
141
+ # to use during the backup process
142
+ def compress_with(name, &block)
143
+ @compressors << Backup::Compressor.const_get(
144
+ last_constant(name)
145
+ ).new(&block)
146
+ end
147
+
148
+ ##
149
+ # Adds a notifier to the array of notifiers
150
+ # to use during the backup process
151
+ def notify_by(name, &block)
152
+ @notifiers << Backup::Notifier.const_get(
153
+ last_constant(name)
154
+ ).new(&block)
155
+ end
156
+
157
+ ##
158
+ # Adds a storage method to the array of storage
159
+ # methods to use during the backup process
160
+ def store_with(storage, &block)
161
+ @storages << Backup::Storage.const_get(
162
+ last_constant(storage)
163
+ ).new(&block)
164
+ end
165
+
166
+ ##
167
+ # Adds a syncer method to the array of syncer
168
+ # methods to use during the backup process
169
+ def sync_with(syncer, &block)
170
+ @syncers << Backup::Syncer.const_get(
171
+ last_constant(syncer)
172
+ ).new(&block)
173
+ end
174
+
175
+ ##
176
+ # Performs the backup process
177
+ ##
178
+ # [Databases]
179
+ # Runs all (if any) database objects to dump the databases
180
+ ##
181
+ # [Archives]
182
+ # Runs all (if any) archive objects to package all their
183
+ # paths in to a single tar file and places it in the backup folder
184
+ ##
185
+ # [Package]
186
+ # After all the database dumps and archives are placed inside
187
+ # the folder, it'll make a single .tar package (archive) out of it
188
+ ##
189
+ # [Encryption]
190
+ # Optionally encrypts the packaged file with one or more encryptors
191
+ ##
192
+ # [Compression]
193
+ # Optionally compresses the packaged file with one or more compressors
194
+ ##
195
+ # [Storages]
196
+ # Runs all (if any) storage objects to store the backups to remote locations
197
+ # and (if configured) it'll cycle the files on the remote location to limit the
198
+ # amount of backups stored on each individual location
199
+ ##
200
+ # [Syncers]
201
+ # Runs all (if any) sync objects to store the backups to remote locations.
202
+ # A Syncer does not go through the process of packaging, compressing, encrypting backups.
203
+ # A Syncer directly transfers data from the filesystem to the remote location
204
+ ##
205
+ # [Notifiers]
206
+ # Runs all (if any) notifier objects when a backup proces finished with or without
207
+ # any errors.
208
+ ##
209
+ # [Cleaning]
210
+ # After the whole backup process finishes, it'll go ahead and remove any temporary
211
+ # file that it produced. If an exception(error) is raised during this process which
212
+ # breaks the process, it'll always ensure it removes the temporary files regardless
213
+ # to avoid mass consumption of storage space on the machine
214
+ def perform!
215
+ begin
216
+ if databases.any? or archives.any?
217
+ databases.each { |d| d.perform! }
218
+ archives.each { |a| a.perform! }
219
+ package!
220
+ compressors.each { |c| c.perform! }
221
+ encryptors.each { |e| e.perform! }
222
+ storages.each { |s| s.perform! }
223
+ clean!
224
+ end
225
+
226
+ syncers.each { |s| s.perform! }
227
+ notifiers.each { |n| n.perform!(self) }
228
+ rescue Exception => exception
229
+ clean!
230
+ notifiers.each { |n| n.perform!(self, exception) }
231
+ show_exception!(exception)
232
+ end
233
+ end
234
+
235
+ private
236
+
237
+ ##
238
+ # After all the databases and archives have been dumped and sorted,
239
+ # these files will be bundled in to a .tar archive (uncompressed) so it
240
+ # becomes a single (transferrable) packaged file.
241
+ def package!
242
+ Logger.message "Backup started packaging everything to a single archive file."
243
+ run("#{ utility(:tar) } -c -C '#{ TMP_PATH }' '#{ TRIGGER }' > '#{ File.join(TMP_PATH, "#{TIME}.#{TRIGGER}.tar") }'")
244
+ end
245
+
246
+ ##
247
+ # Cleans up the temporary files that were created after the backup process finishes
248
+ def clean!
249
+ Logger.message "Backup started cleaning up the temporary files."
250
+ run("#{ utility(:rm) } -rf '#{ File.join(TMP_PATH, TRIGGER) }' '#{ File.join(TMP_PATH, "#{TIME}.#{TRIGGER}.#{Backup::Model.extension}") }'")
251
+ end
252
+
253
+ ##
254
+ # Returns the string representation of the last value of a nested constant
255
+ # example:
256
+ # Backup::Model::MySQL
257
+ # becomes and returns:
258
+ # "MySQL"
259
+ def last_constant(constant)
260
+ constant.to_s.split("::").last
261
+ end
262
+
263
+ ##
264
+ # Formats an exception
265
+ def show_exception!(exception)
266
+ puts ("=" * 75) + "\nException that got raised:\n#{exception}\n" + ("=" * 75) + "\n" + exception.backtrace.join("\n")
267
+ puts ("=" * 75) + "\n\nYou are running Backup version \"#{Backup::Version.current}\" and Ruby version \"#{ENV['RUBY_VERSION']}\".\n"
268
+ puts "If you've setup a \"Notification\" in your configuration file, the above error will have been sent."
269
+ end
270
+
271
+ end
272
+ end