cm-backup 1.0.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (133) hide show
  1. checksums.yaml +7 -0
  2. data/README.md +20 -0
  3. data/bin/backup +5 -0
  4. data/lib/backup.rb +144 -0
  5. data/lib/backup/archive.rb +170 -0
  6. data/lib/backup/binder.rb +22 -0
  7. data/lib/backup/cleaner.rb +116 -0
  8. data/lib/backup/cli.rb +374 -0
  9. data/lib/backup/cloud_io/base.rb +41 -0
  10. data/lib/backup/cloud_io/cloud_files.rb +298 -0
  11. data/lib/backup/cloud_io/s3.rb +260 -0
  12. data/lib/backup/compressor/base.rb +35 -0
  13. data/lib/backup/compressor/bzip2.rb +39 -0
  14. data/lib/backup/compressor/custom.rb +53 -0
  15. data/lib/backup/compressor/gzip.rb +74 -0
  16. data/lib/backup/config.rb +119 -0
  17. data/lib/backup/config/dsl.rb +103 -0
  18. data/lib/backup/config/helpers.rb +143 -0
  19. data/lib/backup/database/base.rb +85 -0
  20. data/lib/backup/database/mongodb.rb +187 -0
  21. data/lib/backup/database/mysql.rb +192 -0
  22. data/lib/backup/database/openldap.rb +95 -0
  23. data/lib/backup/database/postgresql.rb +133 -0
  24. data/lib/backup/database/redis.rb +179 -0
  25. data/lib/backup/database/riak.rb +82 -0
  26. data/lib/backup/database/sqlite.rb +57 -0
  27. data/lib/backup/encryptor/base.rb +29 -0
  28. data/lib/backup/encryptor/gpg.rb +747 -0
  29. data/lib/backup/encryptor/open_ssl.rb +77 -0
  30. data/lib/backup/errors.rb +58 -0
  31. data/lib/backup/logger.rb +199 -0
  32. data/lib/backup/logger/console.rb +51 -0
  33. data/lib/backup/logger/fog_adapter.rb +29 -0
  34. data/lib/backup/logger/logfile.rb +133 -0
  35. data/lib/backup/logger/syslog.rb +116 -0
  36. data/lib/backup/model.rb +479 -0
  37. data/lib/backup/notifier/base.rb +128 -0
  38. data/lib/backup/notifier/campfire.rb +63 -0
  39. data/lib/backup/notifier/command.rb +102 -0
  40. data/lib/backup/notifier/datadog.rb +107 -0
  41. data/lib/backup/notifier/flowdock.rb +103 -0
  42. data/lib/backup/notifier/hipchat.rb +118 -0
  43. data/lib/backup/notifier/http_post.rb +117 -0
  44. data/lib/backup/notifier/mail.rb +249 -0
  45. data/lib/backup/notifier/nagios.rb +69 -0
  46. data/lib/backup/notifier/pagerduty.rb +81 -0
  47. data/lib/backup/notifier/prowl.rb +68 -0
  48. data/lib/backup/notifier/pushover.rb +74 -0
  49. data/lib/backup/notifier/ses.rb +105 -0
  50. data/lib/backup/notifier/slack.rb +148 -0
  51. data/lib/backup/notifier/twitter.rb +58 -0
  52. data/lib/backup/notifier/zabbix.rb +63 -0
  53. data/lib/backup/package.rb +55 -0
  54. data/lib/backup/packager.rb +107 -0
  55. data/lib/backup/pipeline.rb +124 -0
  56. data/lib/backup/splitter.rb +76 -0
  57. data/lib/backup/storage/base.rb +69 -0
  58. data/lib/backup/storage/cloud_files.rb +158 -0
  59. data/lib/backup/storage/cycler.rb +75 -0
  60. data/lib/backup/storage/dropbox.rb +212 -0
  61. data/lib/backup/storage/ftp.rb +112 -0
  62. data/lib/backup/storage/local.rb +64 -0
  63. data/lib/backup/storage/qiniu.rb +65 -0
  64. data/lib/backup/storage/rsync.rb +248 -0
  65. data/lib/backup/storage/s3.rb +156 -0
  66. data/lib/backup/storage/scp.rb +67 -0
  67. data/lib/backup/storage/sftp.rb +82 -0
  68. data/lib/backup/syncer/base.rb +70 -0
  69. data/lib/backup/syncer/cloud/base.rb +179 -0
  70. data/lib/backup/syncer/cloud/cloud_files.rb +83 -0
  71. data/lib/backup/syncer/cloud/local_file.rb +100 -0
  72. data/lib/backup/syncer/cloud/s3.rb +110 -0
  73. data/lib/backup/syncer/rsync/base.rb +54 -0
  74. data/lib/backup/syncer/rsync/local.rb +31 -0
  75. data/lib/backup/syncer/rsync/pull.rb +51 -0
  76. data/lib/backup/syncer/rsync/push.rb +205 -0
  77. data/lib/backup/template.rb +46 -0
  78. data/lib/backup/utilities.rb +224 -0
  79. data/lib/backup/version.rb +5 -0
  80. data/templates/cli/archive +28 -0
  81. data/templates/cli/compressor/bzip2 +4 -0
  82. data/templates/cli/compressor/custom +7 -0
  83. data/templates/cli/compressor/gzip +4 -0
  84. data/templates/cli/config +123 -0
  85. data/templates/cli/databases/mongodb +15 -0
  86. data/templates/cli/databases/mysql +18 -0
  87. data/templates/cli/databases/openldap +24 -0
  88. data/templates/cli/databases/postgresql +16 -0
  89. data/templates/cli/databases/redis +16 -0
  90. data/templates/cli/databases/riak +17 -0
  91. data/templates/cli/databases/sqlite +11 -0
  92. data/templates/cli/encryptor/gpg +27 -0
  93. data/templates/cli/encryptor/openssl +9 -0
  94. data/templates/cli/model +26 -0
  95. data/templates/cli/notifier/zabbix +15 -0
  96. data/templates/cli/notifiers/campfire +12 -0
  97. data/templates/cli/notifiers/command +32 -0
  98. data/templates/cli/notifiers/datadog +57 -0
  99. data/templates/cli/notifiers/flowdock +16 -0
  100. data/templates/cli/notifiers/hipchat +16 -0
  101. data/templates/cli/notifiers/http_post +32 -0
  102. data/templates/cli/notifiers/mail +24 -0
  103. data/templates/cli/notifiers/nagios +13 -0
  104. data/templates/cli/notifiers/pagerduty +12 -0
  105. data/templates/cli/notifiers/prowl +11 -0
  106. data/templates/cli/notifiers/pushover +11 -0
  107. data/templates/cli/notifiers/ses +15 -0
  108. data/templates/cli/notifiers/slack +22 -0
  109. data/templates/cli/notifiers/twitter +13 -0
  110. data/templates/cli/splitter +7 -0
  111. data/templates/cli/storages/cloud_files +11 -0
  112. data/templates/cli/storages/dropbox +20 -0
  113. data/templates/cli/storages/ftp +13 -0
  114. data/templates/cli/storages/local +8 -0
  115. data/templates/cli/storages/qiniu +12 -0
  116. data/templates/cli/storages/rsync +17 -0
  117. data/templates/cli/storages/s3 +16 -0
  118. data/templates/cli/storages/scp +15 -0
  119. data/templates/cli/storages/sftp +15 -0
  120. data/templates/cli/syncers/cloud_files +22 -0
  121. data/templates/cli/syncers/rsync_local +20 -0
  122. data/templates/cli/syncers/rsync_pull +28 -0
  123. data/templates/cli/syncers/rsync_push +28 -0
  124. data/templates/cli/syncers/s3 +27 -0
  125. data/templates/general/links +3 -0
  126. data/templates/general/version.erb +2 -0
  127. data/templates/notifier/mail/failure.erb +16 -0
  128. data/templates/notifier/mail/success.erb +16 -0
  129. data/templates/notifier/mail/warning.erb +16 -0
  130. data/templates/storage/dropbox/authorization_url.erb +6 -0
  131. data/templates/storage/dropbox/authorized.erb +4 -0
  132. data/templates/storage/dropbox/cache_file_written.erb +10 -0
  133. metadata +1077 -0
@@ -0,0 +1,46 @@
1
+ # encoding: utf-8
2
+
3
+ require 'erb'
4
+
5
+ module Backup
6
+ class Template
7
+
8
+ # Holds a binding object. Nil if not provided.
9
+ attr_accessor :binding
10
+
11
+ ##
12
+ # Creates a new instance of the Backup::Template class
13
+ # and optionally takes an argument that can be either a binding object, a Hash or nil
14
+ def initialize(object = nil)
15
+ if object.is_a?(Binding)
16
+ @binding = object
17
+ elsif object.is_a?(Hash)
18
+ @binding = Backup::Binder.new(object).get_binding
19
+ else
20
+ @binding = nil
21
+ end
22
+ end
23
+
24
+ ##
25
+ # Renders the provided file (in the context of the binding if any) to the console
26
+ def render(file)
27
+ puts result(file)
28
+ end
29
+
30
+ ##
31
+ # Returns a String object containing the contents of the file (in the context of the binding if any)
32
+ def result(file)
33
+ ERB.new(file_contents(file), nil, '<>').result(binding)
34
+ end
35
+
36
+ private
37
+
38
+ ##
39
+ # Reads and returns the contents of the provided file path,
40
+ # relative from the Backup::TEMPLATE_PATH
41
+ def file_contents(file)
42
+ File.read(File.join(Backup::TEMPLATE_PATH, file))
43
+ end
44
+
45
+ end
46
+ end
@@ -0,0 +1,224 @@
1
+ # encoding: utf-8
2
+
3
+ module Backup
4
+ module Utilities
5
+ class Error < Backup::Error; end
6
+
7
+ UTILITY = {}
8
+ NAMES = %w{
9
+ tar cat split sudo chown hostname
10
+ gzip bzip2
11
+ mongo mongodump mysqldump innobackupex
12
+ pg_dump pg_dumpall redis-cli riak-admin
13
+ gpg openssl
14
+ rsync ssh
15
+ sendmail exim
16
+ send_nsca
17
+ zabbix_sender
18
+ }
19
+
20
+ module DSL
21
+ class << self
22
+ ##
23
+ # Allow users to set the path for all utilities in the .configure block.
24
+ #
25
+ # Utility names with dashes ('redis-cli') will be set using method calls
26
+ # with an underscore ('redis_cli').
27
+ NAMES.each do |name|
28
+ define_method name.gsub('-', '_'), lambda {|val|
29
+ path = File.expand_path(val)
30
+ unless File.executable?(path)
31
+ raise Utilities::Error, <<-EOS
32
+ The path given for '#{ name }' was not found or not executable.
33
+ Path was: #{ path }
34
+ EOS
35
+ end
36
+ UTILITY[name] = path
37
+ }
38
+ end
39
+
40
+ ##
41
+ # Allow users to set the +tar+ distribution if needed. (:gnu or :bsd)
42
+ def tar_dist(val)
43
+ Utilities.tar_dist(val)
44
+ end
45
+ end
46
+ end
47
+
48
+ class << self
49
+ ##
50
+ # Configure the path to system utilities used by Backup.
51
+ #
52
+ # Backup will attempt to locate any required system utilities using a
53
+ # +which+ command call. If a utility can not be found, or you need to
54
+ # specify an alternate path for a utility, you may do so in your
55
+ # +config.rb+ file using this method.
56
+ #
57
+ # Backup supports both GNU and BSD utilities.
58
+ # While Backup uses these utilities in a manner compatible with either
59
+ # version, the +tar+ utility requires some special handling with respect
60
+ # to +Archive+s. Backup will attempt to detect if the +tar+ command
61
+ # found (or set here) is GNU or BSD. If for some reason this fails,
62
+ # this may be set using the +tar_dist+ command shown below.
63
+ #
64
+ # Backup::Utilities.configure do
65
+ # # General Utilites
66
+ # tar '/path/to/tar'
67
+ # tar_dist :gnu # or :bsd
68
+ # cat '/path/to/cat'
69
+ # split '/path/to/split'
70
+ # sudo '/path/to/sudo'
71
+ # chown '/path/to/chown'
72
+ # hostname '/path/to/hostname'
73
+ #
74
+ # # Compressors
75
+ # gzip '/path/to/gzip'
76
+ # bzip2 '/path/to/bzip2'
77
+ #
78
+ # # Database Utilities
79
+ # mongo '/path/to/mongo'
80
+ # mongodump '/path/to/mongodump'
81
+ # mysqldump '/path/to/mysqldump'
82
+ # pg_dump '/path/to/pg_dump'
83
+ # pg_dumpall '/path/to/pg_dumpall'
84
+ # redis_cli '/path/to/redis-cli'
85
+ # riak_admin '/path/to/riak-admin'
86
+ #
87
+ # # Encryptors
88
+ # gpg '/path/to/gpg'
89
+ # openssl '/path/to/openssl'
90
+ #
91
+ # # Syncer and Storage
92
+ # rsync '/path/to/rsync'
93
+ # ssh '/path/to/ssh'
94
+ #
95
+ # # Notifiers
96
+ # sendmail '/path/to/sendmail'
97
+ # exim '/path/to/exim'
98
+ # send_nsca '/path/to/send_nsca'
99
+ # zabbix_sender '/path/to/zabbix_sender'
100
+ # end
101
+ #
102
+ # These paths may be set using absolute paths, or relative to the
103
+ # working directory when Backup is run.
104
+ def configure(&block)
105
+ DSL.instance_eval(&block)
106
+ end
107
+
108
+ def tar_dist(val)
109
+ # the acceptance tests need to be able to reset this to nil
110
+ @gnu_tar = val.nil? ? nil : val == :gnu
111
+ end
112
+
113
+ def gnu_tar?
114
+ return @gnu_tar unless @gnu_tar.nil?
115
+ @gnu_tar = !!run("#{ utility(:tar) } --version").match(/GNU/)
116
+ end
117
+
118
+ private
119
+
120
+ ##
121
+ # Returns the full path to the specified utility.
122
+ # Raises an error if utility can not be found in the system's $PATH
123
+ def utility(name)
124
+ name = name.to_s.strip
125
+ raise Error, 'Utility Name Empty' if name.empty?
126
+
127
+ UTILITY[name] ||= %x[which '#{ name }' 2>/dev/null].chomp
128
+ raise Error, <<-EOS if UTILITY[name].empty?
129
+ Could not locate '#{ name }'.
130
+ Make sure the specified utility is installed
131
+ and available in your system's $PATH, or specify it's location
132
+ in your 'config.rb' file using Backup::Utilities.configure
133
+ EOS
134
+
135
+ UTILITY[name].dup
136
+ end
137
+
138
+ ##
139
+ # Returns the name of the command name from the given command line.
140
+ # This is only used to simplify log messages.
141
+ def command_name(command)
142
+ parts = []
143
+ command = command.split(' ')
144
+ command.shift while command[0].to_s.include?('=')
145
+ parts << command.shift.split('/')[-1]
146
+ if parts[0] == 'sudo'
147
+ until command.empty?
148
+ part = command.shift
149
+ if part.include?('/')
150
+ parts << part.split('/')[-1]
151
+ break
152
+ else
153
+ parts << part
154
+ end
155
+ end
156
+ end
157
+ parts.join(' ')
158
+ end
159
+
160
+ ##
161
+ # Runs a system command
162
+ #
163
+ # All messages generated by the command will be logged.
164
+ # Messages on STDERR will be logged as warnings.
165
+ #
166
+ # If the command fails to execute, or returns a non-zero exit status
167
+ # an Error will be raised.
168
+ #
169
+ # Returns STDOUT
170
+ def run(command)
171
+ name = command_name(command)
172
+ Logger.info "Running system utility '#{ name }'..."
173
+
174
+ begin
175
+ out, err = '', ''
176
+ ps = Open4.popen4(command) do |pid, stdin, stdout, stderr|
177
+ stdin.close
178
+ out, err = stdout.read.strip, stderr.read.strip
179
+ end
180
+ rescue Exception => e
181
+ raise Error.wrap(e, "Failed to execute '#{ name }'")
182
+ end
183
+
184
+ if ps.success?
185
+ unless out.empty?
186
+ Logger.info(
187
+ out.lines.map {|line| "#{ name }:STDOUT: #{ line }" }.join
188
+ )
189
+ end
190
+
191
+ unless err.empty?
192
+ Logger.warn(
193
+ err.lines.map {|line| "#{ name }:STDERR: #{ line }" }.join
194
+ )
195
+ end
196
+
197
+ return out
198
+ else
199
+ raise Error, <<-EOS
200
+ '#{ name }' failed with exit status: #{ ps.exitstatus }
201
+ STDOUT Messages: #{ out.empty? ? 'None' : "\n#{ out }" }
202
+ STDERR Messages: #{ err.empty? ? 'None' : "\n#{ err }" }
203
+ EOS
204
+ end
205
+ end
206
+
207
+ def reset!
208
+ UTILITY.clear
209
+ @gnu_tar = nil
210
+ end
211
+ end
212
+
213
+ # Allows these utility methods to be included in other classes,
214
+ # while allowing them to be stubbed in spec_helper for all specs.
215
+ module Helpers
216
+ [:utility, :command_name, :run].each do |name|
217
+ define_method name, lambda {|arg| Utilities.send(name, arg) }
218
+ private name
219
+ end
220
+ private
221
+ def gnu_tar?; Utilities.gnu_tar?; end
222
+ end
223
+ end
224
+ end
@@ -0,0 +1,5 @@
1
+ # encoding: utf-8
2
+
3
+ module Backup
4
+ VERSION = '4.4.1'
5
+ end
@@ -0,0 +1,28 @@
1
+ ##
2
+ # Archive [Archive]
3
+ #
4
+ # Adding a file or directory (including sub-directories):
5
+ # archive.add "/path/to/a/file.rb"
6
+ # archive.add "/path/to/a/directory/"
7
+ #
8
+ # Excluding a file or directory (including sub-directories):
9
+ # archive.exclude "/path/to/an/excluded_file.rb"
10
+ # archive.exclude "/path/to/an/excluded_directory
11
+ #
12
+ # By default, relative paths will be relative to the directory
13
+ # where `backup perform` is executed, and they will be expanded
14
+ # to the root of the filesystem when added to the archive.
15
+ #
16
+ # If a `root` path is set, relative paths will be relative to the
17
+ # given `root` path and will not be expanded when added to the archive.
18
+ #
19
+ # archive.root '/path/to/archive/root'
20
+ #
21
+ archive :my_archive do |archive|
22
+ # Run the `tar` command using `sudo`
23
+ # archive.use_sudo
24
+ archive.add "/path/to/a/file.rb"
25
+ archive.add "/path/to/a/folder/"
26
+ archive.exclude "/path/to/a/excluded_file.rb"
27
+ archive.exclude "/path/to/a/excluded_folder"
28
+ end
@@ -0,0 +1,4 @@
1
+ ##
2
+ # Bzip2 [Compressor]
3
+ #
4
+ compress_with Bzip2
@@ -0,0 +1,7 @@
1
+ ##
2
+ # Custom [Compressor]
3
+ #
4
+ compress_with Custom do |compressor|
5
+ compressor.command = 'gzip'
6
+ compressor.extension = '.gz'
7
+ end
@@ -0,0 +1,4 @@
1
+ ##
2
+ # Gzip [Compressor]
3
+ #
4
+ compress_with Gzip
@@ -0,0 +1,123 @@
1
+ # encoding: utf-8
2
+
3
+ ##
4
+ # Backup v<%= Backup::VERSION.split('.').first %>.x Configuration
5
+ #
6
+ # Documentation: http://backup.github.io/backup
7
+ # Issue Tracker: https://github.com/backup/backup/issues
8
+
9
+ ##
10
+ # Config Options
11
+ #
12
+ # The options here may be overridden on the command line, but the result
13
+ # will depend on the use of --root-path on the command line.
14
+ #
15
+ # If --root-path is used on the command line, then all paths set here
16
+ # will be overridden. If a path (like --tmp-path) is not given along with
17
+ # --root-path, that path will use it's default location _relative to --root-path_.
18
+ #
19
+ # If --root-path is not used on the command line, a path option (like --tmp-path)
20
+ # given on the command line will override the tmp_path set here, but all other
21
+ # paths set here will be used.
22
+ #
23
+ # Note that relative paths given on the command line without --root-path
24
+ # are relative to the current directory. The root_path set here only applies
25
+ # to relative paths set here.
26
+ #
27
+ # ---
28
+ #
29
+ # Sets the root path for all relative paths, including default paths.
30
+ # May be an absolute path, or relative to the current working directory.
31
+ #
32
+ # root_path 'my/root'
33
+ #
34
+ # Sets the path where backups are processed until they're stored.
35
+ # This must have enough free space to hold apx. 2 backups.
36
+ # May be an absolute path, or relative to the current directory or +root_path+.
37
+ #
38
+ # tmp_path 'my/tmp'
39
+ #
40
+ # Sets the path where backup stores persistent information.
41
+ # When Backup's Cycler is used, small YAML files are stored here.
42
+ # May be an absolute path, or relative to the current directory or +root_path+.
43
+ #
44
+ # data_path 'my/data'
45
+
46
+ ##
47
+ # Utilities
48
+ #
49
+ # If you need to use a utility other than the one Backup detects,
50
+ # or a utility can not be found in your $PATH.
51
+ #
52
+ # Utilities.configure do
53
+ # tar '/usr/bin/gnutar'
54
+ # redis_cli '/opt/redis/redis-cli'
55
+ # end
56
+
57
+ ##
58
+ # Logging
59
+ #
60
+ # Logging options may be set on the command line, but certain settings
61
+ # may only be configured here.
62
+ #
63
+ # Logger.configure do
64
+ # console.quiet = true # Same as command line: --quiet
65
+ # logfile.max_bytes = 2_000_000 # Default: 500_000
66
+ # syslog.enabled = true # Same as command line: --syslog
67
+ # syslog.ident = 'my_app_backup' # Default: 'backup'
68
+ # end
69
+ #
70
+ # Command line options will override those set here.
71
+ # For example, the following would override the example settings above
72
+ # to disable syslog and enable console output.
73
+ # backup perform --trigger my_backup --no-syslog --no-quiet
74
+
75
+ ##
76
+ # Component Defaults
77
+ #
78
+ # Set default options to be applied to components in all models.
79
+ # Options set within a model will override those set here.
80
+ #
81
+ # Storage::S3.defaults do |s3|
82
+ # s3.access_key_id = "my_access_key_id"
83
+ # s3.secret_access_key = "my_secret_access_key"
84
+ # end
85
+ #
86
+ # Notifier::Mail.defaults do |mail|
87
+ # mail.from = 'sender@email.com'
88
+ # mail.to = 'receiver@email.com'
89
+ # mail.address = 'smtp.gmail.com'
90
+ # mail.port = 587
91
+ # mail.domain = 'your.host.name'
92
+ # mail.user_name = 'sender@email.com'
93
+ # mail.password = 'my_password'
94
+ # mail.authentication = 'plain'
95
+ # mail.encryption = :starttls
96
+ # end
97
+
98
+ ##
99
+ # Preconfigured Models
100
+ #
101
+ # Create custom models with preconfigured components.
102
+ # Components added within the model definition will
103
+ # +add to+ the preconfigured components.
104
+ #
105
+ # preconfigure 'MyModel' do
106
+ # archive :user_pictures do |archive|
107
+ # archive.add '~/pictures'
108
+ # end
109
+ #
110
+ # notify_by Mail do |mail|
111
+ # mail.to = 'admin@email.com'
112
+ # end
113
+ # end
114
+ #
115
+ # MyModel.new(:john_smith, 'John Smith Backup') do
116
+ # archive :user_music do |archive|
117
+ # archive.add '~/music'
118
+ # end
119
+ #
120
+ # notify_by Mail do |mail|
121
+ # mail.to = 'john.smith@email.com'
122
+ # end
123
+ # end