cm-backup 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (133) hide show
  1. checksums.yaml +7 -0
  2. data/README.md +20 -0
  3. data/bin/backup +5 -0
  4. data/lib/backup.rb +144 -0
  5. data/lib/backup/archive.rb +170 -0
  6. data/lib/backup/binder.rb +22 -0
  7. data/lib/backup/cleaner.rb +116 -0
  8. data/lib/backup/cli.rb +374 -0
  9. data/lib/backup/cloud_io/base.rb +41 -0
  10. data/lib/backup/cloud_io/cloud_files.rb +298 -0
  11. data/lib/backup/cloud_io/s3.rb +260 -0
  12. data/lib/backup/compressor/base.rb +35 -0
  13. data/lib/backup/compressor/bzip2.rb +39 -0
  14. data/lib/backup/compressor/custom.rb +53 -0
  15. data/lib/backup/compressor/gzip.rb +74 -0
  16. data/lib/backup/config.rb +119 -0
  17. data/lib/backup/config/dsl.rb +103 -0
  18. data/lib/backup/config/helpers.rb +143 -0
  19. data/lib/backup/database/base.rb +85 -0
  20. data/lib/backup/database/mongodb.rb +187 -0
  21. data/lib/backup/database/mysql.rb +192 -0
  22. data/lib/backup/database/openldap.rb +95 -0
  23. data/lib/backup/database/postgresql.rb +133 -0
  24. data/lib/backup/database/redis.rb +179 -0
  25. data/lib/backup/database/riak.rb +82 -0
  26. data/lib/backup/database/sqlite.rb +57 -0
  27. data/lib/backup/encryptor/base.rb +29 -0
  28. data/lib/backup/encryptor/gpg.rb +747 -0
  29. data/lib/backup/encryptor/open_ssl.rb +77 -0
  30. data/lib/backup/errors.rb +58 -0
  31. data/lib/backup/logger.rb +199 -0
  32. data/lib/backup/logger/console.rb +51 -0
  33. data/lib/backup/logger/fog_adapter.rb +29 -0
  34. data/lib/backup/logger/logfile.rb +133 -0
  35. data/lib/backup/logger/syslog.rb +116 -0
  36. data/lib/backup/model.rb +479 -0
  37. data/lib/backup/notifier/base.rb +128 -0
  38. data/lib/backup/notifier/campfire.rb +63 -0
  39. data/lib/backup/notifier/command.rb +102 -0
  40. data/lib/backup/notifier/datadog.rb +107 -0
  41. data/lib/backup/notifier/flowdock.rb +103 -0
  42. data/lib/backup/notifier/hipchat.rb +118 -0
  43. data/lib/backup/notifier/http_post.rb +117 -0
  44. data/lib/backup/notifier/mail.rb +249 -0
  45. data/lib/backup/notifier/nagios.rb +69 -0
  46. data/lib/backup/notifier/pagerduty.rb +81 -0
  47. data/lib/backup/notifier/prowl.rb +68 -0
  48. data/lib/backup/notifier/pushover.rb +74 -0
  49. data/lib/backup/notifier/ses.rb +105 -0
  50. data/lib/backup/notifier/slack.rb +148 -0
  51. data/lib/backup/notifier/twitter.rb +58 -0
  52. data/lib/backup/notifier/zabbix.rb +63 -0
  53. data/lib/backup/package.rb +55 -0
  54. data/lib/backup/packager.rb +107 -0
  55. data/lib/backup/pipeline.rb +124 -0
  56. data/lib/backup/splitter.rb +76 -0
  57. data/lib/backup/storage/base.rb +69 -0
  58. data/lib/backup/storage/cloud_files.rb +158 -0
  59. data/lib/backup/storage/cycler.rb +75 -0
  60. data/lib/backup/storage/dropbox.rb +212 -0
  61. data/lib/backup/storage/ftp.rb +112 -0
  62. data/lib/backup/storage/local.rb +64 -0
  63. data/lib/backup/storage/qiniu.rb +65 -0
  64. data/lib/backup/storage/rsync.rb +248 -0
  65. data/lib/backup/storage/s3.rb +156 -0
  66. data/lib/backup/storage/scp.rb +67 -0
  67. data/lib/backup/storage/sftp.rb +82 -0
  68. data/lib/backup/syncer/base.rb +70 -0
  69. data/lib/backup/syncer/cloud/base.rb +179 -0
  70. data/lib/backup/syncer/cloud/cloud_files.rb +83 -0
  71. data/lib/backup/syncer/cloud/local_file.rb +100 -0
  72. data/lib/backup/syncer/cloud/s3.rb +110 -0
  73. data/lib/backup/syncer/rsync/base.rb +54 -0
  74. data/lib/backup/syncer/rsync/local.rb +31 -0
  75. data/lib/backup/syncer/rsync/pull.rb +51 -0
  76. data/lib/backup/syncer/rsync/push.rb +205 -0
  77. data/lib/backup/template.rb +46 -0
  78. data/lib/backup/utilities.rb +224 -0
  79. data/lib/backup/version.rb +5 -0
  80. data/templates/cli/archive +28 -0
  81. data/templates/cli/compressor/bzip2 +4 -0
  82. data/templates/cli/compressor/custom +7 -0
  83. data/templates/cli/compressor/gzip +4 -0
  84. data/templates/cli/config +123 -0
  85. data/templates/cli/databases/mongodb +15 -0
  86. data/templates/cli/databases/mysql +18 -0
  87. data/templates/cli/databases/openldap +24 -0
  88. data/templates/cli/databases/postgresql +16 -0
  89. data/templates/cli/databases/redis +16 -0
  90. data/templates/cli/databases/riak +17 -0
  91. data/templates/cli/databases/sqlite +11 -0
  92. data/templates/cli/encryptor/gpg +27 -0
  93. data/templates/cli/encryptor/openssl +9 -0
  94. data/templates/cli/model +26 -0
  95. data/templates/cli/notifier/zabbix +15 -0
  96. data/templates/cli/notifiers/campfire +12 -0
  97. data/templates/cli/notifiers/command +32 -0
  98. data/templates/cli/notifiers/datadog +57 -0
  99. data/templates/cli/notifiers/flowdock +16 -0
  100. data/templates/cli/notifiers/hipchat +16 -0
  101. data/templates/cli/notifiers/http_post +32 -0
  102. data/templates/cli/notifiers/mail +24 -0
  103. data/templates/cli/notifiers/nagios +13 -0
  104. data/templates/cli/notifiers/pagerduty +12 -0
  105. data/templates/cli/notifiers/prowl +11 -0
  106. data/templates/cli/notifiers/pushover +11 -0
  107. data/templates/cli/notifiers/ses +15 -0
  108. data/templates/cli/notifiers/slack +22 -0
  109. data/templates/cli/notifiers/twitter +13 -0
  110. data/templates/cli/splitter +7 -0
  111. data/templates/cli/storages/cloud_files +11 -0
  112. data/templates/cli/storages/dropbox +20 -0
  113. data/templates/cli/storages/ftp +13 -0
  114. data/templates/cli/storages/local +8 -0
  115. data/templates/cli/storages/qiniu +12 -0
  116. data/templates/cli/storages/rsync +17 -0
  117. data/templates/cli/storages/s3 +16 -0
  118. data/templates/cli/storages/scp +15 -0
  119. data/templates/cli/storages/sftp +15 -0
  120. data/templates/cli/syncers/cloud_files +22 -0
  121. data/templates/cli/syncers/rsync_local +20 -0
  122. data/templates/cli/syncers/rsync_pull +28 -0
  123. data/templates/cli/syncers/rsync_push +28 -0
  124. data/templates/cli/syncers/s3 +27 -0
  125. data/templates/general/links +3 -0
  126. data/templates/general/version.erb +2 -0
  127. data/templates/notifier/mail/failure.erb +16 -0
  128. data/templates/notifier/mail/success.erb +16 -0
  129. data/templates/notifier/mail/warning.erb +16 -0
  130. data/templates/storage/dropbox/authorization_url.erb +6 -0
  131. data/templates/storage/dropbox/authorized.erb +4 -0
  132. data/templates/storage/dropbox/cache_file_written.erb +10 -0
  133. metadata +1077 -0
@@ -0,0 +1,46 @@
1
+ # encoding: utf-8
2
+
3
+ require 'erb'
4
+
5
+ module Backup
6
+ class Template
7
+
8
+ # Holds a binding object. Nil if not provided.
9
+ attr_accessor :binding
10
+
11
+ ##
12
+ # Creates a new instance of the Backup::Template class
13
+ # and optionally takes an argument that can be either a binding object, a Hash or nil
14
+ def initialize(object = nil)
15
+ if object.is_a?(Binding)
16
+ @binding = object
17
+ elsif object.is_a?(Hash)
18
+ @binding = Backup::Binder.new(object).get_binding
19
+ else
20
+ @binding = nil
21
+ end
22
+ end
23
+
24
+ ##
25
+ # Renders the provided file (in the context of the binding if any) to the console
26
+ def render(file)
27
+ puts result(file)
28
+ end
29
+
30
+ ##
31
+ # Returns a String object containing the contents of the file (in the context of the binding if any)
32
+ def result(file)
33
+ ERB.new(file_contents(file), nil, '<>').result(binding)
34
+ end
35
+
36
+ private
37
+
38
+ ##
39
+ # Reads and returns the contents of the provided file path,
40
+ # relative from the Backup::TEMPLATE_PATH
41
+ def file_contents(file)
42
+ File.read(File.join(Backup::TEMPLATE_PATH, file))
43
+ end
44
+
45
+ end
46
+ end
@@ -0,0 +1,224 @@
1
+ # encoding: utf-8
2
+
3
+ module Backup
4
+ module Utilities
5
+ class Error < Backup::Error; end
6
+
7
+ UTILITY = {}
8
+ NAMES = %w{
9
+ tar cat split sudo chown hostname
10
+ gzip bzip2
11
+ mongo mongodump mysqldump innobackupex
12
+ pg_dump pg_dumpall redis-cli riak-admin
13
+ gpg openssl
14
+ rsync ssh
15
+ sendmail exim
16
+ send_nsca
17
+ zabbix_sender
18
+ }
19
+
20
+ module DSL
21
+ class << self
22
+ ##
23
+ # Allow users to set the path for all utilities in the .configure block.
24
+ #
25
+ # Utility names with dashes ('redis-cli') will be set using method calls
26
+ # with an underscore ('redis_cli').
27
+ NAMES.each do |name|
28
+ define_method name.gsub('-', '_'), lambda {|val|
29
+ path = File.expand_path(val)
30
+ unless File.executable?(path)
31
+ raise Utilities::Error, <<-EOS
32
+ The path given for '#{ name }' was not found or not executable.
33
+ Path was: #{ path }
34
+ EOS
35
+ end
36
+ UTILITY[name] = path
37
+ }
38
+ end
39
+
40
+ ##
41
+ # Allow users to set the +tar+ distribution if needed. (:gnu or :bsd)
42
+ def tar_dist(val)
43
+ Utilities.tar_dist(val)
44
+ end
45
+ end
46
+ end
47
+
48
+ class << self
49
+ ##
50
+ # Configure the path to system utilities used by Backup.
51
+ #
52
+ # Backup will attempt to locate any required system utilities using a
53
+ # +which+ command call. If a utility can not be found, or you need to
54
+ # specify an alternate path for a utility, you may do so in your
55
+ # +config.rb+ file using this method.
56
+ #
57
+ # Backup supports both GNU and BSD utilities.
58
+ # While Backup uses these utilities in a manner compatible with either
59
+ # version, the +tar+ utility requires some special handling with respect
60
+ # to +Archive+s. Backup will attempt to detect if the +tar+ command
61
+ # found (or set here) is GNU or BSD. If for some reason this fails,
62
+ # this may be set using the +tar_dist+ command shown below.
63
+ #
64
+ # Backup::Utilities.configure do
65
+ # # General Utilites
66
+ # tar '/path/to/tar'
67
+ # tar_dist :gnu # or :bsd
68
+ # cat '/path/to/cat'
69
+ # split '/path/to/split'
70
+ # sudo '/path/to/sudo'
71
+ # chown '/path/to/chown'
72
+ # hostname '/path/to/hostname'
73
+ #
74
+ # # Compressors
75
+ # gzip '/path/to/gzip'
76
+ # bzip2 '/path/to/bzip2'
77
+ #
78
+ # # Database Utilities
79
+ # mongo '/path/to/mongo'
80
+ # mongodump '/path/to/mongodump'
81
+ # mysqldump '/path/to/mysqldump'
82
+ # pg_dump '/path/to/pg_dump'
83
+ # pg_dumpall '/path/to/pg_dumpall'
84
+ # redis_cli '/path/to/redis-cli'
85
+ # riak_admin '/path/to/riak-admin'
86
+ #
87
+ # # Encryptors
88
+ # gpg '/path/to/gpg'
89
+ # openssl '/path/to/openssl'
90
+ #
91
+ # # Syncer and Storage
92
+ # rsync '/path/to/rsync'
93
+ # ssh '/path/to/ssh'
94
+ #
95
+ # # Notifiers
96
+ # sendmail '/path/to/sendmail'
97
+ # exim '/path/to/exim'
98
+ # send_nsca '/path/to/send_nsca'
99
+ # zabbix_sender '/path/to/zabbix_sender'
100
+ # end
101
+ #
102
+ # These paths may be set using absolute paths, or relative to the
103
+ # working directory when Backup is run.
104
+ def configure(&block)
105
+ DSL.instance_eval(&block)
106
+ end
107
+
108
+ def tar_dist(val)
109
+ # the acceptance tests need to be able to reset this to nil
110
+ @gnu_tar = val.nil? ? nil : val == :gnu
111
+ end
112
+
113
+ def gnu_tar?
114
+ return @gnu_tar unless @gnu_tar.nil?
115
+ @gnu_tar = !!run("#{ utility(:tar) } --version").match(/GNU/)
116
+ end
117
+
118
+ private
119
+
120
+ ##
121
+ # Returns the full path to the specified utility.
122
+ # Raises an error if utility can not be found in the system's $PATH
123
+ def utility(name)
124
+ name = name.to_s.strip
125
+ raise Error, 'Utility Name Empty' if name.empty?
126
+
127
+ UTILITY[name] ||= %x[which '#{ name }' 2>/dev/null].chomp
128
+ raise Error, <<-EOS if UTILITY[name].empty?
129
+ Could not locate '#{ name }'.
130
+ Make sure the specified utility is installed
131
+ and available in your system's $PATH, or specify it's location
132
+ in your 'config.rb' file using Backup::Utilities.configure
133
+ EOS
134
+
135
+ UTILITY[name].dup
136
+ end
137
+
138
+ ##
139
+ # Returns the name of the command name from the given command line.
140
+ # This is only used to simplify log messages.
141
+ def command_name(command)
142
+ parts = []
143
+ command = command.split(' ')
144
+ command.shift while command[0].to_s.include?('=')
145
+ parts << command.shift.split('/')[-1]
146
+ if parts[0] == 'sudo'
147
+ until command.empty?
148
+ part = command.shift
149
+ if part.include?('/')
150
+ parts << part.split('/')[-1]
151
+ break
152
+ else
153
+ parts << part
154
+ end
155
+ end
156
+ end
157
+ parts.join(' ')
158
+ end
159
+
160
+ ##
161
+ # Runs a system command
162
+ #
163
+ # All messages generated by the command will be logged.
164
+ # Messages on STDERR will be logged as warnings.
165
+ #
166
+ # If the command fails to execute, or returns a non-zero exit status
167
+ # an Error will be raised.
168
+ #
169
+ # Returns STDOUT
170
+ def run(command)
171
+ name = command_name(command)
172
+ Logger.info "Running system utility '#{ name }'..."
173
+
174
+ begin
175
+ out, err = '', ''
176
+ ps = Open4.popen4(command) do |pid, stdin, stdout, stderr|
177
+ stdin.close
178
+ out, err = stdout.read.strip, stderr.read.strip
179
+ end
180
+ rescue Exception => e
181
+ raise Error.wrap(e, "Failed to execute '#{ name }'")
182
+ end
183
+
184
+ if ps.success?
185
+ unless out.empty?
186
+ Logger.info(
187
+ out.lines.map {|line| "#{ name }:STDOUT: #{ line }" }.join
188
+ )
189
+ end
190
+
191
+ unless err.empty?
192
+ Logger.warn(
193
+ err.lines.map {|line| "#{ name }:STDERR: #{ line }" }.join
194
+ )
195
+ end
196
+
197
+ return out
198
+ else
199
+ raise Error, <<-EOS
200
+ '#{ name }' failed with exit status: #{ ps.exitstatus }
201
+ STDOUT Messages: #{ out.empty? ? 'None' : "\n#{ out }" }
202
+ STDERR Messages: #{ err.empty? ? 'None' : "\n#{ err }" }
203
+ EOS
204
+ end
205
+ end
206
+
207
+ def reset!
208
+ UTILITY.clear
209
+ @gnu_tar = nil
210
+ end
211
+ end
212
+
213
+ # Allows these utility methods to be included in other classes,
214
+ # while allowing them to be stubbed in spec_helper for all specs.
215
+ module Helpers
216
+ [:utility, :command_name, :run].each do |name|
217
+ define_method name, lambda {|arg| Utilities.send(name, arg) }
218
+ private name
219
+ end
220
+ private
221
+ def gnu_tar?; Utilities.gnu_tar?; end
222
+ end
223
+ end
224
+ end
@@ -0,0 +1,5 @@
1
+ # encoding: utf-8
2
+
3
+ module Backup
4
+ VERSION = '4.4.1'
5
+ end
@@ -0,0 +1,28 @@
1
+ ##
2
+ # Archive [Archive]
3
+ #
4
+ # Adding a file or directory (including sub-directories):
5
+ # archive.add "/path/to/a/file.rb"
6
+ # archive.add "/path/to/a/directory/"
7
+ #
8
+ # Excluding a file or directory (including sub-directories):
9
+ # archive.exclude "/path/to/an/excluded_file.rb"
10
+ # archive.exclude "/path/to/an/excluded_directory
11
+ #
12
+ # By default, relative paths will be relative to the directory
13
+ # where `backup perform` is executed, and they will be expanded
14
+ # to the root of the filesystem when added to the archive.
15
+ #
16
+ # If a `root` path is set, relative paths will be relative to the
17
+ # given `root` path and will not be expanded when added to the archive.
18
+ #
19
+ # archive.root '/path/to/archive/root'
20
+ #
21
+ archive :my_archive do |archive|
22
+ # Run the `tar` command using `sudo`
23
+ # archive.use_sudo
24
+ archive.add "/path/to/a/file.rb"
25
+ archive.add "/path/to/a/folder/"
26
+ archive.exclude "/path/to/a/excluded_file.rb"
27
+ archive.exclude "/path/to/a/excluded_folder"
28
+ end
@@ -0,0 +1,4 @@
1
+ ##
2
+ # Bzip2 [Compressor]
3
+ #
4
+ compress_with Bzip2
@@ -0,0 +1,7 @@
1
+ ##
2
+ # Custom [Compressor]
3
+ #
4
+ compress_with Custom do |compressor|
5
+ compressor.command = 'gzip'
6
+ compressor.extension = '.gz'
7
+ end
@@ -0,0 +1,4 @@
1
+ ##
2
+ # Gzip [Compressor]
3
+ #
4
+ compress_with Gzip
@@ -0,0 +1,123 @@
1
+ # encoding: utf-8
2
+
3
+ ##
4
+ # Backup v<%= Backup::VERSION.split('.').first %>.x Configuration
5
+ #
6
+ # Documentation: http://backup.github.io/backup
7
+ # Issue Tracker: https://github.com/backup/backup/issues
8
+
9
+ ##
10
+ # Config Options
11
+ #
12
+ # The options here may be overridden on the command line, but the result
13
+ # will depend on the use of --root-path on the command line.
14
+ #
15
+ # If --root-path is used on the command line, then all paths set here
16
+ # will be overridden. If a path (like --tmp-path) is not given along with
17
+ # --root-path, that path will use it's default location _relative to --root-path_.
18
+ #
19
+ # If --root-path is not used on the command line, a path option (like --tmp-path)
20
+ # given on the command line will override the tmp_path set here, but all other
21
+ # paths set here will be used.
22
+ #
23
+ # Note that relative paths given on the command line without --root-path
24
+ # are relative to the current directory. The root_path set here only applies
25
+ # to relative paths set here.
26
+ #
27
+ # ---
28
+ #
29
+ # Sets the root path for all relative paths, including default paths.
30
+ # May be an absolute path, or relative to the current working directory.
31
+ #
32
+ # root_path 'my/root'
33
+ #
34
+ # Sets the path where backups are processed until they're stored.
35
+ # This must have enough free space to hold apx. 2 backups.
36
+ # May be an absolute path, or relative to the current directory or +root_path+.
37
+ #
38
+ # tmp_path 'my/tmp'
39
+ #
40
+ # Sets the path where backup stores persistent information.
41
+ # When Backup's Cycler is used, small YAML files are stored here.
42
+ # May be an absolute path, or relative to the current directory or +root_path+.
43
+ #
44
+ # data_path 'my/data'
45
+
46
+ ##
47
+ # Utilities
48
+ #
49
+ # If you need to use a utility other than the one Backup detects,
50
+ # or a utility can not be found in your $PATH.
51
+ #
52
+ # Utilities.configure do
53
+ # tar '/usr/bin/gnutar'
54
+ # redis_cli '/opt/redis/redis-cli'
55
+ # end
56
+
57
+ ##
58
+ # Logging
59
+ #
60
+ # Logging options may be set on the command line, but certain settings
61
+ # may only be configured here.
62
+ #
63
+ # Logger.configure do
64
+ # console.quiet = true # Same as command line: --quiet
65
+ # logfile.max_bytes = 2_000_000 # Default: 500_000
66
+ # syslog.enabled = true # Same as command line: --syslog
67
+ # syslog.ident = 'my_app_backup' # Default: 'backup'
68
+ # end
69
+ #
70
+ # Command line options will override those set here.
71
+ # For example, the following would override the example settings above
72
+ # to disable syslog and enable console output.
73
+ # backup perform --trigger my_backup --no-syslog --no-quiet
74
+
75
+ ##
76
+ # Component Defaults
77
+ #
78
+ # Set default options to be applied to components in all models.
79
+ # Options set within a model will override those set here.
80
+ #
81
+ # Storage::S3.defaults do |s3|
82
+ # s3.access_key_id = "my_access_key_id"
83
+ # s3.secret_access_key = "my_secret_access_key"
84
+ # end
85
+ #
86
+ # Notifier::Mail.defaults do |mail|
87
+ # mail.from = 'sender@email.com'
88
+ # mail.to = 'receiver@email.com'
89
+ # mail.address = 'smtp.gmail.com'
90
+ # mail.port = 587
91
+ # mail.domain = 'your.host.name'
92
+ # mail.user_name = 'sender@email.com'
93
+ # mail.password = 'my_password'
94
+ # mail.authentication = 'plain'
95
+ # mail.encryption = :starttls
96
+ # end
97
+
98
+ ##
99
+ # Preconfigured Models
100
+ #
101
+ # Create custom models with preconfigured components.
102
+ # Components added within the model definition will
103
+ # +add to+ the preconfigured components.
104
+ #
105
+ # preconfigure 'MyModel' do
106
+ # archive :user_pictures do |archive|
107
+ # archive.add '~/pictures'
108
+ # end
109
+ #
110
+ # notify_by Mail do |mail|
111
+ # mail.to = 'admin@email.com'
112
+ # end
113
+ # end
114
+ #
115
+ # MyModel.new(:john_smith, 'John Smith Backup') do
116
+ # archive :user_music do |archive|
117
+ # archive.add '~/music'
118
+ # end
119
+ #
120
+ # notify_by Mail do |mail|
121
+ # mail.to = 'john.smith@email.com'
122
+ # end
123
+ # end