backup-bouchard 4.4.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (134) hide show
  1. checksums.yaml +7 -0
  2. data/LICENSE +19 -0
  3. data/README.md +29 -0
  4. data/bin/backup +5 -0
  5. data/lib/backup.rb +140 -0
  6. data/lib/backup/archive.rb +169 -0
  7. data/lib/backup/binder.rb +18 -0
  8. data/lib/backup/cleaner.rb +112 -0
  9. data/lib/backup/cli.rb +370 -0
  10. data/lib/backup/cloud_io/base.rb +38 -0
  11. data/lib/backup/cloud_io/cloud_files.rb +296 -0
  12. data/lib/backup/cloud_io/s3.rb +253 -0
  13. data/lib/backup/compressor/base.rb +32 -0
  14. data/lib/backup/compressor/bzip2.rb +35 -0
  15. data/lib/backup/compressor/custom.rb +49 -0
  16. data/lib/backup/compressor/gzip.rb +73 -0
  17. data/lib/backup/config.rb +118 -0
  18. data/lib/backup/config/dsl.rb +100 -0
  19. data/lib/backup/config/helpers.rb +137 -0
  20. data/lib/backup/database/base.rb +86 -0
  21. data/lib/backup/database/mongodb.rb +187 -0
  22. data/lib/backup/database/mysql.rb +191 -0
  23. data/lib/backup/database/openldap.rb +93 -0
  24. data/lib/backup/database/postgresql.rb +132 -0
  25. data/lib/backup/database/redis.rb +177 -0
  26. data/lib/backup/database/riak.rb +79 -0
  27. data/lib/backup/database/sqlite.rb +55 -0
  28. data/lib/backup/encryptor/base.rb +27 -0
  29. data/lib/backup/encryptor/gpg.rb +740 -0
  30. data/lib/backup/encryptor/open_ssl.rb +74 -0
  31. data/lib/backup/errors.rb +53 -0
  32. data/lib/backup/logger.rb +197 -0
  33. data/lib/backup/logger/console.rb +48 -0
  34. data/lib/backup/logger/fog_adapter.rb +25 -0
  35. data/lib/backup/logger/logfile.rb +131 -0
  36. data/lib/backup/logger/syslog.rb +114 -0
  37. data/lib/backup/model.rb +477 -0
  38. data/lib/backup/notifier/base.rb +126 -0
  39. data/lib/backup/notifier/campfire.rb +61 -0
  40. data/lib/backup/notifier/command.rb +99 -0
  41. data/lib/backup/notifier/datadog.rb +104 -0
  42. data/lib/backup/notifier/flowdock.rb +99 -0
  43. data/lib/backup/notifier/hipchat.rb +116 -0
  44. data/lib/backup/notifier/http_post.rb +114 -0
  45. data/lib/backup/notifier/mail.rb +246 -0
  46. data/lib/backup/notifier/nagios.rb +65 -0
  47. data/lib/backup/notifier/pagerduty.rb +79 -0
  48. data/lib/backup/notifier/prowl.rb +68 -0
  49. data/lib/backup/notifier/pushover.rb +71 -0
  50. data/lib/backup/notifier/ses.rb +103 -0
  51. data/lib/backup/notifier/slack.rb +147 -0
  52. data/lib/backup/notifier/twitter.rb +55 -0
  53. data/lib/backup/notifier/zabbix.rb +60 -0
  54. data/lib/backup/package.rb +51 -0
  55. data/lib/backup/packager.rb +105 -0
  56. data/lib/backup/pipeline.rb +120 -0
  57. data/lib/backup/splitter.rb +73 -0
  58. data/lib/backup/storage/base.rb +66 -0
  59. data/lib/backup/storage/cloud_files.rb +156 -0
  60. data/lib/backup/storage/cycler.rb +70 -0
  61. data/lib/backup/storage/dropbox.rb +210 -0
  62. data/lib/backup/storage/ftp.rb +110 -0
  63. data/lib/backup/storage/local.rb +61 -0
  64. data/lib/backup/storage/qiniu.rb +65 -0
  65. data/lib/backup/storage/rsync.rb +246 -0
  66. data/lib/backup/storage/s3.rb +155 -0
  67. data/lib/backup/storage/scp.rb +65 -0
  68. data/lib/backup/storage/sftp.rb +80 -0
  69. data/lib/backup/syncer/base.rb +67 -0
  70. data/lib/backup/syncer/cloud/base.rb +176 -0
  71. data/lib/backup/syncer/cloud/cloud_files.rb +81 -0
  72. data/lib/backup/syncer/cloud/local_file.rb +97 -0
  73. data/lib/backup/syncer/cloud/s3.rb +109 -0
  74. data/lib/backup/syncer/rsync/base.rb +50 -0
  75. data/lib/backup/syncer/rsync/local.rb +27 -0
  76. data/lib/backup/syncer/rsync/pull.rb +47 -0
  77. data/lib/backup/syncer/rsync/push.rb +201 -0
  78. data/lib/backup/template.rb +41 -0
  79. data/lib/backup/utilities.rb +228 -0
  80. data/lib/backup/version.rb +3 -0
  81. data/templates/cli/archive +28 -0
  82. data/templates/cli/compressor/bzip2 +4 -0
  83. data/templates/cli/compressor/custom +7 -0
  84. data/templates/cli/compressor/gzip +4 -0
  85. data/templates/cli/config +123 -0
  86. data/templates/cli/databases/mongodb +15 -0
  87. data/templates/cli/databases/mysql +18 -0
  88. data/templates/cli/databases/openldap +24 -0
  89. data/templates/cli/databases/postgresql +16 -0
  90. data/templates/cli/databases/redis +16 -0
  91. data/templates/cli/databases/riak +17 -0
  92. data/templates/cli/databases/sqlite +11 -0
  93. data/templates/cli/encryptor/gpg +27 -0
  94. data/templates/cli/encryptor/openssl +9 -0
  95. data/templates/cli/model +26 -0
  96. data/templates/cli/notifier/zabbix +15 -0
  97. data/templates/cli/notifiers/campfire +12 -0
  98. data/templates/cli/notifiers/command +32 -0
  99. data/templates/cli/notifiers/datadog +57 -0
  100. data/templates/cli/notifiers/flowdock +16 -0
  101. data/templates/cli/notifiers/hipchat +16 -0
  102. data/templates/cli/notifiers/http_post +32 -0
  103. data/templates/cli/notifiers/mail +24 -0
  104. data/templates/cli/notifiers/nagios +13 -0
  105. data/templates/cli/notifiers/pagerduty +12 -0
  106. data/templates/cli/notifiers/prowl +11 -0
  107. data/templates/cli/notifiers/pushover +11 -0
  108. data/templates/cli/notifiers/ses +15 -0
  109. data/templates/cli/notifiers/slack +22 -0
  110. data/templates/cli/notifiers/twitter +13 -0
  111. data/templates/cli/splitter +7 -0
  112. data/templates/cli/storages/cloud_files +11 -0
  113. data/templates/cli/storages/dropbox +20 -0
  114. data/templates/cli/storages/ftp +13 -0
  115. data/templates/cli/storages/local +8 -0
  116. data/templates/cli/storages/qiniu +12 -0
  117. data/templates/cli/storages/rsync +17 -0
  118. data/templates/cli/storages/s3 +16 -0
  119. data/templates/cli/storages/scp +15 -0
  120. data/templates/cli/storages/sftp +15 -0
  121. data/templates/cli/syncers/cloud_files +22 -0
  122. data/templates/cli/syncers/rsync_local +20 -0
  123. data/templates/cli/syncers/rsync_pull +28 -0
  124. data/templates/cli/syncers/rsync_push +28 -0
  125. data/templates/cli/syncers/s3 +27 -0
  126. data/templates/general/links +3 -0
  127. data/templates/general/version.erb +2 -0
  128. data/templates/notifier/mail/failure.erb +16 -0
  129. data/templates/notifier/mail/success.erb +16 -0
  130. data/templates/notifier/mail/warning.erb +16 -0
  131. data/templates/storage/dropbox/authorization_url.erb +6 -0
  132. data/templates/storage/dropbox/authorized.erb +4 -0
  133. data/templates/storage/dropbox/cache_file_written.erb +10 -0
  134. metadata +518 -0
@@ -0,0 +1,41 @@
1
+ require "erb"
2
+
3
+ module Backup
4
+ class Template
5
+ # Holds a binding object. Nil if not provided.
6
+ attr_accessor :binding
7
+
8
+ ##
9
+ # Creates a new instance of the Backup::Template class
10
+ # and optionally takes an argument that can be either a binding object, a Hash or nil
11
+ def initialize(object = nil)
12
+ @binding =
13
+ if object.is_a?(Binding)
14
+ object
15
+ elsif object.is_a?(Hash)
16
+ Backup::Binder.new(object).get_binding
17
+ end
18
+ end
19
+
20
+ ##
21
+ # Renders the provided file (in the context of the binding if any) to the console
22
+ def render(file)
23
+ puts result(file)
24
+ end
25
+
26
+ ##
27
+ # Returns a String object containing the contents of the file (in the context of the binding if any)
28
+ def result(file)
29
+ ERB.new(file_contents(file), nil, "<>").result(binding)
30
+ end
31
+
32
+ private
33
+
34
+ ##
35
+ # Reads and returns the contents of the provided file path,
36
+ # relative from the Backup::TEMPLATE_PATH
37
+ def file_contents(file)
38
+ File.read(File.join(Backup::TEMPLATE_PATH, file))
39
+ end
40
+ end
41
+ end
@@ -0,0 +1,228 @@
1
+ module Backup
2
+ module Utilities
3
+ class Error < Backup::Error; end
4
+
5
+ UTILITY = {}
6
+ NAMES = %w(
7
+ tar cat split sudo chown hostname
8
+ gzip bzip2
9
+ mongo mongodump mysqldump innobackupex
10
+ pg_dump pg_dumpall redis-cli riak-admin
11
+ gpg openssl
12
+ rsync ssh
13
+ sendmail exim
14
+ send_nsca
15
+ zabbix_sender
16
+ )
17
+
18
+ module DSL
19
+ class << self
20
+ ##
21
+ # Allow users to set the path for all utilities in the .configure block.
22
+ #
23
+ # Utility names with dashes ('redis-cli') will be set using method calls
24
+ # with an underscore ('redis_cli').
25
+ NAMES.each do |name|
26
+ define_method name.tr("-", "_"), lambda { |val|
27
+ path = File.expand_path(val)
28
+ unless File.executable?(path)
29
+ raise Utilities::Error, <<-EOS
30
+ The path given for '#{name}' was not found or not executable.
31
+ Path was: #{path}
32
+ EOS
33
+ end
34
+ UTILITY[name] = path
35
+ }
36
+ end
37
+
38
+ ##
39
+ # Allow users to set the +tar+ distribution if needed. (:gnu or :bsd)
40
+ def tar_dist(val)
41
+ Utilities.tar_dist(val)
42
+ end
43
+ end
44
+ end
45
+
46
+ class << self
47
+ ##
48
+ # Configure the path to system utilities used by Backup.
49
+ #
50
+ # Backup will attempt to locate any required system utilities using a
51
+ # +which+ command call. If a utility can not be found, or you need to
52
+ # specify an alternate path for a utility, you may do so in your
53
+ # +config.rb+ file using this method.
54
+ #
55
+ # Backup supports both GNU and BSD utilities.
56
+ # While Backup uses these utilities in a manner compatible with either
57
+ # version, the +tar+ utility requires some special handling with respect
58
+ # to +Archive+s. Backup will attempt to detect if the +tar+ command
59
+ # found (or set here) is GNU or BSD. If for some reason this fails,
60
+ # this may be set using the +tar_dist+ command shown below.
61
+ #
62
+ # Backup::Utilities.configure do
63
+ # # General Utilites
64
+ # tar '/path/to/tar'
65
+ # tar_dist :gnu # or :bsd
66
+ # cat '/path/to/cat'
67
+ # split '/path/to/split'
68
+ # sudo '/path/to/sudo'
69
+ # chown '/path/to/chown'
70
+ # hostname '/path/to/hostname'
71
+ #
72
+ # # Compressors
73
+ # gzip '/path/to/gzip'
74
+ # bzip2 '/path/to/bzip2'
75
+ #
76
+ # # Database Utilities
77
+ # mongo '/path/to/mongo'
78
+ # mongodump '/path/to/mongodump'
79
+ # mysqldump '/path/to/mysqldump'
80
+ # pg_dump '/path/to/pg_dump'
81
+ # pg_dumpall '/path/to/pg_dumpall'
82
+ # redis_cli '/path/to/redis-cli'
83
+ # riak_admin '/path/to/riak-admin'
84
+ #
85
+ # # Encryptors
86
+ # gpg '/path/to/gpg'
87
+ # openssl '/path/to/openssl'
88
+ #
89
+ # # Syncer and Storage
90
+ # rsync '/path/to/rsync'
91
+ # ssh '/path/to/ssh'
92
+ #
93
+ # # Notifiers
94
+ # sendmail '/path/to/sendmail'
95
+ # exim '/path/to/exim'
96
+ # send_nsca '/path/to/send_nsca'
97
+ # zabbix_sender '/path/to/zabbix_sender'
98
+ # end
99
+ #
100
+ # These paths may be set using absolute paths, or relative to the
101
+ # working directory when Backup is run.
102
+ def configure(&block)
103
+ DSL.instance_eval(&block)
104
+ end
105
+
106
+ def tar_dist(val)
107
+ # the acceptance tests need to be able to reset this to nil
108
+ @gnu_tar = val.nil? ? nil : val == :gnu
109
+ end
110
+
111
+ def gnu_tar?
112
+ return @gnu_tar unless @gnu_tar.nil?
113
+ @gnu_tar = !!run("#{utility(:tar)} --version").match(/GNU/)
114
+ end
115
+
116
+ private
117
+
118
+ ##
119
+ # Returns the full path to the specified utility.
120
+ # Raises an error if utility can not be found in the system's $PATH
121
+ def utility(name)
122
+ name = name.to_s.strip
123
+ raise Error, "Utility Name Empty" if name.empty?
124
+
125
+ UTILITY[name] ||= `which '#{name}' 2>/dev/null`.chomp
126
+ raise Error, <<-EOS if UTILITY[name].empty?
127
+ Could not locate '#{name}'.
128
+ Make sure the specified utility is installed
129
+ and available in your system's $PATH, or specify it's location
130
+ in your 'config.rb' file using Backup::Utilities.configure
131
+ EOS
132
+
133
+ UTILITY[name].dup
134
+ end
135
+
136
+ ##
137
+ # Returns the name of the command name from the given command line.
138
+ # This is only used to simplify log messages.
139
+ def command_name(command)
140
+ parts = []
141
+ command = command.split(" ")
142
+ command.shift while command[0].to_s.include?("=")
143
+ parts << command.shift.split("/")[-1]
144
+ if parts[0] == "sudo"
145
+ until command.empty?
146
+ part = command.shift
147
+ if part.include?("/")
148
+ parts << part.split("/")[-1]
149
+ break
150
+ else
151
+ parts << part
152
+ end
153
+ end
154
+ end
155
+ parts.join(" ")
156
+ end
157
+
158
+ ##
159
+ # Runs a system command
160
+ #
161
+ # All messages generated by the command will be logged.
162
+ # Messages on STDERR will be logged as warnings.
163
+ #
164
+ # If the command fails to execute, or returns a non-zero exit status
165
+ # an Error will be raised.
166
+ #
167
+ # Returns STDOUT
168
+ def run(command)
169
+ name = command_name(command)
170
+ Logger.info "Running system utility '#{name}'..."
171
+
172
+ begin
173
+ out = ""
174
+ err = ""
175
+ ps = Open4.popen4(command) do |_pid, stdin, stdout, stderr|
176
+ stdin.close
177
+ out = stdout.read.strip
178
+ err = stderr.read.strip
179
+ end
180
+ rescue Exception => e
181
+ raise Error.wrap(e, "Failed to execute '#{name}'")
182
+ end
183
+
184
+ if ps.success?
185
+ unless out.empty?
186
+ Logger.info(
187
+ out.lines.map { |line| "#{name}:STDOUT: #{line}" }.join
188
+ )
189
+ end
190
+
191
+ unless err.empty?
192
+ Logger.warn(
193
+ err.lines.map { |line| "#{name}:STDERR: #{line}" }.join
194
+ )
195
+ end
196
+
197
+ return out
198
+ else
199
+ raise Error, <<-EOS
200
+ '#{name}' failed with exit status: #{ps.exitstatus}
201
+ STDOUT Messages: #{out.empty? ? "None" : "\n#{out}"}
202
+ STDERR Messages: #{err.empty? ? "None" : "\n#{err}"}
203
+ EOS
204
+ end
205
+ end
206
+
207
+ def reset!
208
+ UTILITY.clear
209
+ @gnu_tar = nil
210
+ end
211
+ end
212
+
213
+ # Allows these utility methods to be included in other classes,
214
+ # while allowing them to be stubbed in spec_helper for all specs.
215
+ module Helpers
216
+ [:utility, :command_name, :run].each do |name|
217
+ define_method name, ->(arg) { Utilities.send(name, arg) }
218
+ private name
219
+ end
220
+
221
+ private
222
+
223
+ def gnu_tar?
224
+ Utilities.gnu_tar?
225
+ end
226
+ end
227
+ end
228
+ end
@@ -0,0 +1,3 @@
1
+ module Backup
2
+ VERSION = "4.4.1"
3
+ end
@@ -0,0 +1,28 @@
1
+ ##
2
+ # Archive [Archive]
3
+ #
4
+ # Adding a file or directory (including sub-directories):
5
+ # archive.add "/path/to/a/file.rb"
6
+ # archive.add "/path/to/a/directory/"
7
+ #
8
+ # Excluding a file or directory (including sub-directories):
9
+ # archive.exclude "/path/to/an/excluded_file.rb"
10
+ # archive.exclude "/path/to/an/excluded_directory
11
+ #
12
+ # By default, relative paths will be relative to the directory
13
+ # where `backup perform` is executed, and they will be expanded
14
+ # to the root of the filesystem when added to the archive.
15
+ #
16
+ # If a `root` path is set, relative paths will be relative to the
17
+ # given `root` path and will not be expanded when added to the archive.
18
+ #
19
+ # archive.root '/path/to/archive/root'
20
+ #
21
+ archive :my_archive do |archive|
22
+ # Run the `tar` command using `sudo`
23
+ # archive.use_sudo
24
+ archive.add "/path/to/a/file.rb"
25
+ archive.add "/path/to/a/folder/"
26
+ archive.exclude "/path/to/a/excluded_file.rb"
27
+ archive.exclude "/path/to/a/excluded_folder"
28
+ end
@@ -0,0 +1,4 @@
1
+ ##
2
+ # Bzip2 [Compressor]
3
+ #
4
+ compress_with Bzip2
@@ -0,0 +1,7 @@
1
+ ##
2
+ # Custom [Compressor]
3
+ #
4
+ compress_with Custom do |compressor|
5
+ compressor.command = 'gzip'
6
+ compressor.extension = '.gz'
7
+ end
@@ -0,0 +1,4 @@
1
+ ##
2
+ # Gzip [Compressor]
3
+ #
4
+ compress_with Gzip
@@ -0,0 +1,123 @@
1
+ # encoding: utf-8
2
+
3
+ ##
4
+ # Backup v<%= Backup::VERSION.split('.').first %>.x Configuration
5
+ #
6
+ # Documentation: http://backup.github.io/backup
7
+ # Issue Tracker: https://github.com/backup/backup/issues
8
+
9
+ ##
10
+ # Config Options
11
+ #
12
+ # The options here may be overridden on the command line, but the result
13
+ # will depend on the use of --root-path on the command line.
14
+ #
15
+ # If --root-path is used on the command line, then all paths set here
16
+ # will be overridden. If a path (like --tmp-path) is not given along with
17
+ # --root-path, that path will use it's default location _relative to --root-path_.
18
+ #
19
+ # If --root-path is not used on the command line, a path option (like --tmp-path)
20
+ # given on the command line will override the tmp_path set here, but all other
21
+ # paths set here will be used.
22
+ #
23
+ # Note that relative paths given on the command line without --root-path
24
+ # are relative to the current directory. The root_path set here only applies
25
+ # to relative paths set here.
26
+ #
27
+ # ---
28
+ #
29
+ # Sets the root path for all relative paths, including default paths.
30
+ # May be an absolute path, or relative to the current working directory.
31
+ #
32
+ # root_path 'my/root'
33
+ #
34
+ # Sets the path where backups are processed until they're stored.
35
+ # This must have enough free space to hold apx. 2 backups.
36
+ # May be an absolute path, or relative to the current directory or +root_path+.
37
+ #
38
+ # tmp_path 'my/tmp'
39
+ #
40
+ # Sets the path where backup stores persistent information.
41
+ # When Backup's Cycler is used, small YAML files are stored here.
42
+ # May be an absolute path, or relative to the current directory or +root_path+.
43
+ #
44
+ # data_path 'my/data'
45
+
46
+ ##
47
+ # Utilities
48
+ #
49
+ # If you need to use a utility other than the one Backup detects,
50
+ # or a utility can not be found in your $PATH.
51
+ #
52
+ # Utilities.configure do
53
+ # tar '/usr/bin/gnutar'
54
+ # redis_cli '/opt/redis/redis-cli'
55
+ # end
56
+
57
+ ##
58
+ # Logging
59
+ #
60
+ # Logging options may be set on the command line, but certain settings
61
+ # may only be configured here.
62
+ #
63
+ # Logger.configure do
64
+ # console.quiet = true # Same as command line: --quiet
65
+ # logfile.max_bytes = 2_000_000 # Default: 500_000
66
+ # syslog.enabled = true # Same as command line: --syslog
67
+ # syslog.ident = 'my_app_backup' # Default: 'backup'
68
+ # end
69
+ #
70
+ # Command line options will override those set here.
71
+ # For example, the following would override the example settings above
72
+ # to disable syslog and enable console output.
73
+ # backup perform --trigger my_backup --no-syslog --no-quiet
74
+
75
+ ##
76
+ # Component Defaults
77
+ #
78
+ # Set default options to be applied to components in all models.
79
+ # Options set within a model will override those set here.
80
+ #
81
+ # Storage::S3.defaults do |s3|
82
+ # s3.access_key_id = "my_access_key_id"
83
+ # s3.secret_access_key = "my_secret_access_key"
84
+ # end
85
+ #
86
+ # Notifier::Mail.defaults do |mail|
87
+ # mail.from = 'sender@email.com'
88
+ # mail.to = 'receiver@email.com'
89
+ # mail.address = 'smtp.gmail.com'
90
+ # mail.port = 587
91
+ # mail.domain = 'your.host.name'
92
+ # mail.user_name = 'sender@email.com'
93
+ # mail.password = 'my_password'
94
+ # mail.authentication = 'plain'
95
+ # mail.encryption = :starttls
96
+ # end
97
+
98
+ ##
99
+ # Preconfigured Models
100
+ #
101
+ # Create custom models with preconfigured components.
102
+ # Components added within the model definition will
103
+ # +add to+ the preconfigured components.
104
+ #
105
+ # preconfigure 'MyModel' do
106
+ # archive :user_pictures do |archive|
107
+ # archive.add '~/pictures'
108
+ # end
109
+ #
110
+ # notify_by Mail do |mail|
111
+ # mail.to = 'admin@email.com'
112
+ # end
113
+ # end
114
+ #
115
+ # MyModel.new(:john_smith, 'John Smith Backup') do
116
+ # archive :user_music do |archive|
117
+ # archive.add '~/music'
118
+ # end
119
+ #
120
+ # notify_by Mail do |mail|
121
+ # mail.to = 'john.smith@email.com'
122
+ # end
123
+ # end