nfm-backup 4.0.1a

Sign up to get free protection for your applications and to get access to all the features.
Files changed (117) hide show
  1. checksums.yaml +7 -0
  2. data/LICENSE.md +24 -0
  3. data/README.md +20 -0
  4. data/bin/backup +5 -0
  5. data/lib/backup.rb +133 -0
  6. data/lib/backup/archive.rb +170 -0
  7. data/lib/backup/binder.rb +22 -0
  8. data/lib/backup/cleaner.rb +116 -0
  9. data/lib/backup/cli.rb +364 -0
  10. data/lib/backup/cloud_io/base.rb +41 -0
  11. data/lib/backup/cloud_io/cloud_files.rb +298 -0
  12. data/lib/backup/cloud_io/s3.rb +260 -0
  13. data/lib/backup/compressor/base.rb +35 -0
  14. data/lib/backup/compressor/bzip2.rb +39 -0
  15. data/lib/backup/compressor/custom.rb +53 -0
  16. data/lib/backup/compressor/gzip.rb +74 -0
  17. data/lib/backup/config.rb +119 -0
  18. data/lib/backup/config/dsl.rb +102 -0
  19. data/lib/backup/config/helpers.rb +143 -0
  20. data/lib/backup/database/base.rb +85 -0
  21. data/lib/backup/database/mongodb.rb +186 -0
  22. data/lib/backup/database/mysql.rb +123 -0
  23. data/lib/backup/database/postgresql.rb +133 -0
  24. data/lib/backup/database/redis.rb +179 -0
  25. data/lib/backup/database/riak.rb +82 -0
  26. data/lib/backup/encryptor/base.rb +29 -0
  27. data/lib/backup/encryptor/gpg.rb +747 -0
  28. data/lib/backup/encryptor/open_ssl.rb +72 -0
  29. data/lib/backup/errors.rb +58 -0
  30. data/lib/backup/logger.rb +199 -0
  31. data/lib/backup/logger/console.rb +51 -0
  32. data/lib/backup/logger/fog_adapter.rb +29 -0
  33. data/lib/backup/logger/logfile.rb +119 -0
  34. data/lib/backup/logger/syslog.rb +116 -0
  35. data/lib/backup/model.rb +454 -0
  36. data/lib/backup/notifier/base.rb +98 -0
  37. data/lib/backup/notifier/campfire.rb +69 -0
  38. data/lib/backup/notifier/hipchat.rb +93 -0
  39. data/lib/backup/notifier/http_post.rb +122 -0
  40. data/lib/backup/notifier/mail.rb +238 -0
  41. data/lib/backup/notifier/nagios.rb +69 -0
  42. data/lib/backup/notifier/prowl.rb +69 -0
  43. data/lib/backup/notifier/pushover.rb +80 -0
  44. data/lib/backup/notifier/slack.rb +149 -0
  45. data/lib/backup/notifier/twitter.rb +65 -0
  46. data/lib/backup/package.rb +51 -0
  47. data/lib/backup/packager.rb +101 -0
  48. data/lib/backup/pipeline.rb +124 -0
  49. data/lib/backup/splitter.rb +76 -0
  50. data/lib/backup/storage/base.rb +57 -0
  51. data/lib/backup/storage/cloud_files.rb +158 -0
  52. data/lib/backup/storage/cycler.rb +65 -0
  53. data/lib/backup/storage/dropbox.rb +236 -0
  54. data/lib/backup/storage/ftp.rb +98 -0
  55. data/lib/backup/storage/local.rb +64 -0
  56. data/lib/backup/storage/ninefold.rb +74 -0
  57. data/lib/backup/storage/rsync.rb +248 -0
  58. data/lib/backup/storage/s3.rb +154 -0
  59. data/lib/backup/storage/scp.rb +67 -0
  60. data/lib/backup/storage/sftp.rb +82 -0
  61. data/lib/backup/syncer/base.rb +70 -0
  62. data/lib/backup/syncer/cloud/base.rb +179 -0
  63. data/lib/backup/syncer/cloud/cloud_files.rb +83 -0
  64. data/lib/backup/syncer/cloud/local_file.rb +100 -0
  65. data/lib/backup/syncer/cloud/s3.rb +110 -0
  66. data/lib/backup/syncer/rsync/base.rb +48 -0
  67. data/lib/backup/syncer/rsync/local.rb +31 -0
  68. data/lib/backup/syncer/rsync/pull.rb +51 -0
  69. data/lib/backup/syncer/rsync/push.rb +205 -0
  70. data/lib/backup/template.rb +46 -0
  71. data/lib/backup/utilities.rb +221 -0
  72. data/lib/backup/version.rb +5 -0
  73. data/templates/cli/archive +28 -0
  74. data/templates/cli/compressor/bzip2 +4 -0
  75. data/templates/cli/compressor/custom +7 -0
  76. data/templates/cli/compressor/gzip +4 -0
  77. data/templates/cli/config +123 -0
  78. data/templates/cli/databases/mongodb +15 -0
  79. data/templates/cli/databases/mysql +18 -0
  80. data/templates/cli/databases/postgresql +16 -0
  81. data/templates/cli/databases/redis +16 -0
  82. data/templates/cli/databases/riak +17 -0
  83. data/templates/cli/encryptor/gpg +27 -0
  84. data/templates/cli/encryptor/openssl +9 -0
  85. data/templates/cli/model +26 -0
  86. data/templates/cli/notifiers/campfire +12 -0
  87. data/templates/cli/notifiers/hipchat +15 -0
  88. data/templates/cli/notifiers/http_post +32 -0
  89. data/templates/cli/notifiers/mail +21 -0
  90. data/templates/cli/notifiers/nagios +13 -0
  91. data/templates/cli/notifiers/prowl +11 -0
  92. data/templates/cli/notifiers/pushover +11 -0
  93. data/templates/cli/notifiers/twitter +13 -0
  94. data/templates/cli/splitter +7 -0
  95. data/templates/cli/storages/cloud_files +11 -0
  96. data/templates/cli/storages/dropbox +19 -0
  97. data/templates/cli/storages/ftp +12 -0
  98. data/templates/cli/storages/local +7 -0
  99. data/templates/cli/storages/ninefold +9 -0
  100. data/templates/cli/storages/rsync +17 -0
  101. data/templates/cli/storages/s3 +14 -0
  102. data/templates/cli/storages/scp +14 -0
  103. data/templates/cli/storages/sftp +14 -0
  104. data/templates/cli/syncers/cloud_files +22 -0
  105. data/templates/cli/syncers/rsync_local +20 -0
  106. data/templates/cli/syncers/rsync_pull +28 -0
  107. data/templates/cli/syncers/rsync_push +28 -0
  108. data/templates/cli/syncers/s3 +27 -0
  109. data/templates/general/links +3 -0
  110. data/templates/general/version.erb +2 -0
  111. data/templates/notifier/mail/failure.erb +16 -0
  112. data/templates/notifier/mail/success.erb +16 -0
  113. data/templates/notifier/mail/warning.erb +16 -0
  114. data/templates/storage/dropbox/authorization_url.erb +6 -0
  115. data/templates/storage/dropbox/authorized.erb +4 -0
  116. data/templates/storage/dropbox/cache_file_written.erb +10 -0
  117. metadata +688 -0
@@ -0,0 +1,46 @@
1
+ # encoding: utf-8
2
+
3
+ require 'erb'
4
+
5
+ module Backup
6
+ class Template
7
+
8
+ # Holds a binding object. Nil if not provided.
9
+ attr_accessor :binding
10
+
11
+ ##
12
+ # Creates a new instance of the Backup::Template class
13
+ # and optionally takes an argument that can be either a binding object, a Hash or nil
14
+ def initialize(object = nil)
15
+ if object.is_a?(Binding)
16
+ @binding = object
17
+ elsif object.is_a?(Hash)
18
+ @binding = Backup::Binder.new(object).get_binding
19
+ else
20
+ @binding = nil
21
+ end
22
+ end
23
+
24
+ ##
25
+ # Renders the provided file (in the context of the binding if any) to the console
26
+ def render(file)
27
+ puts result(file)
28
+ end
29
+
30
+ ##
31
+ # Returns a String object containing the contents of the file (in the context of the binding if any)
32
+ def result(file)
33
+ ERB.new(file_contents(file), nil, '<>').result(binding)
34
+ end
35
+
36
+ private
37
+
38
+ ##
39
+ # Reads and returns the contents of the provided file path,
40
+ # relative from the Backup::TEMPLATE_PATH
41
+ def file_contents(file)
42
+ File.read(File.join(Backup::TEMPLATE_PATH, file))
43
+ end
44
+
45
+ end
46
+ end
@@ -0,0 +1,221 @@
1
+ # encoding: utf-8
2
+
3
+ module Backup
4
+ module Utilities
5
+ class Error < Backup::Error; end
6
+
7
+ UTILITY = {}
8
+ NAMES = %w{
9
+ tar cat split sudo chown hostname
10
+ gzip bzip2
11
+ mongo mongodump mysqldump pg_dump pg_dumpall redis-cli riak-admin
12
+ gpg openssl
13
+ rsync ssh
14
+ sendmail exim
15
+ send_nsca
16
+ }
17
+
18
+ module DSL
19
+ class << self
20
+ ##
21
+ # Allow users to set the path for all utilities in the .configure block.
22
+ #
23
+ # Utility names with dashes ('redis-cli') will be set using method calls
24
+ # with an underscore ('redis_cli').
25
+ NAMES.each do |name|
26
+ define_method name.gsub('-', '_'), lambda {|val|
27
+ path = File.expand_path(val)
28
+ unless File.executable?(path)
29
+ raise Utilities::Error, <<-EOS
30
+ The path given for '#{ name }' was not found or not executable.
31
+ Path was: #{ path }
32
+ EOS
33
+ end
34
+ UTILITY[name] = path
35
+ }
36
+ end
37
+
38
+ ##
39
+ # Allow users to set the +tar+ distribution if needed. (:gnu or :bsd)
40
+ def tar_dist(val)
41
+ Utilities.tar_dist(val)
42
+ end
43
+ end
44
+ end
45
+
46
+ class << self
47
+ ##
48
+ # Configure the path to system utilities used by Backup.
49
+ #
50
+ # Backup will attempt to locate any required system utilities using a
51
+ # +which+ command call. If a utility can not be found, or you need to
52
+ # specify an alternate path for a utility, you may do so in your
53
+ # +config.rb+ file using this method.
54
+ #
55
+ # Backup supports both GNU and BSD utilities.
56
+ # While Backup uses these utilities in a manner compatible with either
57
+ # version, the +tar+ utility requires some special handling with respect
58
+ # to +Archive+s. Backup will attempt to detect if the +tar+ command
59
+ # found (or set here) is GNU or BSD. If for some reason this fails,
60
+ # this may be set using the +tar_dist+ command shown below.
61
+ #
62
+ # Backup::Utilities.configure do
63
+ # # General Utilites
64
+ # tar '/path/to/tar'
65
+ # tar_dist :gnu # or :bsd
66
+ # cat '/path/to/cat'
67
+ # split '/path/to/split'
68
+ # sudo '/path/to/sudo'
69
+ # chown '/path/to/chown'
70
+ # hostname '/path/to/hostname'
71
+ #
72
+ # # Compressors
73
+ # gzip '/path/to/gzip'
74
+ # bzip2 '/path/to/bzip2'
75
+ #
76
+ # # Database Utilities
77
+ # mongo '/path/to/mongo'
78
+ # mongodump '/path/to/mongodump'
79
+ # mysqldump '/path/to/mysqldump'
80
+ # pg_dump '/path/to/pg_dump'
81
+ # pg_dumpall '/path/to/pg_dumpall'
82
+ # redis_cli '/path/to/redis-cli'
83
+ # riak_admin '/path/to/riak-admin'
84
+ #
85
+ # # Encryptors
86
+ # gpg '/path/to/gpg'
87
+ # openssl '/path/to/openssl'
88
+ #
89
+ # # Syncer and Storage
90
+ # rsync '/path/to/rsync'
91
+ # ssh '/path/to/ssh'
92
+ #
93
+ # # Notifiers
94
+ # sendmail '/path/to/sendmail'
95
+ # exim '/path/to/exim'
96
+ # send_nsca '/path/to/send_nsca'
97
+ # end
98
+ #
99
+ # These paths may be set using absolute paths, or relative to the
100
+ # working directory when Backup is run.
101
+ def configure(&block)
102
+ DSL.instance_eval(&block)
103
+ end
104
+
105
+ def tar_dist(val)
106
+ # the acceptance tests need to be able to reset this to nil
107
+ @gnu_tar = val.nil? ? nil : val == :gnu
108
+ end
109
+
110
+ def gnu_tar?
111
+ return @gnu_tar unless @gnu_tar.nil?
112
+ @gnu_tar = !!run("#{ utility(:tar) } --version").match(/GNU/)
113
+ end
114
+
115
+ private
116
+
117
+ ##
118
+ # Returns the full path to the specified utility.
119
+ # Raises an error if utility can not be found in the system's $PATH
120
+ def utility(name)
121
+ name = name.to_s.strip
122
+ raise Error, 'Utility Name Empty' if name.empty?
123
+
124
+ UTILITY[name] ||= %x[which '#{ name }' 2>/dev/null].chomp
125
+ raise Error, <<-EOS if UTILITY[name].empty?
126
+ Could not locate '#{ name }'.
127
+ Make sure the specified utility is installed
128
+ and available in your system's $PATH, or specify it's location
129
+ in your 'config.rb' file using Backup::Utilities.configure
130
+ EOS
131
+
132
+ UTILITY[name].dup
133
+ end
134
+
135
+ ##
136
+ # Returns the name of the command name from the given command line.
137
+ # This is only used to simplify log messages.
138
+ def command_name(command)
139
+ parts = []
140
+ command = command.split(' ')
141
+ command.shift while command[0].to_s.include?('=')
142
+ parts << command.shift.split('/')[-1]
143
+ if parts[0] == 'sudo'
144
+ until command.empty?
145
+ part = command.shift
146
+ if part.include?('/')
147
+ parts << part.split('/')[-1]
148
+ break
149
+ else
150
+ parts << part
151
+ end
152
+ end
153
+ end
154
+ parts.join(' ')
155
+ end
156
+
157
+ ##
158
+ # Runs a system command
159
+ #
160
+ # All messages generated by the command will be logged.
161
+ # Messages on STDERR will be logged as warnings.
162
+ #
163
+ # If the command fails to execute, or returns a non-zero exit status
164
+ # an Error will be raised.
165
+ #
166
+ # Returns STDOUT
167
+ def run(command)
168
+ name = command_name(command)
169
+ Logger.info "Running system utility '#{ name }'..."
170
+
171
+ begin
172
+ out, err = '', ''
173
+ ps = Open4.popen4(command) do |pid, stdin, stdout, stderr|
174
+ stdin.close
175
+ out, err = stdout.read.strip, stderr.read.strip
176
+ end
177
+ rescue Exception => e
178
+ raise Error.wrap(e, "Failed to execute '#{ name }'")
179
+ end
180
+
181
+ if ps.success?
182
+ unless out.empty?
183
+ Logger.info(
184
+ out.lines.map {|line| "#{ name }:STDOUT: #{ line }" }.join
185
+ )
186
+ end
187
+
188
+ unless err.empty?
189
+ Logger.warn(
190
+ err.lines.map {|line| "#{ name }:STDERR: #{ line }" }.join
191
+ )
192
+ end
193
+
194
+ return out
195
+ else
196
+ raise Error, <<-EOS
197
+ '#{ name }' failed with exit status: #{ ps.exitstatus }
198
+ STDOUT Messages: #{ out.empty? ? 'None' : "\n#{ out }" }
199
+ STDERR Messages: #{ err.empty? ? 'None' : "\n#{ err }" }
200
+ EOS
201
+ end
202
+ end
203
+
204
+ def reset!
205
+ UTILITY.clear
206
+ @gnu_tar = nil
207
+ end
208
+ end
209
+
210
+ # Allows these utility methods to be included in other classes,
211
+ # while allowing them to be stubbed in spec_helper for all specs.
212
+ module Helpers
213
+ [:utility, :command_name, :run].each do |name|
214
+ define_method name, lambda {|arg| Utilities.send(name, arg) }
215
+ private name
216
+ end
217
+ private
218
+ def gnu_tar?; Utilities.gnu_tar?; end
219
+ end
220
+ end
221
+ end
@@ -0,0 +1,5 @@
1
+ # encoding: utf-8
2
+
3
+ module Backup
4
+ VERSION = '4.0.1a'
5
+ end
@@ -0,0 +1,28 @@
1
+ ##
2
+ # Archive [Archive]
3
+ #
4
+ # Adding a file or directory (including sub-directories):
5
+ # archive.add "/path/to/a/file.rb"
6
+ # archive.add "/path/to/a/directory/"
7
+ #
8
+ # Excluding a file or directory (including sub-directories):
9
+ # archive.exclude "/path/to/an/excluded_file.rb"
10
+ # archive.exclude "/path/to/an/excluded_directory
11
+ #
12
+ # By default, relative paths will be relative to the directory
13
+ # where `backup perform` is executed, and they will be expanded
14
+ # to the root of the filesystem when added to the archive.
15
+ #
16
+ # If a `root` path is set, relative paths will be relative to the
17
+ # given `root` path and will not be expanded when added to the archive.
18
+ #
19
+ # archive.root '/path/to/archive/root'
20
+ #
21
+ archive :my_archive do |archive|
22
+ # Run the `tar` command using `sudo`
23
+ # archive.use_sudo
24
+ archive.add "/path/to/a/file.rb"
25
+ archive.add "/path/to/a/folder/"
26
+ archive.exclude "/path/to/a/excluded_file.rb"
27
+ archive.exclude "/path/to/a/excluded_folder"
28
+ end
@@ -0,0 +1,4 @@
1
+ ##
2
+ # Bzip2 [Compressor]
3
+ #
4
+ compress_with Bzip2
@@ -0,0 +1,7 @@
1
+ ##
2
+ # Custom [Compressor]
3
+ #
4
+ compress_with Custom do |compressor|
5
+ compressor.command = 'gzip'
6
+ compressor.extension = '.gz'
7
+ end
@@ -0,0 +1,4 @@
1
+ ##
2
+ # Gzip [Compressor]
3
+ #
4
+ compress_with Gzip
@@ -0,0 +1,123 @@
1
+ # encoding: utf-8
2
+
3
+ ##
4
+ # Backup v<%= Backup::VERSION.split('.').first %>.x Configuration
5
+ #
6
+ # Documentation: http://meskyanichi.github.io/backup
7
+ # Issue Tracker: https://github.com/meskyanichi/backup/issues
8
+
9
+ ##
10
+ # Config Options
11
+ #
12
+ # The options here may be overridden on the command line, but the result
13
+ # will depend on the use of --root-path on the command line.
14
+ #
15
+ # If --root-path is used on the command line, then all paths set here
16
+ # will be overridden. If a path (like --tmp-path) is not given along with
17
+ # --root-path, that path will use it's default location _relative to --root-path_.
18
+ #
19
+ # If --root-path is not used on the command line, a path option (like --tmp-path)
20
+ # given on the command line will override the tmp_path set here, but all other
21
+ # paths set here will be used.
22
+ #
23
+ # Note that relative paths given on the command line without --root-path
24
+ # are relative to the current directory. The root_path set here only applies
25
+ # to relative paths set here.
26
+ #
27
+ # ---
28
+ #
29
+ # Sets the root path for all relative paths, including default paths.
30
+ # May be an absolute path, or relative to the current working directory.
31
+ #
32
+ # root_path 'my/root'
33
+ #
34
+ # Sets the path where backups are processed until they're stored.
35
+ # This must have enough free space to hold apx. 2 backups.
36
+ # May be an absolute path, or relative to the current directory or +root_path+.
37
+ #
38
+ # tmp_path 'my/tmp'
39
+ #
40
+ # Sets the path where backup stores persistent information.
41
+ # When Backup's Cycler is used, small YAML files are stored here.
42
+ # May be an absolute path, or relative to the current directory or +root_path+.
43
+ #
44
+ # data_path 'my/data'
45
+
46
+ ##
47
+ # Utilities
48
+ #
49
+ # If you need to use a utility other than the one Backup detects,
50
+ # or a utility can not be found in your $PATH.
51
+ #
52
+ # Utilities.configure do
53
+ # tar '/usr/bin/gnutar'
54
+ # redis_cli '/opt/redis/redis-cli'
55
+ # end
56
+
57
+ ##
58
+ # Logging
59
+ #
60
+ # Logging options may be set on the command line, but certain settings
61
+ # may only be configured here.
62
+ #
63
+ # Logger.configure do
64
+ # console.quiet = true # Same as command line: --quiet
65
+ # logfile.max_bytes = 2_000_000 # Default: 500_000
66
+ # syslog.enabled = true # Same as command line: --syslog
67
+ # syslog.ident = 'my_app_backup' # Default: 'backup'
68
+ # end
69
+ #
70
+ # Command line options will override those set here.
71
+ # For example, the following would override the example settings above
72
+ # to disable syslog and enable console output.
73
+ # backup perform --trigger my_backup --no-syslog --no-quiet
74
+
75
+ ##
76
+ # Component Defaults
77
+ #
78
+ # Set default options to be applied to components in all models.
79
+ # Options set within a model will override those set here.
80
+ #
81
+ # Storage::S3.defaults do |s3|
82
+ # s3.access_key_id = "my_access_key_id"
83
+ # s3.secret_access_key = "my_secret_access_key"
84
+ # end
85
+ #
86
+ # Notifier::Mail.defaults do |mail|
87
+ # mail.from = 'sender@email.com'
88
+ # mail.to = 'receiver@email.com'
89
+ # mail.address = 'smtp.gmail.com'
90
+ # mail.port = 587
91
+ # mail.domain = 'your.host.name'
92
+ # mail.user_name = 'sender@email.com'
93
+ # mail.password = 'my_password'
94
+ # mail.authentication = 'plain'
95
+ # mail.encryption = :starttls
96
+ # end
97
+
98
+ ##
99
+ # Preconfigured Models
100
+ #
101
+ # Create custom models with preconfigured components.
102
+ # Components added within the model definition will
103
+ # +add to+ the preconfigured components.
104
+ #
105
+ # preconfigure 'MyModel' do
106
+ # archive :user_pictures do |archive|
107
+ # archive.add '~/pictures'
108
+ # end
109
+ #
110
+ # notify_by Mail do |mail|
111
+ # mail.to = 'admin@email.com'
112
+ # end
113
+ # end
114
+ #
115
+ # MyModel.new(:john_smith, 'John Smith Backup') do
116
+ # archive :user_music do |archive|
117
+ # archive.add '~/music'
118
+ # end
119
+ #
120
+ # notify_by Mail do |mail|
121
+ # mail.to = 'john.smith@email.com'
122
+ # end
123
+ # end