backup-ssh 4.1.10

Sign up to get free protection for your applications and to get access to all the features.
Files changed (132) hide show
  1. checksums.yaml +7 -0
  2. data/LICENSE.md +24 -0
  3. data/README.md +25 -0
  4. data/bin/backup +5 -0
  5. data/lib/backup.rb +141 -0
  6. data/lib/backup/archive.rb +170 -0
  7. data/lib/backup/binder.rb +22 -0
  8. data/lib/backup/cleaner.rb +116 -0
  9. data/lib/backup/cli.rb +374 -0
  10. data/lib/backup/cloud_io/base.rb +41 -0
  11. data/lib/backup/cloud_io/cloud_files.rb +298 -0
  12. data/lib/backup/cloud_io/s3.rb +260 -0
  13. data/lib/backup/compressor/base.rb +35 -0
  14. data/lib/backup/compressor/bzip2.rb +39 -0
  15. data/lib/backup/compressor/custom.rb +53 -0
  16. data/lib/backup/compressor/gzip.rb +74 -0
  17. data/lib/backup/config.rb +119 -0
  18. data/lib/backup/config/dsl.rb +103 -0
  19. data/lib/backup/config/helpers.rb +143 -0
  20. data/lib/backup/database/base.rb +85 -0
  21. data/lib/backup/database/mongodb.rb +186 -0
  22. data/lib/backup/database/mysql.rb +200 -0
  23. data/lib/backup/database/openldap.rb +95 -0
  24. data/lib/backup/database/postgresql.rb +133 -0
  25. data/lib/backup/database/redis.rb +179 -0
  26. data/lib/backup/database/riak.rb +82 -0
  27. data/lib/backup/database/sqlite.rb +57 -0
  28. data/lib/backup/encryptor/base.rb +29 -0
  29. data/lib/backup/encryptor/gpg.rb +747 -0
  30. data/lib/backup/encryptor/open_ssl.rb +77 -0
  31. data/lib/backup/errors.rb +58 -0
  32. data/lib/backup/logger.rb +199 -0
  33. data/lib/backup/logger/console.rb +51 -0
  34. data/lib/backup/logger/fog_adapter.rb +29 -0
  35. data/lib/backup/logger/logfile.rb +133 -0
  36. data/lib/backup/logger/syslog.rb +116 -0
  37. data/lib/backup/model.rb +454 -0
  38. data/lib/backup/notifier/base.rb +98 -0
  39. data/lib/backup/notifier/campfire.rb +69 -0
  40. data/lib/backup/notifier/datadog.rb +116 -0
  41. data/lib/backup/notifier/flowdock.rb +102 -0
  42. data/lib/backup/notifier/hipchat.rb +93 -0
  43. data/lib/backup/notifier/http_post.rb +122 -0
  44. data/lib/backup/notifier/mail.rb +238 -0
  45. data/lib/backup/notifier/nagios.rb +74 -0
  46. data/lib/backup/notifier/pagerduty.rb +81 -0
  47. data/lib/backup/notifier/prowl.rb +69 -0
  48. data/lib/backup/notifier/pushover.rb +80 -0
  49. data/lib/backup/notifier/ses.rb +94 -0
  50. data/lib/backup/notifier/slack.rb +154 -0
  51. data/lib/backup/notifier/twitter.rb +64 -0
  52. data/lib/backup/notifier/zabbix.rb +68 -0
  53. data/lib/backup/package.rb +51 -0
  54. data/lib/backup/packager.rb +101 -0
  55. data/lib/backup/pipeline.rb +124 -0
  56. data/lib/backup/splitter.rb +76 -0
  57. data/lib/backup/storage/base.rb +57 -0
  58. data/lib/backup/storage/cloud_files.rb +158 -0
  59. data/lib/backup/storage/cycler.rb +65 -0
  60. data/lib/backup/storage/dropbox.rb +236 -0
  61. data/lib/backup/storage/ftp.rb +98 -0
  62. data/lib/backup/storage/local.rb +64 -0
  63. data/lib/backup/storage/ninefold.rb +74 -0
  64. data/lib/backup/storage/rsync.rb +248 -0
  65. data/lib/backup/storage/s3.rb +155 -0
  66. data/lib/backup/storage/scp.rb +67 -0
  67. data/lib/backup/storage/sftp.rb +82 -0
  68. data/lib/backup/syncer/base.rb +70 -0
  69. data/lib/backup/syncer/cloud/base.rb +179 -0
  70. data/lib/backup/syncer/cloud/cloud_files.rb +83 -0
  71. data/lib/backup/syncer/cloud/local_file.rb +100 -0
  72. data/lib/backup/syncer/cloud/s3.rb +110 -0
  73. data/lib/backup/syncer/rsync/base.rb +48 -0
  74. data/lib/backup/syncer/rsync/local.rb +31 -0
  75. data/lib/backup/syncer/rsync/pull.rb +51 -0
  76. data/lib/backup/syncer/rsync/push.rb +205 -0
  77. data/lib/backup/template.rb +46 -0
  78. data/lib/backup/utilities.rb +224 -0
  79. data/lib/backup/version.rb +5 -0
  80. data/templates/cli/archive +28 -0
  81. data/templates/cli/compressor/bzip2 +4 -0
  82. data/templates/cli/compressor/custom +7 -0
  83. data/templates/cli/compressor/gzip +4 -0
  84. data/templates/cli/config +123 -0
  85. data/templates/cli/databases/mongodb +15 -0
  86. data/templates/cli/databases/mysql +18 -0
  87. data/templates/cli/databases/openldap +24 -0
  88. data/templates/cli/databases/postgresql +16 -0
  89. data/templates/cli/databases/redis +16 -0
  90. data/templates/cli/databases/riak +17 -0
  91. data/templates/cli/databases/sqlite +11 -0
  92. data/templates/cli/encryptor/gpg +27 -0
  93. data/templates/cli/encryptor/openssl +9 -0
  94. data/templates/cli/model +26 -0
  95. data/templates/cli/notifier/zabbix +15 -0
  96. data/templates/cli/notifiers/campfire +12 -0
  97. data/templates/cli/notifiers/datadog +57 -0
  98. data/templates/cli/notifiers/flowdock +16 -0
  99. data/templates/cli/notifiers/hipchat +15 -0
  100. data/templates/cli/notifiers/http_post +32 -0
  101. data/templates/cli/notifiers/mail +21 -0
  102. data/templates/cli/notifiers/nagios +13 -0
  103. data/templates/cli/notifiers/pagerduty +12 -0
  104. data/templates/cli/notifiers/prowl +11 -0
  105. data/templates/cli/notifiers/pushover +11 -0
  106. data/templates/cli/notifiers/ses +15 -0
  107. data/templates/cli/notifiers/slack +22 -0
  108. data/templates/cli/notifiers/twitter +13 -0
  109. data/templates/cli/splitter +7 -0
  110. data/templates/cli/storages/cloud_files +11 -0
  111. data/templates/cli/storages/dropbox +19 -0
  112. data/templates/cli/storages/ftp +12 -0
  113. data/templates/cli/storages/local +7 -0
  114. data/templates/cli/storages/ninefold +9 -0
  115. data/templates/cli/storages/rsync +17 -0
  116. data/templates/cli/storages/s3 +14 -0
  117. data/templates/cli/storages/scp +14 -0
  118. data/templates/cli/storages/sftp +14 -0
  119. data/templates/cli/syncers/cloud_files +22 -0
  120. data/templates/cli/syncers/rsync_local +20 -0
  121. data/templates/cli/syncers/rsync_pull +28 -0
  122. data/templates/cli/syncers/rsync_push +28 -0
  123. data/templates/cli/syncers/s3 +27 -0
  124. data/templates/general/links +3 -0
  125. data/templates/general/version.erb +2 -0
  126. data/templates/notifier/mail/failure.erb +16 -0
  127. data/templates/notifier/mail/success.erb +16 -0
  128. data/templates/notifier/mail/warning.erb +16 -0
  129. data/templates/storage/dropbox/authorization_url.erb +6 -0
  130. data/templates/storage/dropbox/authorized.erb +4 -0
  131. data/templates/storage/dropbox/cache_file_written.erb +10 -0
  132. metadata +1057 -0
@@ -0,0 +1,98 @@
1
+ # encoding: utf-8
2
+ require 'net/ftp'
3
+
4
+ module Backup
5
+ module Storage
6
+ class FTP < Base
7
+ include Storage::Cycler
8
+
9
+ ##
10
+ # Server credentials
11
+ attr_accessor :username, :password
12
+
13
+ ##
14
+ # Server IP Address and FTP port
15
+ attr_accessor :ip, :port
16
+
17
+ ##
18
+ # use passive mode?
19
+ attr_accessor :passive_mode
20
+
21
+ def initialize(model, storage_id = nil)
22
+ super
23
+
24
+ @port ||= 21
25
+ @path ||= 'backups'
26
+ @passive_mode ||= false
27
+ path.sub!(/^~\//, '')
28
+ end
29
+
30
+ private
31
+
32
+ ##
33
+ # Establishes a connection to the remote server
34
+ #
35
+ # Note:
36
+ # Since the FTP port is defined as a constant in the Net::FTP class, and
37
+ # might be required to change by the user, we dynamically remove and
38
+ # re-add the constant with the provided port value
39
+ def connection
40
+ if Net::FTP.const_defined?(:FTP_PORT)
41
+ Net::FTP.send(:remove_const, :FTP_PORT)
42
+ end; Net::FTP.send(:const_set, :FTP_PORT, port)
43
+
44
+ Net::FTP.open(ip, username, password) do |ftp|
45
+ ftp.passive = true if passive_mode
46
+ yield ftp
47
+ end
48
+ end
49
+
50
+ def transfer!
51
+ connection do |ftp|
52
+ create_remote_path(ftp)
53
+
54
+ package.filenames.each do |filename|
55
+ src = File.join(Config.tmp_path, filename)
56
+ dest = File.join(remote_path, filename)
57
+ Logger.info "Storing '#{ ip }:#{ dest }'..."
58
+ ftp.put(src, dest)
59
+ end
60
+ end
61
+ end
62
+
63
+ # Called by the Cycler.
64
+ # Any error raised will be logged as a warning.
65
+ def remove!(package)
66
+ Logger.info "Removing backup package dated #{ package.time }..."
67
+
68
+ remote_path = remote_path_for(package)
69
+ connection do |ftp|
70
+ package.filenames.each do |filename|
71
+ ftp.delete(File.join(remote_path, filename))
72
+ end
73
+
74
+ ftp.rmdir(remote_path)
75
+ end
76
+ end
77
+
78
+ ##
79
+ # Creates (if they don't exist yet) all the directories on the remote
80
+ # server in order to upload the backup file. Net::FTP does not support
81
+ # paths to directories that don't yet exist when creating new
82
+ # directories. Instead, we split the parts up in to an array (for each
83
+ # '/') and loop through that to create the directories one by one.
84
+ # Net::FTP raises an exception when the directory it's trying to create
85
+ # already exists, so we have rescue it
86
+ def create_remote_path(ftp)
87
+ path_parts = Array.new
88
+ remote_path.split('/').each do |path_part|
89
+ path_parts << path_part
90
+ begin
91
+ ftp.mkdir(path_parts.join('/'))
92
+ rescue Net::FTPPermError; end
93
+ end
94
+ end
95
+
96
+ end
97
+ end
98
+ end
@@ -0,0 +1,64 @@
1
+ # encoding: utf-8
2
+
3
+ module Backup
4
+ module Storage
5
+ class Local < Base
6
+ include Storage::Cycler
7
+ class Error < Backup::Error; end
8
+
9
+ def initialize(model, storage_id = nil)
10
+ super
11
+
12
+ @path ||= '~/backups'
13
+ end
14
+
15
+ private
16
+
17
+ def transfer!
18
+ FileUtils.mkdir_p(remote_path)
19
+
20
+ transfer_method = package_movable? ? :mv : :cp
21
+ package.filenames.each do |filename|
22
+ src = File.join(Config.tmp_path, filename)
23
+ dest = File.join(remote_path, filename)
24
+ Logger.info "Storing '#{ dest }'..."
25
+
26
+ FileUtils.send(transfer_method, src, dest)
27
+ end
28
+ end
29
+
30
+ # Called by the Cycler.
31
+ # Any error raised will be logged as a warning.
32
+ def remove!(package)
33
+ Logger.info "Removing backup package dated #{ package.time }..."
34
+
35
+ FileUtils.rm_r(remote_path_for(package))
36
+ end
37
+
38
+ # expanded since this is a local path
39
+ def remote_path(pkg = package)
40
+ File.expand_path(super)
41
+ end
42
+ alias :remote_path_for :remote_path
43
+
44
+ ##
45
+ # If this Local Storage is not the last Storage for the Model,
46
+ # force the transfer to use a *copy* operation and issue a warning.
47
+ def package_movable?
48
+ if self == model.storages.last
49
+ true
50
+ else
51
+ Logger.warn Error.new(<<-EOS)
52
+ Local File Copy Warning!
53
+ The final backup file(s) for '#{ model.label }' (#{ model.trigger })
54
+ will be *copied* to '#{ remote_path }'
55
+ To avoid this, when using more than one Storage, the 'Local' Storage
56
+ should be added *last* so the files may be *moved* to their destination.
57
+ EOS
58
+ false
59
+ end
60
+ end
61
+
62
+ end
63
+ end
64
+ end
@@ -0,0 +1,74 @@
1
+ # encoding: utf-8
2
+ require 'fog'
3
+
4
+ module Backup
5
+ module Storage
6
+ class Ninefold < Base
7
+ include Storage::Cycler
8
+ class Error < Backup::Error; end
9
+
10
+ ##
11
+ # Ninefold Credentials
12
+ attr_accessor :storage_token, :storage_secret
13
+
14
+ def initialize(model, storage_id = nil)
15
+ super
16
+
17
+ @path ||= 'backups'
18
+ path.sub!(/^\//, '')
19
+ end
20
+
21
+ private
22
+
23
+ def connection
24
+ @connection ||= Fog::Storage.new(
25
+ :provider => 'Ninefold',
26
+ :ninefold_storage_token => storage_token,
27
+ :ninefold_storage_secret => storage_secret
28
+ )
29
+ end
30
+
31
+ def transfer!
32
+ directory = directory_for(remote_path, true)
33
+ package.filenames.each do |filename|
34
+ src = File.join(Config.tmp_path, filename)
35
+ dest = File.join(remote_path, filename)
36
+ Logger.info "Storing '#{ dest }'..."
37
+ File.open(src, 'r') do |file|
38
+ directory.files.create(:key => filename, :body => file)
39
+ end
40
+ end
41
+ end
42
+
43
+ ##
44
+ # Queries the connection for the directory for the given +remote_path+
45
+ # Returns nil if not found, or creates the directory if +create+ is true.
46
+ def directory_for(remote_path, create = false)
47
+ directory = connection.directories.get(remote_path)
48
+ if directory.nil? && create
49
+ directory = connection.directories.create(:key => remote_path)
50
+ end
51
+ directory
52
+ end
53
+
54
+ # Called by the Cycler.
55
+ # Any error raised will be logged as a warning.
56
+ def remove!(package)
57
+ Logger.info "Removing backup package dated #{ package.time }..."
58
+
59
+ remote_path = remote_path_for(package)
60
+ directory = directory_for(remote_path)
61
+
62
+ raise Error, "Directory at '#{ remote_path }' not found" unless directory
63
+
64
+ package.filenames.each do |filename|
65
+ file = directory.files.get(filename)
66
+ file.destroy if file
67
+ end
68
+
69
+ directory.destroy
70
+ end
71
+
72
+ end
73
+ end
74
+ end
@@ -0,0 +1,248 @@
1
+ # encoding: utf-8
2
+
3
+ module Backup
4
+ module Storage
5
+ class RSync < Base
6
+ include Utilities::Helpers
7
+
8
+ ##
9
+ # Mode of operation
10
+ #
11
+ # [:ssh (default)]
12
+ # Connects to the remote via SSH.
13
+ # Does not use an rsync daemon on the remote.
14
+ #
15
+ # [:ssh_daemon]
16
+ # Connects to the remote via SSH.
17
+ # Spawns a single-use daemon on the remote, which allows certain
18
+ # daemon features (like modules) to be used.
19
+ #
20
+ # [:rsync_daemon]
21
+ # Connects directly to an rsync daemon via TCP.
22
+ # Data transferred is not encrypted.
23
+ #
24
+ attr_accessor :mode
25
+
26
+ ##
27
+ # Server Address
28
+ #
29
+ # If not specified, the storage operation will be local.
30
+ attr_accessor :host
31
+
32
+ ##
33
+ # SSH or RSync port
34
+ #
35
+ # For `:ssh` or `:ssh_daemon` mode, this specifies the SSH port to use
36
+ # and defaults to 22.
37
+ #
38
+ # For `:rsync_daemon` mode, this specifies the TCP port to use
39
+ # and defaults to 873.
40
+ attr_accessor :port
41
+
42
+ ##
43
+ # SSH User
44
+ #
45
+ # If the user running the backup is not the same user that needs to
46
+ # authenticate with the remote server, specify the user here.
47
+ #
48
+ # The user must have SSH keys setup for passphrase-less access to the
49
+ # remote. If the SSH User does not have passphrase-less keys, or no
50
+ # default keys in their `~/.ssh` directory, you will need to use the
51
+ # `-i` option in `:additional_ssh_options` to specify the
52
+ # passphrase-less key to use.
53
+ #
54
+ # Used only for `:ssh` and `:ssh_daemon` modes.
55
+ attr_accessor :ssh_user
56
+
57
+ ##
58
+ # Additional SSH Options
59
+ #
60
+ # Used to supply a String or Array of options to be passed to the SSH
61
+ # command in `:ssh` and `:ssh_daemon` modes.
62
+ #
63
+ # For example, if you need to supply a specific SSH key for the `ssh_user`,
64
+ # you would set this to: "-i '/path/to/id_rsa'". Which would produce:
65
+ #
66
+ # rsync -e "ssh -p 22 -i '/path/to/id_rsa'"
67
+ #
68
+ # Arguments may be single-quoted, but should not contain any double-quotes.
69
+ #
70
+ # Used only for `:ssh` and `:ssh_daemon` modes.
71
+ attr_accessor :additional_ssh_options
72
+
73
+ ##
74
+ # RSync User
75
+ #
76
+ # If the user running the backup is not the same user that needs to
77
+ # authenticate with the rsync daemon, specify the user here.
78
+ #
79
+ # Used only for `:ssh_daemon` and `:rsync_daemon` modes.
80
+ attr_accessor :rsync_user
81
+
82
+ ##
83
+ # RSync Password
84
+ #
85
+ # If specified, Backup will write the password to a temporary file and
86
+ # use it with rsync's `--password-file` option for daemon authentication.
87
+ #
88
+ # Note that setting this will override `rsync_password_file`.
89
+ #
90
+ # Used only for `:ssh_daemon` and `:rsync_daemon` modes.
91
+ attr_accessor :rsync_password
92
+
93
+ ##
94
+ # RSync Password File
95
+ #
96
+ # If specified, this path will be passed to rsync's `--password-file`
97
+ # option for daemon authentication.
98
+ #
99
+ # Used only for `:ssh_daemon` and `:rsync_daemon` modes.
100
+ attr_accessor :rsync_password_file
101
+
102
+ ##
103
+ # Additional String or Array of options for the rsync cli
104
+ attr_accessor :additional_rsync_options
105
+
106
+ ##
107
+ # Flag for compressing (only compresses for the transfer)
108
+ attr_accessor :compress
109
+
110
+ ##
111
+ # Path to store the synced backup package file(s) to.
112
+ #
113
+ # If no +host+ is specified, then +path+ will be local, and the only
114
+ # other used option would be +additional_rsync_options+.
115
+ # +path+ will be expanded, so '~/my_path' will expand to '$HOME/my_path'.
116
+ #
117
+ # If a +host+ is specified, this will be a path on the host.
118
+ # If +mode+ is `:ssh` (default), then any relative path, or path starting
119
+ # with '~/' will be relative to the directory the ssh_user is logged
120
+ # into. For `:ssh_daemon` or `:rsync_daemon` modes, this would reference
121
+ # an rsync module/path.
122
+ #
123
+ # In :ssh_daemon and :rsync_daemon modes, +path+ (or path defined by
124
+ # your rsync module) must already exist.
125
+ #
126
+ # In :ssh mode or local operation (no +host+ specified), +path+ will
127
+ # be created if needed - either locally, or on the remote for :ssh mode.
128
+ attr_accessor :path
129
+
130
+ def initialize(model, storage_id = nil)
131
+ super
132
+
133
+ @mode ||= :ssh
134
+ @port ||= mode == :rsync_daemon ? 873 : 22
135
+ @compress ||= false
136
+ @path ||= '~/backups'
137
+ end
138
+
139
+ private
140
+
141
+ def transfer!
142
+ write_password_file
143
+ create_remote_path
144
+
145
+ package.filenames.each do |filename|
146
+ src = "'#{ File.join(Config.tmp_path, filename) }'"
147
+ dest = "#{ host_options }'#{ File.join(remote_path, filename) }'"
148
+ Logger.info "Syncing to #{ dest }..."
149
+ run("#{ rsync_command } #{ src } #{ dest }")
150
+ end
151
+ ensure
152
+ remove_password_file
153
+ end
154
+
155
+ ##
156
+ # Other storages add an additional timestamp directory to this path.
157
+ # This is not desired here, since we need to transfer the package files
158
+ # to the same location each time.
159
+ def remote_path
160
+ @remote_path ||= begin
161
+ if host
162
+ path.sub(/^~\//, '').sub(/\/$/, '')
163
+ else
164
+ File.expand_path(path)
165
+ end
166
+ end
167
+ end
168
+
169
+ ##
170
+ # Runs a 'mkdir -p' command on the host (or locally) to ensure the
171
+ # dest_path exists. This is used because we're transferring a single
172
+ # file, and rsync won't attempt to create the intermediate directories.
173
+ #
174
+ # This is only applicable locally and in :ssh mode.
175
+ # In :ssh_daemon and :rsync_daemon modes the `path` would include a
176
+ # module name that must define a path on the remote that already exists.
177
+ def create_remote_path
178
+ if host
179
+ run("#{ utility(:ssh) } #{ ssh_transport_args } #{ host } " +
180
+ %Q["mkdir -p '#{ remote_path }'"]) if mode == :ssh
181
+ else
182
+ FileUtils.mkdir_p(remote_path)
183
+ end
184
+ end
185
+
186
+ def host_options
187
+ @host_options ||= begin
188
+ if !host
189
+ ''
190
+ elsif mode == :ssh
191
+ "#{ host }:"
192
+ else
193
+ user = "#{ rsync_user }@" if rsync_user
194
+ "#{ user }#{ host }::"
195
+ end
196
+ end
197
+ end
198
+
199
+ def rsync_command
200
+ @rsync_command ||= begin
201
+ cmd = utility(:rsync) << ' --archive' <<
202
+ " #{ Array(additional_rsync_options).join(' ') }".rstrip
203
+ cmd << compress_option << password_option << transport_options if host
204
+ cmd
205
+ end
206
+ end
207
+
208
+ def compress_option
209
+ compress ? ' --compress' : ''
210
+ end
211
+
212
+ def password_option
213
+ return '' if mode == :ssh
214
+
215
+ path = @password_file ? @password_file.path : rsync_password_file
216
+ path ? " --password-file='#{ File.expand_path(path) }'" : ''
217
+ end
218
+
219
+ def transport_options
220
+ if mode == :rsync_daemon
221
+ " --port #{ port }"
222
+ else
223
+ %Q[ -e "#{ utility(:ssh) } #{ ssh_transport_args }"]
224
+ end
225
+ end
226
+
227
+ def ssh_transport_args
228
+ args = "-p #{ port } "
229
+ args << "-l #{ ssh_user } " if ssh_user
230
+ args << Array(additional_ssh_options).join(' ')
231
+ args.rstrip
232
+ end
233
+
234
+ def write_password_file
235
+ return unless host && rsync_password && mode != :ssh
236
+
237
+ @password_file = Tempfile.new('backup-rsync-password')
238
+ @password_file.write(rsync_password)
239
+ @password_file.close
240
+ end
241
+
242
+ def remove_password_file
243
+ @password_file.delete if @password_file
244
+ end
245
+
246
+ end
247
+ end
248
+ end