backedup 5.0.0.beta.3

Sign up to get free protection for your applications and to get access to all the features.
Files changed (144) hide show
  1. checksums.yaml +7 -0
  2. data/LICENSE +19 -0
  3. data/README.md +33 -0
  4. data/bin/backedup +5 -0
  5. data/bin/docker_test +24 -0
  6. data/lib/backup/archive.rb +169 -0
  7. data/lib/backup/binder.rb +18 -0
  8. data/lib/backup/cleaner.rb +112 -0
  9. data/lib/backup/cli.rb +370 -0
  10. data/lib/backup/cloud_io/base.rb +38 -0
  11. data/lib/backup/cloud_io/cloud_files.rb +296 -0
  12. data/lib/backup/cloud_io/gcs.rb +121 -0
  13. data/lib/backup/cloud_io/s3.rb +253 -0
  14. data/lib/backup/cloud_io/swift.rb +96 -0
  15. data/lib/backup/compressor/base.rb +32 -0
  16. data/lib/backup/compressor/bzip2.rb +35 -0
  17. data/lib/backup/compressor/custom.rb +49 -0
  18. data/lib/backup/compressor/gzip.rb +73 -0
  19. data/lib/backup/compressor/pbzip2.rb +45 -0
  20. data/lib/backup/config/dsl.rb +102 -0
  21. data/lib/backup/config/helpers.rb +137 -0
  22. data/lib/backup/config.rb +118 -0
  23. data/lib/backup/database/base.rb +86 -0
  24. data/lib/backup/database/mongodb.rb +186 -0
  25. data/lib/backup/database/mysql.rb +191 -0
  26. data/lib/backup/database/openldap.rb +93 -0
  27. data/lib/backup/database/postgresql.rb +164 -0
  28. data/lib/backup/database/redis.rb +176 -0
  29. data/lib/backup/database/riak.rb +79 -0
  30. data/lib/backup/database/sqlite.rb +55 -0
  31. data/lib/backup/encryptor/base.rb +27 -0
  32. data/lib/backup/encryptor/gpg.rb +737 -0
  33. data/lib/backup/encryptor/open_ssl.rb +74 -0
  34. data/lib/backup/errors.rb +53 -0
  35. data/lib/backup/logger/console.rb +48 -0
  36. data/lib/backup/logger/fog_adapter.rb +25 -0
  37. data/lib/backup/logger/logfile.rb +131 -0
  38. data/lib/backup/logger/syslog.rb +114 -0
  39. data/lib/backup/logger.rb +197 -0
  40. data/lib/backup/model.rb +472 -0
  41. data/lib/backup/notifier/base.rb +126 -0
  42. data/lib/backup/notifier/campfire.rb +61 -0
  43. data/lib/backup/notifier/command.rb +99 -0
  44. data/lib/backup/notifier/datadog.rb +104 -0
  45. data/lib/backup/notifier/flowdock.rb +99 -0
  46. data/lib/backup/notifier/hipchat.rb +116 -0
  47. data/lib/backup/notifier/http_post.rb +114 -0
  48. data/lib/backup/notifier/mail.rb +232 -0
  49. data/lib/backup/notifier/nagios.rb +65 -0
  50. data/lib/backup/notifier/pagerduty.rb +79 -0
  51. data/lib/backup/notifier/prowl.rb +68 -0
  52. data/lib/backup/notifier/pushover.rb +71 -0
  53. data/lib/backup/notifier/ses.rb +123 -0
  54. data/lib/backup/notifier/slack.rb +147 -0
  55. data/lib/backup/notifier/twitter.rb +55 -0
  56. data/lib/backup/notifier/zabbix.rb +60 -0
  57. data/lib/backup/package.rb +51 -0
  58. data/lib/backup/packager.rb +106 -0
  59. data/lib/backup/pipeline.rb +120 -0
  60. data/lib/backup/splitter.rb +73 -0
  61. data/lib/backup/storage/base.rb +66 -0
  62. data/lib/backup/storage/cloud_files.rb +156 -0
  63. data/lib/backup/storage/cycler.rb +70 -0
  64. data/lib/backup/storage/dropbox.rb +206 -0
  65. data/lib/backup/storage/ftp.rb +116 -0
  66. data/lib/backup/storage/gcs.rb +93 -0
  67. data/lib/backup/storage/local.rb +61 -0
  68. data/lib/backup/storage/qiniu.rb +65 -0
  69. data/lib/backup/storage/rsync.rb +246 -0
  70. data/lib/backup/storage/s3.rb +155 -0
  71. data/lib/backup/storage/scp.rb +65 -0
  72. data/lib/backup/storage/sftp.rb +80 -0
  73. data/lib/backup/storage/swift.rb +124 -0
  74. data/lib/backup/storage/webdav.rb +102 -0
  75. data/lib/backup/syncer/base.rb +67 -0
  76. data/lib/backup/syncer/cloud/base.rb +176 -0
  77. data/lib/backup/syncer/cloud/cloud_files.rb +81 -0
  78. data/lib/backup/syncer/cloud/local_file.rb +97 -0
  79. data/lib/backup/syncer/cloud/s3.rb +109 -0
  80. data/lib/backup/syncer/rsync/base.rb +50 -0
  81. data/lib/backup/syncer/rsync/local.rb +27 -0
  82. data/lib/backup/syncer/rsync/pull.rb +47 -0
  83. data/lib/backup/syncer/rsync/push.rb +201 -0
  84. data/lib/backup/template.rb +41 -0
  85. data/lib/backup/utilities.rb +234 -0
  86. data/lib/backup/version.rb +3 -0
  87. data/lib/backup.rb +145 -0
  88. data/templates/cli/archive +28 -0
  89. data/templates/cli/compressor/bzip2 +4 -0
  90. data/templates/cli/compressor/custom +7 -0
  91. data/templates/cli/compressor/gzip +4 -0
  92. data/templates/cli/config +123 -0
  93. data/templates/cli/databases/mongodb +15 -0
  94. data/templates/cli/databases/mysql +18 -0
  95. data/templates/cli/databases/openldap +24 -0
  96. data/templates/cli/databases/postgresql +16 -0
  97. data/templates/cli/databases/redis +16 -0
  98. data/templates/cli/databases/riak +17 -0
  99. data/templates/cli/databases/sqlite +11 -0
  100. data/templates/cli/encryptor/gpg +27 -0
  101. data/templates/cli/encryptor/openssl +9 -0
  102. data/templates/cli/model +26 -0
  103. data/templates/cli/notifier/zabbix +15 -0
  104. data/templates/cli/notifiers/campfire +12 -0
  105. data/templates/cli/notifiers/command +32 -0
  106. data/templates/cli/notifiers/datadog +57 -0
  107. data/templates/cli/notifiers/flowdock +16 -0
  108. data/templates/cli/notifiers/hipchat +16 -0
  109. data/templates/cli/notifiers/http_post +32 -0
  110. data/templates/cli/notifiers/mail +24 -0
  111. data/templates/cli/notifiers/nagios +13 -0
  112. data/templates/cli/notifiers/pagerduty +12 -0
  113. data/templates/cli/notifiers/prowl +11 -0
  114. data/templates/cli/notifiers/pushover +11 -0
  115. data/templates/cli/notifiers/ses +15 -0
  116. data/templates/cli/notifiers/slack +22 -0
  117. data/templates/cli/notifiers/twitter +13 -0
  118. data/templates/cli/splitter +7 -0
  119. data/templates/cli/storages/cloud_files +11 -0
  120. data/templates/cli/storages/dropbox +20 -0
  121. data/templates/cli/storages/ftp +13 -0
  122. data/templates/cli/storages/gcs +8 -0
  123. data/templates/cli/storages/local +8 -0
  124. data/templates/cli/storages/qiniu +12 -0
  125. data/templates/cli/storages/rsync +17 -0
  126. data/templates/cli/storages/s3 +16 -0
  127. data/templates/cli/storages/scp +15 -0
  128. data/templates/cli/storages/sftp +15 -0
  129. data/templates/cli/storages/swift +19 -0
  130. data/templates/cli/storages/webdav +13 -0
  131. data/templates/cli/syncers/cloud_files +22 -0
  132. data/templates/cli/syncers/rsync_local +20 -0
  133. data/templates/cli/syncers/rsync_pull +28 -0
  134. data/templates/cli/syncers/rsync_push +28 -0
  135. data/templates/cli/syncers/s3 +27 -0
  136. data/templates/general/links +3 -0
  137. data/templates/general/version.erb +2 -0
  138. data/templates/notifier/mail/failure.erb +16 -0
  139. data/templates/notifier/mail/success.erb +16 -0
  140. data/templates/notifier/mail/warning.erb +16 -0
  141. data/templates/storage/dropbox/authorization_url.erb +6 -0
  142. data/templates/storage/dropbox/authorized.erb +4 -0
  143. data/templates/storage/dropbox/cache_file_written.erb +10 -0
  144. metadata +1255 -0
checksums.yaml ADDED
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA256:
3
+ metadata.gz: 95f5e7b8987faf43d4943d4a3103388a0e6844558d6df3af64f6d5d10069cb1b
4
+ data.tar.gz: 78fc4933f513616125f8e9fef886e8c9702193413603289c73a63b3255b4edd4
5
+ SHA512:
6
+ metadata.gz: 67029dfa0115b98ff740eb401b6d9cec77ac1192585432f2b140779251a62a83fbae782e233c272e67d58083138a8f269a6bd455bb3fe59538f39fc2efabd60d
7
+ data.tar.gz: 407a8eed541ca05fa3aeb2b271b25c11ddedf0b601dc36f15bba0bd50ff2b5cfef1582356fa9e2e41c0eee7642650f6381c30c6ba03afc0927c78a726c80c129
data/LICENSE ADDED
@@ -0,0 +1,19 @@
1
+ Copyright (c) 2009-2017 Michael van Rooijen
2
+
3
+ Permission is hereby granted, free of charge, to any person obtaining a copy
4
+ of this software and associated documentation files (the "Software"), to deal
5
+ in the Software without restriction, including without limitation the rights
6
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
7
+ copies of the Software, and to permit persons to whom the Software is
8
+ furnished to do so, subject to the following conditions:
9
+
10
+ The above copyright notice and this permission notice shall be included in
11
+ all copies or substantial portions of the Software.
12
+
13
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
16
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
17
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
18
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
19
+ THE SOFTWARE.
data/README.md ADDED
@@ -0,0 +1,33 @@
1
+ Backup
2
+ ======
3
+
4
+ [![Code Climate](https://codeclimate.com/github/backup/backup.svg)](https://codeclimate.com/github/backup/backup)
5
+ [![Build Status](https://travis-ci.org/backup/backup.svg?branch=master)](https://travis-ci.org/backup/backup)
6
+ [![Join the chat at https://gitter.im/backup/backup](https://badges.gitter.im/Join%20Chat.svg)][Gitter]
7
+
8
+ > This project is now in maintenance. No new features are planned.
9
+
10
+ Backup is a system utility for Linux and Mac OS X, distributed as a RubyGem, that allows you to easily perform backup
11
+ operations. It provides an elegant DSL in Ruby for _modeling_ your backups. Backup has built-in support for various
12
+ databases, storage protocols/services, syncers, compressors, encryptors and notifiers which you can mix and match. It
13
+ was built with modularity, extensibility and simplicity in mind.
14
+
15
+ [Installation][] · [Release Notes][] · [Documentation][] · [Issues][] · [Chat][Gitter]
16
+
17
+ ## Project Status: Maintenance-Only ##
18
+
19
+ This project is not under active development, although we will continue to provide support for current users, and at least one more maintenance release: version 5.0. The version 5.0 release will include support for Ruby 2.4, Ruby 2.5, and various other fixes. Future releases of Backup will only include bug fixes.
20
+
21
+ If you use this project and would like to develop it further, please introduce yourself on the [maintainers wanted][Maintainers wanted] ticket.
22
+
23
+ **Copyright (c) 2009-2018 [Michael van Rooijen][] ( [@mrrooijen] )**
24
+ Released under the **MIT** [LICENSE](LICENSE).
25
+
26
+ [Installation]: http://backup.github.io/backup/v4/installation
27
+ [Release Notes]: http://backup.github.io/backup/v4/release-notes
28
+ [Documentation]: http://backup.github.io/backup/v4
29
+ [Issues]: https://github.com/backup/backup/issues
30
+ [Gitter]: https://gitter.im/backup/backup?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge
31
+ [Maintainers wanted]: https://github.com/backup/backup/issues/803
32
+ [Michael van Rooijen]: http://github.com/mrrooijen
33
+ [@mrrooijen]: http://twitter.com/mrrooijen
data/bin/backedup ADDED
@@ -0,0 +1,5 @@
1
+ #!/usr/bin/env ruby
2
+ # encoding: utf-8
3
+
4
+ require File.expand_path("../../lib/backup", __FILE__)
5
+ Backup::CLI.start
data/bin/docker_test ADDED
@@ -0,0 +1,24 @@
1
+ #!/bin/bash
2
+ set -eu
3
+
4
+ export BUNDLE_PATH=/tmp/docker/bundle
5
+ export BUNDLE_JOBS=4
6
+
7
+ case "${1-}" in
8
+ prepare)
9
+ bundle check || bundle install
10
+ ;;
11
+ rspec)
12
+ bundle exec rspec spec
13
+ ;;
14
+ integration)
15
+ bundle exec rspec integration/acceptance/
16
+ ;;
17
+ console)
18
+ bash
19
+ ;;
20
+ *)
21
+ echo "Task not found"
22
+ exit 1
23
+ ;;
24
+ esac
@@ -0,0 +1,169 @@
1
+ module Backup
2
+ class Archive
3
+ class Error < Backup::Error; end
4
+
5
+ include Utilities::Helpers
6
+ attr_reader :name, :options
7
+
8
+ ##
9
+ # Adds a new Archive to a Backup Model.
10
+ #
11
+ # Backup::Model.new(:my_backup, 'My Backup') do
12
+ # archive :my_archive do |archive|
13
+ # archive.add 'path/to/archive'
14
+ # archive.add '/another/path/to/archive'
15
+ # archive.exclude 'path/to/exclude'
16
+ # archive.exclude '/another/path/to/exclude'
17
+ # end
18
+ # end
19
+ #
20
+ # All paths added using `add` or `exclude` will be expanded to their
21
+ # full paths from the root of the filesystem. Files will be added to
22
+ # the tar archive using these full paths, and their leading `/` will
23
+ # be preserved (using tar's `-P` option).
24
+ #
25
+ # /path/to/pwd/path/to/archive/...
26
+ # /another/path/to/archive/...
27
+ #
28
+ # When a `root` path is given, paths to add/exclude are taken as
29
+ # relative to the `root` path, unless given as absolute paths.
30
+ #
31
+ # Backup::Model.new(:my_backup, 'My Backup') do
32
+ # archive :my_archive do |archive|
33
+ # archive.root '~/my_data'
34
+ # archive.add 'path/to/archive'
35
+ # archive.add '/another/path/to/archive'
36
+ # archive.exclude 'path/to/exclude'
37
+ # archive.exclude '/another/path/to/exclude'
38
+ # end
39
+ # end
40
+ #
41
+ # This directs `tar` to change directories to the `root` path to create
42
+ # the archive. Unless paths were given as absolute, the paths within the
43
+ # archive will be relative to the `root` path.
44
+ #
45
+ # path/to/archive/...
46
+ # /another/path/to/archive/...
47
+ #
48
+ # For absolute paths added to this archive, the leading `/` will be
49
+ # preserved. Take note that when archives are extracted, leading `/` are
50
+ # stripped by default, so care must be taken when extracting archives with
51
+ # mixed relative/absolute paths.
52
+ def initialize(model, name, &block)
53
+ @model = model
54
+ @name = name.to_s
55
+ @options = {
56
+ sudo: false,
57
+ root: false,
58
+ paths: [],
59
+ excludes: [],
60
+ tar_options: ""
61
+ }
62
+ DSL.new(@options).instance_eval(&block)
63
+ end
64
+
65
+ def perform!
66
+ Logger.info "Creating Archive '#{name}'..."
67
+
68
+ path = File.join(Config.tmp_path, @model.trigger, "archives")
69
+ FileUtils.mkdir_p(path)
70
+
71
+ pipeline = Pipeline.new
72
+ with_files_from(paths_to_package) do |files_from|
73
+ pipeline.add(
74
+ "#{tar_command} #{tar_options} -cPf -#{tar_root} " \
75
+ "#{paths_to_exclude} #{files_from}",
76
+ tar_success_codes
77
+ )
78
+
79
+ extension = "tar"
80
+ if @model.compressor
81
+ @model.compressor.compress_with do |command, ext|
82
+ pipeline << command
83
+ extension << ext
84
+ end
85
+ end
86
+
87
+ pipeline << "#{utility(:cat)} > " \
88
+ "'#{File.join(path, "#{name}.#{extension}")}'"
89
+ pipeline.run
90
+ end
91
+
92
+ if pipeline.success?
93
+ Logger.info "Archive '#{name}' Complete!"
94
+ else
95
+ raise Error, "Failed to Create Archive '#{name}'\n" +
96
+ pipeline.error_messages
97
+ end
98
+ end
99
+
100
+ private
101
+
102
+ def tar_command
103
+ tar = utility(:tar)
104
+ options[:sudo] ? "#{utility(:sudo)} -n #{tar}" : tar
105
+ end
106
+
107
+ def tar_root
108
+ options[:root] ? " -C '#{File.expand_path(options[:root])}'" : ""
109
+ end
110
+
111
+ def paths_to_package
112
+ options[:paths].map { |path| prepare_path(path) }
113
+ end
114
+
115
+ def with_files_from(paths)
116
+ tmpfile = Tempfile.new("backup-archive-paths")
117
+ paths.each { |path| tmpfile.puts path }
118
+ tmpfile.close
119
+ yield "-T '#{tmpfile.path}'"
120
+ ensure
121
+ tmpfile.delete
122
+ end
123
+
124
+ def paths_to_exclude
125
+ options[:excludes].map do |path|
126
+ "--exclude='#{prepare_path(path)}'"
127
+ end.join(" ")
128
+ end
129
+
130
+ def prepare_path(path)
131
+ options[:root] ? path : File.expand_path(path)
132
+ end
133
+
134
+ def tar_options
135
+ args = options[:tar_options]
136
+ gnu_tar? ? "--ignore-failed-read #{args}".strip : args
137
+ end
138
+
139
+ def tar_success_codes
140
+ gnu_tar? ? [0, 1] : [0]
141
+ end
142
+
143
+ class DSL
144
+ def initialize(options)
145
+ @options = options
146
+ end
147
+
148
+ def use_sudo(val = true)
149
+ @options[:sudo] = val
150
+ end
151
+
152
+ def root(path)
153
+ @options[:root] = path
154
+ end
155
+
156
+ def add(path)
157
+ @options[:paths] << path
158
+ end
159
+
160
+ def exclude(path)
161
+ @options[:excludes] << path
162
+ end
163
+
164
+ def tar_options(opts)
165
+ @options[:tar_options] = opts
166
+ end
167
+ end
168
+ end
169
+ end
@@ -0,0 +1,18 @@
1
+ module Backup
2
+ class Binder
3
+ ##
4
+ # Creates a new Backup::Notifier::Binder instance. Loops through the provided
5
+ # Hash to set instance variables
6
+ def initialize(key_and_values)
7
+ key_and_values.each do |key, value|
8
+ instance_variable_set("@#{key}", value)
9
+ end
10
+ end
11
+
12
+ ##
13
+ # Returns the binding (needs a wrapper method because #binding is a private method)
14
+ def get_binding
15
+ binding
16
+ end
17
+ end
18
+ end
@@ -0,0 +1,112 @@
1
+ module Backup
2
+ module Cleaner
3
+ class Error < Backup::Error; end
4
+
5
+ class << self
6
+ ##
7
+ # Logs warnings if any temporary files still exist
8
+ # from the last time this model/trigger was run,
9
+ # then removes the files.
10
+ def prepare(model)
11
+ messages = []
12
+
13
+ packaging_folder = File.join(Config.tmp_path, model.trigger)
14
+ if File.exist?(packaging_folder)
15
+ messages << <<-EOS
16
+ The temporary packaging folder still exists!
17
+ '#{packaging_folder}'
18
+ It will now be removed.
19
+ EOS
20
+ FileUtils.rm_rf(packaging_folder)
21
+ end
22
+
23
+ package_files = package_files_for(model.trigger)
24
+ unless package_files.empty?
25
+ # the chances of the packaging folder AND
26
+ # the package files existing are practically nil
27
+ messages << ("-" * 74) unless messages.empty?
28
+
29
+ messages << <<-EOS
30
+ The temporary backup folder '#{Config.tmp_path}'
31
+ appears to contain the package files from the previous backup!
32
+ #{package_files.join("\n")}
33
+ These files will now be removed.
34
+ EOS
35
+ package_files.each { |file| FileUtils.rm_f(file) }
36
+ end
37
+
38
+ unless messages.empty?
39
+ Logger.warn Error.new(<<-EOS)
40
+ Cleanup Warning
41
+ #{messages.join("\n")}
42
+ Please check the log for messages and/or your notifications
43
+ concerning this backup: '#{model.label} (#{model.trigger})'
44
+ The temporary files which had to be removed should not have existed.
45
+ EOS
46
+ end
47
+ end
48
+
49
+ ##
50
+ # Remove the temporary folder used during packaging
51
+ def remove_packaging(model)
52
+ Logger.info "Cleaning up the temporary files..."
53
+ FileUtils.rm_rf(File.join(Config.tmp_path, model.trigger))
54
+ end
55
+
56
+ ##
57
+ # Remove the final package files from tmp_path
58
+ # Note: 'force' is used, since a Local Storage may *move* these files.
59
+ def remove_package(package)
60
+ Logger.info "Cleaning up the package files..."
61
+ package.filenames.each do |file|
62
+ FileUtils.rm_f(File.join(Config.tmp_path, file))
63
+ end
64
+ end
65
+
66
+ ##
67
+ # Logs warnings if any temporary files still exist
68
+ # when errors occur during the backup
69
+ def warnings(model)
70
+ messages = []
71
+
72
+ packaging_folder = File.join(Config.tmp_path, model.trigger)
73
+ if File.exist?(packaging_folder)
74
+ messages << <<-EOS
75
+ The temporary packaging folder still exists!
76
+ '#{packaging_folder}'
77
+ This folder may contain completed Archives and/or Database backups.
78
+ EOS
79
+ end
80
+
81
+ package_files = package_files_for(model.trigger)
82
+ unless package_files.empty?
83
+ # the chances of the packaging folder AND
84
+ # the package files existing are practically nil
85
+ messages << ("-" * 74) unless messages.empty?
86
+
87
+ messages << <<-EOS
88
+ The temporary backup folder '#{Config.tmp_path}'
89
+ appears to contain the backup files which were to be stored:
90
+ #{package_files.join("\n")}
91
+ EOS
92
+ end
93
+
94
+ unless messages.empty?
95
+ Logger.warn Error.new(<<-EOS)
96
+ Cleanup Warning
97
+ #{messages.join("\n")}
98
+ Make sure you check these files before the next scheduled backup for
99
+ '#{model.label} (#{model.trigger})'
100
+ These files will be removed at that time!
101
+ EOS
102
+ end
103
+ end
104
+
105
+ private
106
+
107
+ def package_files_for(trigger)
108
+ Dir[File.join(Config.tmp_path, "#{trigger}.tar{,[.-]*}")]
109
+ end
110
+ end
111
+ end
112
+ end