sliday_backup 0.1

Sign up to get free protection for your applications and to get access to all the features.
Files changed (135) hide show
  1. checksums.yaml +7 -0
  2. data/README.md +29 -0
  3. data/bin/sliday_backup +5 -0
  4. data/lib/sliday_backup.rb +147 -0
  5. data/lib/sliday_backup/archive.rb +170 -0
  6. data/lib/sliday_backup/binder.rb +22 -0
  7. data/lib/sliday_backup/cleaner.rb +116 -0
  8. data/lib/sliday_backup/cli.rb +374 -0
  9. data/lib/sliday_backup/cloud_io/base.rb +41 -0
  10. data/lib/sliday_backup/cloud_io/cloud_files.rb +298 -0
  11. data/lib/sliday_backup/cloud_io/s3.rb +260 -0
  12. data/lib/sliday_backup/compressor/base.rb +35 -0
  13. data/lib/sliday_backup/compressor/bzip2.rb +39 -0
  14. data/lib/sliday_backup/compressor/custom.rb +53 -0
  15. data/lib/sliday_backup/compressor/gzip.rb +74 -0
  16. data/lib/sliday_backup/config.rb +119 -0
  17. data/lib/sliday_backup/config/dsl.rb +103 -0
  18. data/lib/sliday_backup/config/helpers.rb +143 -0
  19. data/lib/sliday_backup/database/base.rb +86 -0
  20. data/lib/sliday_backup/database/mongodb.rb +187 -0
  21. data/lib/sliday_backup/database/mysql.rb +192 -0
  22. data/lib/sliday_backup/database/openldap.rb +95 -0
  23. data/lib/sliday_backup/database/postgresql.rb +133 -0
  24. data/lib/sliday_backup/database/redis.rb +179 -0
  25. data/lib/sliday_backup/database/riak.rb +82 -0
  26. data/lib/sliday_backup/database/sqlite.rb +57 -0
  27. data/lib/sliday_backup/encryptor/base.rb +29 -0
  28. data/lib/sliday_backup/encryptor/gpg.rb +747 -0
  29. data/lib/sliday_backup/encryptor/open_ssl.rb +77 -0
  30. data/lib/sliday_backup/errors.rb +58 -0
  31. data/lib/sliday_backup/logger.rb +199 -0
  32. data/lib/sliday_backup/logger/console.rb +51 -0
  33. data/lib/sliday_backup/logger/fog_adapter.rb +29 -0
  34. data/lib/sliday_backup/logger/logfile.rb +133 -0
  35. data/lib/sliday_backup/logger/syslog.rb +116 -0
  36. data/lib/sliday_backup/model.rb +479 -0
  37. data/lib/sliday_backup/notifier/base.rb +128 -0
  38. data/lib/sliday_backup/notifier/campfire.rb +63 -0
  39. data/lib/sliday_backup/notifier/command.rb +99 -0
  40. data/lib/sliday_backup/notifier/datadog.rb +107 -0
  41. data/lib/sliday_backup/notifier/flowdock.rb +103 -0
  42. data/lib/sliday_backup/notifier/hipchat.rb +112 -0
  43. data/lib/sliday_backup/notifier/http_post.rb +117 -0
  44. data/lib/sliday_backup/notifier/mail.rb +244 -0
  45. data/lib/sliday_backup/notifier/nagios.rb +69 -0
  46. data/lib/sliday_backup/notifier/pagerduty.rb +81 -0
  47. data/lib/sliday_backup/notifier/prowl.rb +68 -0
  48. data/lib/sliday_backup/notifier/pushover.rb +74 -0
  49. data/lib/sliday_backup/notifier/ses.rb +88 -0
  50. data/lib/sliday_backup/notifier/slack.rb +148 -0
  51. data/lib/sliday_backup/notifier/twitter.rb +58 -0
  52. data/lib/sliday_backup/notifier/zabbix.rb +63 -0
  53. data/lib/sliday_backup/package.rb +55 -0
  54. data/lib/sliday_backup/packager.rb +107 -0
  55. data/lib/sliday_backup/pipeline.rb +124 -0
  56. data/lib/sliday_backup/splitter.rb +76 -0
  57. data/lib/sliday_backup/storage/base.rb +69 -0
  58. data/lib/sliday_backup/storage/cloud_files.rb +158 -0
  59. data/lib/sliday_backup/storage/cycler.rb +75 -0
  60. data/lib/sliday_backup/storage/dropbox.rb +212 -0
  61. data/lib/sliday_backup/storage/ftp.rb +112 -0
  62. data/lib/sliday_backup/storage/local.rb +64 -0
  63. data/lib/sliday_backup/storage/qiniu.rb +65 -0
  64. data/lib/sliday_backup/storage/rsync.rb +248 -0
  65. data/lib/sliday_backup/storage/s3.rb +156 -0
  66. data/lib/sliday_backup/storage/scp.rb +67 -0
  67. data/lib/sliday_backup/storage/sftp.rb +82 -0
  68. data/lib/sliday_backup/storage/sliday_storage.rb +79 -0
  69. data/lib/sliday_backup/syncer/base.rb +70 -0
  70. data/lib/sliday_backup/syncer/cloud/base.rb +179 -0
  71. data/lib/sliday_backup/syncer/cloud/cloud_files.rb +83 -0
  72. data/lib/sliday_backup/syncer/cloud/local_file.rb +100 -0
  73. data/lib/sliday_backup/syncer/cloud/s3.rb +110 -0
  74. data/lib/sliday_backup/syncer/rsync/base.rb +54 -0
  75. data/lib/sliday_backup/syncer/rsync/local.rb +31 -0
  76. data/lib/sliday_backup/syncer/rsync/pull.rb +51 -0
  77. data/lib/sliday_backup/syncer/rsync/push.rb +205 -0
  78. data/lib/sliday_backup/template.rb +46 -0
  79. data/lib/sliday_backup/utilities.rb +224 -0
  80. data/lib/sliday_backup/version.rb +5 -0
  81. data/templates/cli/archive +28 -0
  82. data/templates/cli/compressor/bzip2 +4 -0
  83. data/templates/cli/compressor/custom +7 -0
  84. data/templates/cli/compressor/gzip +4 -0
  85. data/templates/cli/config +123 -0
  86. data/templates/cli/databases/mongodb +15 -0
  87. data/templates/cli/databases/mysql +18 -0
  88. data/templates/cli/databases/openldap +24 -0
  89. data/templates/cli/databases/postgresql +16 -0
  90. data/templates/cli/databases/redis +16 -0
  91. data/templates/cli/databases/riak +17 -0
  92. data/templates/cli/databases/sqlite +11 -0
  93. data/templates/cli/encryptor/gpg +27 -0
  94. data/templates/cli/encryptor/openssl +9 -0
  95. data/templates/cli/model +26 -0
  96. data/templates/cli/notifier/zabbix +15 -0
  97. data/templates/cli/notifiers/campfire +12 -0
  98. data/templates/cli/notifiers/command +32 -0
  99. data/templates/cli/notifiers/datadog +57 -0
  100. data/templates/cli/notifiers/flowdock +16 -0
  101. data/templates/cli/notifiers/hipchat +16 -0
  102. data/templates/cli/notifiers/http_post +32 -0
  103. data/templates/cli/notifiers/mail +24 -0
  104. data/templates/cli/notifiers/nagios +13 -0
  105. data/templates/cli/notifiers/pagerduty +12 -0
  106. data/templates/cli/notifiers/prowl +11 -0
  107. data/templates/cli/notifiers/pushover +11 -0
  108. data/templates/cli/notifiers/ses +15 -0
  109. data/templates/cli/notifiers/slack +22 -0
  110. data/templates/cli/notifiers/twitter +13 -0
  111. data/templates/cli/splitter +7 -0
  112. data/templates/cli/storages/cloud_files +11 -0
  113. data/templates/cli/storages/dropbox +20 -0
  114. data/templates/cli/storages/ftp +13 -0
  115. data/templates/cli/storages/local +8 -0
  116. data/templates/cli/storages/qiniu +12 -0
  117. data/templates/cli/storages/rsync +17 -0
  118. data/templates/cli/storages/s3 +16 -0
  119. data/templates/cli/storages/scp +15 -0
  120. data/templates/cli/storages/sftp +15 -0
  121. data/templates/cli/storages/sliday_storage +6 -0
  122. data/templates/cli/syncers/cloud_files +22 -0
  123. data/templates/cli/syncers/rsync_local +20 -0
  124. data/templates/cli/syncers/rsync_pull +28 -0
  125. data/templates/cli/syncers/rsync_push +28 -0
  126. data/templates/cli/syncers/s3 +27 -0
  127. data/templates/general/links +3 -0
  128. data/templates/general/version.erb +2 -0
  129. data/templates/notifier/mail/failure.erb +16 -0
  130. data/templates/notifier/mail/success.erb +16 -0
  131. data/templates/notifier/mail/warning.erb +16 -0
  132. data/templates/storage/dropbox/authorization_url.erb +6 -0
  133. data/templates/storage/dropbox/authorized.erb +4 -0
  134. data/templates/storage/dropbox/cache_file_written.erb +10 -0
  135. metadata +1079 -0
@@ -0,0 +1,156 @@
1
+ # encoding: utf-8
2
+ require 'sliday_backup/cloud_io/s3'
3
+
4
+ module SlidayBackup
5
+ module Storage
6
+ class S3 < Base
7
+ include Storage::Cycler
8
+ class Error < SlidayBackup::Error; end
9
+
10
+ ##
11
+ # Amazon Simple Storage Service (S3) Credentials
12
+ attr_accessor :access_key_id, :secret_access_key, :use_iam_profile
13
+
14
+ ##
15
+ # Amazon S3 bucket name
16
+ attr_accessor :bucket
17
+
18
+ ##
19
+ # Region of the specified S3 bucket
20
+ attr_accessor :region
21
+
22
+ ##
23
+ # Multipart chunk size, specified in MiB.
24
+ #
25
+ # Each package file larger than +chunk_size+
26
+ # will be uploaded using S3 Multipart Upload.
27
+ #
28
+ # Minimum: 5 (but may be disabled with 0)
29
+ # Maximum: 5120
30
+ # Default: 5
31
+ attr_accessor :chunk_size
32
+
33
+ ##
34
+ # Number of times to retry failed operations.
35
+ #
36
+ # Default: 10
37
+ attr_accessor :max_retries
38
+
39
+ ##
40
+ # Time in seconds to pause before each retry.
41
+ #
42
+ # Default: 30
43
+ attr_accessor :retry_waitsec
44
+
45
+ ##
46
+ # Encryption algorithm to use for Amazon Server-Side Encryption
47
+ #
48
+ # Supported values:
49
+ #
50
+ # - :aes256
51
+ #
52
+ # Default: nil
53
+ attr_accessor :encryption
54
+
55
+ ##
56
+ # Storage class to use for the S3 objects uploaded
57
+ #
58
+ # Supported values:
59
+ #
60
+ # - :standard (default)
61
+ # - :standard_ia
62
+ # - :reduced_redundancy
63
+ #
64
+ # Default: :standard
65
+ attr_accessor :storage_class
66
+
67
+ ##
68
+ # Additional options to pass along to fog.
69
+ # e.g. Fog::Storage.new({ :provider => 'AWS' }.merge(fog_options))
70
+ attr_accessor :fog_options
71
+
72
+ def initialize(model, storage_id = nil)
73
+ super
74
+
75
+ @chunk_size ||= 5 # MiB
76
+ @max_retries ||= 10
77
+ @retry_waitsec ||= 30
78
+ @path ||= 'backups'
79
+ @storage_class ||= :standard
80
+
81
+ @path = @path.sub(/^\//, '')
82
+
83
+ check_configuration
84
+ end
85
+
86
+ private
87
+
88
+ def cloud_io
89
+ @cloud_io ||= CloudIO::S3.new(
90
+ :access_key_id => access_key_id,
91
+ :secret_access_key => secret_access_key,
92
+ :use_iam_profile => use_iam_profile,
93
+ :region => region,
94
+ :bucket => bucket,
95
+ :encryption => encryption,
96
+ :storage_class => storage_class,
97
+ :max_retries => max_retries,
98
+ :retry_waitsec => retry_waitsec,
99
+ :chunk_size => chunk_size,
100
+ :fog_options => fog_options
101
+ )
102
+ end
103
+
104
+ def transfer!
105
+ package.filenames.each do |filename|
106
+ src = File.join(Config.tmp_path, filename)
107
+ dest = File.join(remote_path, filename)
108
+ Logger.info "Storing '#{ bucket }/#{ dest }'..."
109
+ cloud_io.upload(src, dest)
110
+ end
111
+ end
112
+
113
+ # Called by the Cycler.
114
+ # Any error raised will be logged as a warning.
115
+ def remove!(package)
116
+ Logger.info "Removing backup package dated #{ package.time }..."
117
+
118
+ remote_path = remote_path_for(package)
119
+ objects = cloud_io.objects(remote_path)
120
+
121
+ raise Error, "Package at '#{ remote_path }' not found" if objects.empty?
122
+
123
+ cloud_io.delete(objects)
124
+ end
125
+
126
+ def check_configuration
127
+ if use_iam_profile
128
+ required = %w{ bucket }
129
+ else
130
+ required = %w{ access_key_id secret_access_key bucket }
131
+ end
132
+ raise Error, <<-EOS if required.map {|name| send(name) }.any?(&:nil?)
133
+ Configuration Error
134
+ #{ required.map {|name| "##{ name }"}.join(', ') } are all required
135
+ EOS
136
+
137
+ raise Error, <<-EOS if chunk_size > 0 && !chunk_size.between?(5, 5120)
138
+ Configuration Error
139
+ #chunk_size must be between 5 and 5120 (or 0 to disable multipart)
140
+ EOS
141
+
142
+ raise Error, <<-EOS if encryption && encryption.to_s.upcase != 'AES256'
143
+ Configuration Error
144
+ #encryption must be :aes256 or nil
145
+ EOS
146
+
147
+ classes = ['STANDARD', 'STANDARD_IA', 'REDUCED_REDUNDANCY']
148
+ raise Error, <<-EOS unless classes.include?(storage_class.to_s.upcase)
149
+ Configuration Error
150
+ #storage_class must be :standard or :standard_ia or :reduced_redundancy
151
+ EOS
152
+ end
153
+
154
+ end
155
+ end
156
+ end
@@ -0,0 +1,67 @@
1
+ # encoding: utf-8
2
+ require 'net/scp'
3
+
4
+ module SlidayBackup
5
+ module Storage
6
+ class SCP < Base
7
+ include Storage::Cycler
8
+ class Error < SlidayBackup::Error; end
9
+
10
+ ##
11
+ # Server credentials
12
+ attr_accessor :username, :password, :ssh_options
13
+
14
+ ##
15
+ # Server IP Address and SCP port
16
+ attr_accessor :ip, :port
17
+
18
+ def initialize(model, storage_id = nil)
19
+ super
20
+
21
+ @port ||= 22
22
+ @path ||= 'backups'
23
+ @ssh_options ||= {}
24
+ path.sub!(/^~\//, '')
25
+ end
26
+
27
+ private
28
+
29
+ def connection
30
+ Net::SSH.start(
31
+ ip, username, { :password => password, :port => port }.merge(ssh_options)
32
+ ) {|ssh| yield ssh }
33
+ end
34
+
35
+ def transfer!
36
+ connection do |ssh|
37
+ ssh.exec!("mkdir -p '#{ remote_path }'")
38
+
39
+ package.filenames.each do |filename|
40
+ src = File.join(Config.tmp_path, filename)
41
+ dest = File.join(remote_path, filename)
42
+ Logger.info "Storing '#{ ip }:#{ dest }'..."
43
+ ssh.scp.upload!(src, dest)
44
+ end
45
+ end
46
+ end
47
+
48
+ # Called by the Cycler.
49
+ # Any error raised will be logged as a warning.
50
+ def remove!(package)
51
+ Logger.info "Removing backup package dated #{ package.time }..."
52
+
53
+ errors = []
54
+ connection do |ssh|
55
+ ssh.exec!("rm -r '#{ remote_path_for(package) }'") do |ch, stream, data|
56
+ errors << data if stream == :stderr
57
+ end
58
+ end
59
+ unless errors.empty?
60
+ raise Error, "Net::SSH reported the following errors:\n" +
61
+ errors.join("\n")
62
+ end
63
+ end
64
+
65
+ end
66
+ end
67
+ end
@@ -0,0 +1,82 @@
1
+ # encoding: utf-8
2
+ require 'net/sftp'
3
+
4
+ module SlidayBackup
5
+ module Storage
6
+ class SFTP < Base
7
+ include Storage::Cycler
8
+
9
+ ##
10
+ # Server credentials
11
+ attr_accessor :username, :password, :ssh_options
12
+
13
+ ##
14
+ # Server IP Address and SFTP port
15
+ attr_accessor :ip, :port
16
+
17
+ def initialize(model, storage_id = nil)
18
+ super
19
+
20
+ @ssh_options ||= {}
21
+ @port ||= 22
22
+ @path ||= 'backups'
23
+ path.sub!(/^~\//, '')
24
+ end
25
+
26
+ private
27
+
28
+ def connection
29
+ Net::SFTP.start(
30
+ ip, username, { :password => password, :port => port }.merge(ssh_options)
31
+ ) {|sftp| yield sftp }
32
+ end
33
+
34
+ def transfer!
35
+ connection do |sftp|
36
+ create_remote_path(sftp)
37
+
38
+ package.filenames.each do |filename|
39
+ src = File.join(Config.tmp_path, filename)
40
+ dest = File.join(remote_path, filename)
41
+ Logger.info "Storing '#{ ip }:#{ dest }'..."
42
+ sftp.upload!(src, dest)
43
+ end
44
+ end
45
+ end
46
+
47
+ # Called by the Cycler.
48
+ # Any error raised will be logged as a warning.
49
+ def remove!(package)
50
+ Logger.info "Removing backup package dated #{ package.time }..."
51
+
52
+ remote_path = remote_path_for(package)
53
+ connection do |sftp|
54
+ package.filenames.each do |filename|
55
+ sftp.remove!(File.join(remote_path, filename))
56
+ end
57
+
58
+ sftp.rmdir!(remote_path)
59
+ end
60
+ end
61
+
62
+ ##
63
+ # Creates (if they don't exist yet) all the directories on the remote
64
+ # server in order to upload the backup file. Net::SFTP does not support
65
+ # paths to directories that don't yet exist when creating new
66
+ # directories. Instead, we split the parts up in to an array (for each
67
+ # '/') and loop through that to create the directories one by one.
68
+ # Net::SFTP raises an exception when the directory it's trying to create
69
+ # already exists, so we have rescue it
70
+ def create_remote_path(sftp)
71
+ path_parts = Array.new
72
+ remote_path.split('/').each do |path_part|
73
+ path_parts << path_part
74
+ begin
75
+ sftp.mkdir!(path_parts.join('/'))
76
+ rescue Net::SFTP::StatusException; end
77
+ end
78
+ end
79
+
80
+ end
81
+ end
82
+ end
@@ -0,0 +1,79 @@
1
+ # encoding: utf-8
2
+ require "net/http"
3
+ require "uri"
4
+ require 'json'
5
+
6
+ module SlidayBackup
7
+ module Storage
8
+ class SlidayStorage < Base
9
+ include Storage::Cycler
10
+ class Error < SlidayBackup::Error;
11
+ end
12
+
13
+ attr_accessor :api_key
14
+ attr_accessor :backup_host
15
+
16
+ def initialize(model, storage_id = nil)
17
+ super
18
+ @backup_host ||= 'https://sliday-backups.herokuapp.com'
19
+ end
20
+
21
+ private
22
+
23
+ def transfer!
24
+ uri = URI.parse("#{@backup_host}/api/#{@api_key}")
25
+ response = Net::HTTP.post_form(uri, {})
26
+ if response.code != "200"
27
+ raise Error.wrap(Exception.new, 'Authorization Failed')
28
+ end
29
+ backup = JSON.parse(response.body)
30
+ upload_url = URI.parse(backup['upload_url'])
31
+ package.filenames.each do |filename|
32
+ src = File.join(Config.tmp_path, filename)
33
+ Net::HTTP.start(upload_url.host) do |http|
34
+ http.send_request("PUT", upload_url.request_uri, File.read(src), {
35
+ "content-type" => "",
36
+ })
37
+ end
38
+ Logger.info "Uploading..."
39
+ end
40
+ finalize_uri = URI.parse("#{@backup_host}/api/#{@api_key}/#{backup['id']}/finish")
41
+ Net::HTTP.post_form(finalize_uri, {})
42
+ end
43
+
44
+ # Called by the Cycler.
45
+ # Any error raised will be logged as a warning.
46
+ def remove!(package)
47
+ Logger.info "Removing backup package dated #{ package.time }..."
48
+
49
+ FileUtils.rm_r(remote_path_for(package))
50
+ end
51
+
52
+ # expanded since this is a local path
53
+ def remote_path(pkg = package)
54
+ File.expand_path(super)
55
+ end
56
+
57
+ alias :remote_path_for :remote_path
58
+
59
+ ##
60
+ # If this Local Storage is not the last Storage for the Model,
61
+ # force the transfer to use a *copy* operation and issue a warning.
62
+ def package_movable?
63
+ if self == model.storages.last
64
+ true
65
+ else
66
+ Logger.warn Error.new(<<-EOS)
67
+ Local File Copy Warning!
68
+ The final backup file(s) for '#{ model.label }' (#{ model.trigger })
69
+ will be *copied* to '#{ remote_path }'
70
+ To avoid this, when using more than one Storage, the 'Local' Storage
71
+ should be added *last* so the files may be *moved* to their destination.
72
+ EOS
73
+ false
74
+ end
75
+ end
76
+
77
+ end
78
+ end
79
+ end
@@ -0,0 +1,70 @@
1
+ # encoding: utf-8
2
+
3
+ module SlidayBackup
4
+ module Syncer
5
+ class Base
6
+ include Utilities::Helpers
7
+ include Config::Helpers
8
+
9
+ ##
10
+ # Path to store the synced files/directories to
11
+ attr_accessor :path
12
+
13
+ ##
14
+ # Flag for mirroring the files/directories
15
+ attr_accessor :mirror
16
+
17
+ ##
18
+ # Optional user-defined identifier to differentiate multiple syncers
19
+ # defined within a single backup model. Currently this is only used
20
+ # in the log messages.
21
+ attr_reader :syncer_id
22
+
23
+ attr_reader :excludes
24
+
25
+ def initialize(syncer_id = nil)
26
+ @syncer_id = syncer_id
27
+
28
+ load_defaults!
29
+
30
+ @mirror ||= false
31
+ @directories ||= []
32
+ @excludes ||= []
33
+ end
34
+
35
+ ##
36
+ # Syntactical suger for the DSL for adding directories
37
+ def directories(&block)
38
+ return @directories unless block_given?
39
+ instance_eval(&block)
40
+ end
41
+
42
+ def add(path)
43
+ directories << path
44
+ end
45
+
46
+ # For Cloud Syncers, +pattern+ can be a string (with shell-style
47
+ # wildcards) or a regex.
48
+ # For RSync, each +pattern+ will be passed to rsync's --exclude option.
49
+ def exclude(pattern)
50
+ excludes << pattern
51
+ end
52
+
53
+ private
54
+
55
+ def syncer_name
56
+ @syncer_name ||= self.class.to_s.sub('SlidayBackup::', '') +
57
+ (syncer_id ? " (#{ syncer_id })" : '')
58
+ end
59
+
60
+ def log!(action)
61
+ msg = case action
62
+ when :started then 'Started...'
63
+ when :finished then 'Finished!'
64
+ end
65
+ Logger.info "#{ syncer_name } #{ msg }"
66
+ end
67
+
68
+ end
69
+ end
70
+ end