sliday_backup 0.1

Sign up to get free protection for your applications and to get access to all the features.
Files changed (135) hide show
  1. checksums.yaml +7 -0
  2. data/README.md +29 -0
  3. data/bin/sliday_backup +5 -0
  4. data/lib/sliday_backup.rb +147 -0
  5. data/lib/sliday_backup/archive.rb +170 -0
  6. data/lib/sliday_backup/binder.rb +22 -0
  7. data/lib/sliday_backup/cleaner.rb +116 -0
  8. data/lib/sliday_backup/cli.rb +374 -0
  9. data/lib/sliday_backup/cloud_io/base.rb +41 -0
  10. data/lib/sliday_backup/cloud_io/cloud_files.rb +298 -0
  11. data/lib/sliday_backup/cloud_io/s3.rb +260 -0
  12. data/lib/sliday_backup/compressor/base.rb +35 -0
  13. data/lib/sliday_backup/compressor/bzip2.rb +39 -0
  14. data/lib/sliday_backup/compressor/custom.rb +53 -0
  15. data/lib/sliday_backup/compressor/gzip.rb +74 -0
  16. data/lib/sliday_backup/config.rb +119 -0
  17. data/lib/sliday_backup/config/dsl.rb +103 -0
  18. data/lib/sliday_backup/config/helpers.rb +143 -0
  19. data/lib/sliday_backup/database/base.rb +86 -0
  20. data/lib/sliday_backup/database/mongodb.rb +187 -0
  21. data/lib/sliday_backup/database/mysql.rb +192 -0
  22. data/lib/sliday_backup/database/openldap.rb +95 -0
  23. data/lib/sliday_backup/database/postgresql.rb +133 -0
  24. data/lib/sliday_backup/database/redis.rb +179 -0
  25. data/lib/sliday_backup/database/riak.rb +82 -0
  26. data/lib/sliday_backup/database/sqlite.rb +57 -0
  27. data/lib/sliday_backup/encryptor/base.rb +29 -0
  28. data/lib/sliday_backup/encryptor/gpg.rb +747 -0
  29. data/lib/sliday_backup/encryptor/open_ssl.rb +77 -0
  30. data/lib/sliday_backup/errors.rb +58 -0
  31. data/lib/sliday_backup/logger.rb +199 -0
  32. data/lib/sliday_backup/logger/console.rb +51 -0
  33. data/lib/sliday_backup/logger/fog_adapter.rb +29 -0
  34. data/lib/sliday_backup/logger/logfile.rb +133 -0
  35. data/lib/sliday_backup/logger/syslog.rb +116 -0
  36. data/lib/sliday_backup/model.rb +479 -0
  37. data/lib/sliday_backup/notifier/base.rb +128 -0
  38. data/lib/sliday_backup/notifier/campfire.rb +63 -0
  39. data/lib/sliday_backup/notifier/command.rb +99 -0
  40. data/lib/sliday_backup/notifier/datadog.rb +107 -0
  41. data/lib/sliday_backup/notifier/flowdock.rb +103 -0
  42. data/lib/sliday_backup/notifier/hipchat.rb +112 -0
  43. data/lib/sliday_backup/notifier/http_post.rb +117 -0
  44. data/lib/sliday_backup/notifier/mail.rb +244 -0
  45. data/lib/sliday_backup/notifier/nagios.rb +69 -0
  46. data/lib/sliday_backup/notifier/pagerduty.rb +81 -0
  47. data/lib/sliday_backup/notifier/prowl.rb +68 -0
  48. data/lib/sliday_backup/notifier/pushover.rb +74 -0
  49. data/lib/sliday_backup/notifier/ses.rb +88 -0
  50. data/lib/sliday_backup/notifier/slack.rb +148 -0
  51. data/lib/sliday_backup/notifier/twitter.rb +58 -0
  52. data/lib/sliday_backup/notifier/zabbix.rb +63 -0
  53. data/lib/sliday_backup/package.rb +55 -0
  54. data/lib/sliday_backup/packager.rb +107 -0
  55. data/lib/sliday_backup/pipeline.rb +124 -0
  56. data/lib/sliday_backup/splitter.rb +76 -0
  57. data/lib/sliday_backup/storage/base.rb +69 -0
  58. data/lib/sliday_backup/storage/cloud_files.rb +158 -0
  59. data/lib/sliday_backup/storage/cycler.rb +75 -0
  60. data/lib/sliday_backup/storage/dropbox.rb +212 -0
  61. data/lib/sliday_backup/storage/ftp.rb +112 -0
  62. data/lib/sliday_backup/storage/local.rb +64 -0
  63. data/lib/sliday_backup/storage/qiniu.rb +65 -0
  64. data/lib/sliday_backup/storage/rsync.rb +248 -0
  65. data/lib/sliday_backup/storage/s3.rb +156 -0
  66. data/lib/sliday_backup/storage/scp.rb +67 -0
  67. data/lib/sliday_backup/storage/sftp.rb +82 -0
  68. data/lib/sliday_backup/storage/sliday_storage.rb +79 -0
  69. data/lib/sliday_backup/syncer/base.rb +70 -0
  70. data/lib/sliday_backup/syncer/cloud/base.rb +179 -0
  71. data/lib/sliday_backup/syncer/cloud/cloud_files.rb +83 -0
  72. data/lib/sliday_backup/syncer/cloud/local_file.rb +100 -0
  73. data/lib/sliday_backup/syncer/cloud/s3.rb +110 -0
  74. data/lib/sliday_backup/syncer/rsync/base.rb +54 -0
  75. data/lib/sliday_backup/syncer/rsync/local.rb +31 -0
  76. data/lib/sliday_backup/syncer/rsync/pull.rb +51 -0
  77. data/lib/sliday_backup/syncer/rsync/push.rb +205 -0
  78. data/lib/sliday_backup/template.rb +46 -0
  79. data/lib/sliday_backup/utilities.rb +224 -0
  80. data/lib/sliday_backup/version.rb +5 -0
  81. data/templates/cli/archive +28 -0
  82. data/templates/cli/compressor/bzip2 +4 -0
  83. data/templates/cli/compressor/custom +7 -0
  84. data/templates/cli/compressor/gzip +4 -0
  85. data/templates/cli/config +123 -0
  86. data/templates/cli/databases/mongodb +15 -0
  87. data/templates/cli/databases/mysql +18 -0
  88. data/templates/cli/databases/openldap +24 -0
  89. data/templates/cli/databases/postgresql +16 -0
  90. data/templates/cli/databases/redis +16 -0
  91. data/templates/cli/databases/riak +17 -0
  92. data/templates/cli/databases/sqlite +11 -0
  93. data/templates/cli/encryptor/gpg +27 -0
  94. data/templates/cli/encryptor/openssl +9 -0
  95. data/templates/cli/model +26 -0
  96. data/templates/cli/notifier/zabbix +15 -0
  97. data/templates/cli/notifiers/campfire +12 -0
  98. data/templates/cli/notifiers/command +32 -0
  99. data/templates/cli/notifiers/datadog +57 -0
  100. data/templates/cli/notifiers/flowdock +16 -0
  101. data/templates/cli/notifiers/hipchat +16 -0
  102. data/templates/cli/notifiers/http_post +32 -0
  103. data/templates/cli/notifiers/mail +24 -0
  104. data/templates/cli/notifiers/nagios +13 -0
  105. data/templates/cli/notifiers/pagerduty +12 -0
  106. data/templates/cli/notifiers/prowl +11 -0
  107. data/templates/cli/notifiers/pushover +11 -0
  108. data/templates/cli/notifiers/ses +15 -0
  109. data/templates/cli/notifiers/slack +22 -0
  110. data/templates/cli/notifiers/twitter +13 -0
  111. data/templates/cli/splitter +7 -0
  112. data/templates/cli/storages/cloud_files +11 -0
  113. data/templates/cli/storages/dropbox +20 -0
  114. data/templates/cli/storages/ftp +13 -0
  115. data/templates/cli/storages/local +8 -0
  116. data/templates/cli/storages/qiniu +12 -0
  117. data/templates/cli/storages/rsync +17 -0
  118. data/templates/cli/storages/s3 +16 -0
  119. data/templates/cli/storages/scp +15 -0
  120. data/templates/cli/storages/sftp +15 -0
  121. data/templates/cli/storages/sliday_storage +6 -0
  122. data/templates/cli/syncers/cloud_files +22 -0
  123. data/templates/cli/syncers/rsync_local +20 -0
  124. data/templates/cli/syncers/rsync_pull +28 -0
  125. data/templates/cli/syncers/rsync_push +28 -0
  126. data/templates/cli/syncers/s3 +27 -0
  127. data/templates/general/links +3 -0
  128. data/templates/general/version.erb +2 -0
  129. data/templates/notifier/mail/failure.erb +16 -0
  130. data/templates/notifier/mail/success.erb +16 -0
  131. data/templates/notifier/mail/warning.erb +16 -0
  132. data/templates/storage/dropbox/authorization_url.erb +6 -0
  133. data/templates/storage/dropbox/authorized.erb +4 -0
  134. data/templates/storage/dropbox/cache_file_written.erb +10 -0
  135. metadata +1079 -0
@@ -0,0 +1,133 @@
1
+ # encoding: utf-8
2
+
3
+ module SlidayBackup
4
+ module Database
5
+ class PostgreSQL < Base
6
+ class Error < SlidayBackup::Error; end
7
+
8
+ ##
9
+ # Name of the database that needs to get dumped.
10
+ # To dump all databases, set this to `:all` or leave blank.
11
+ # +username+ must be a PostgreSQL superuser to run `pg_dumpall`.
12
+ attr_accessor :name
13
+
14
+ ##
15
+ # Credentials for the specified database
16
+ attr_accessor :username, :password
17
+
18
+ ##
19
+ # If set the pg_dump(all) command is executed as the given user
20
+ attr_accessor :sudo_user
21
+
22
+ ##
23
+ # Connectivity options
24
+ attr_accessor :host, :port, :socket
25
+
26
+ ##
27
+ # Tables to skip while dumping the database.
28
+ # If `name` is set to :all (or not specified), these are ignored.
29
+ attr_accessor :skip_tables
30
+
31
+ ##
32
+ # Tables to dump. This in only valid if `name` is specified.
33
+ # If none are given, the entire database will be dumped.
34
+ attr_accessor :only_tables
35
+
36
+ ##
37
+ # Additional "pg_dump" or "pg_dumpall" options
38
+ attr_accessor :additional_options
39
+
40
+ def initialize(model, database_id = nil, &block)
41
+ super
42
+ instance_eval(&block) if block_given?
43
+
44
+ @name ||= :all
45
+ end
46
+
47
+ ##
48
+ # Performs the pgdump command and outputs the dump file
49
+ # in the +dump_path+ using +dump_filename+.
50
+ #
51
+ # <trigger>/databases/PostgreSQL[-<database_id>].sql[.gz]
52
+ def perform!
53
+ super
54
+
55
+ pipeline = Pipeline.new
56
+ dump_ext = 'sql'
57
+
58
+ pipeline << (dump_all? ? pgdumpall : pgdump)
59
+
60
+ model.compressor.compress_with do |command, ext|
61
+ pipeline << command
62
+ dump_ext << ext
63
+ end if model.compressor
64
+
65
+ pipeline << "#{ utility(:cat) } > " +
66
+ "'#{ File.join(dump_path, dump_filename) }.#{ dump_ext }'"
67
+
68
+ pipeline.run
69
+ if pipeline.success?
70
+ log!(:finished)
71
+ else
72
+ raise Error, "Dump Failed!\n" + pipeline.error_messages
73
+ end
74
+ end
75
+
76
+ def pgdump
77
+ "#{ password_option }" +
78
+ "#{ sudo_option }" +
79
+ "#{ utility(:pg_dump) } #{ username_option } #{ connectivity_options } " +
80
+ "#{ user_options } #{ tables_to_dump } #{ tables_to_skip } #{ name }"
81
+ end
82
+
83
+ def pgdumpall
84
+ "#{ password_option }" +
85
+ "#{ sudo_option }" +
86
+ "#{ utility(:pg_dumpall) } #{ username_option } " +
87
+ "#{ connectivity_options } #{ user_options }"
88
+ end
89
+
90
+ def password_option
91
+ "PGPASSWORD=#{ Shellwords.escape(password) } " if password
92
+ end
93
+
94
+ def sudo_option
95
+ "#{ utility(:sudo) } -n -u #{ sudo_user } " if sudo_user
96
+ end
97
+
98
+ def username_option
99
+ "--username=#{ Shellwords.escape(username) }" if username
100
+ end
101
+
102
+ def connectivity_options
103
+ return "--host='#{ socket }'" if socket
104
+
105
+ opts = []
106
+ opts << "--host='#{ host }'" if host
107
+ opts << "--port='#{ port }'" if port
108
+ opts.join(' ')
109
+ end
110
+
111
+ def user_options
112
+ Array(additional_options).join(' ')
113
+ end
114
+
115
+ def tables_to_dump
116
+ Array(only_tables).map do |table|
117
+ "--table='#{ table }'"
118
+ end.join(' ')
119
+ end
120
+
121
+ def tables_to_skip
122
+ Array(skip_tables).map do |table|
123
+ "--exclude-table='#{ table }'"
124
+ end.join(' ')
125
+ end
126
+
127
+ def dump_all?
128
+ name == :all
129
+ end
130
+
131
+ end
132
+ end
133
+ end
@@ -0,0 +1,179 @@
1
+ # encoding: utf-8
2
+
3
+ module SlidayBackup
4
+ module Database
5
+ class Redis < Base
6
+ class Error < SlidayBackup::Error; end
7
+
8
+ MODES = [:copy, :sync]
9
+
10
+ ##
11
+ # Mode of operation.
12
+ #
13
+ # [:copy]
14
+ # Copies the redis dump file specified by {#rdb_path}.
15
+ # This data will be current as of the last RDB Snapshot
16
+ # performed by the server (per your redis.conf settings).
17
+ # You may set {#invoke_save} to +true+ to have SlidayBackup issue
18
+ # a +SAVE+ command to update the dump file with the current
19
+ # data before performing the copy.
20
+ #
21
+ # [:sync]
22
+ # Performs a dump of your redis data using +redis-cli --rdb -+.
23
+ # Redis implements this internally using a +SYNC+ command.
24
+ # The operation is analogous to requesting a +BGSAVE+, then having the
25
+ # dump returned. This mode is capable of dumping data from a local or
26
+ # remote server. Requires Redis v2.6 or better.
27
+ #
28
+ # Defaults to +:copy+.
29
+ attr_accessor :mode
30
+
31
+ ##
32
+ # Full path to the redis dump file.
33
+ #
34
+ # Required when {#mode} is +:copy+.
35
+ attr_accessor :rdb_path
36
+
37
+ ##
38
+ # Perform a +SAVE+ command using the +redis-cli+ utility
39
+ # before copying the dump file specified by {#rdb_path}.
40
+ #
41
+ # Only valid when {#mode} is +:copy+.
42
+ attr_accessor :invoke_save
43
+
44
+ ##
45
+ # Connectivity options for the +redis-cli+ utility.
46
+ attr_accessor :host, :port, :socket
47
+
48
+ ##
49
+ # Password for the +redis-cli+ utility.
50
+ attr_accessor :password
51
+
52
+ ##
53
+ # Additional options for the +redis-cli+ utility.
54
+ attr_accessor :additional_options
55
+
56
+ def initialize(model, database_id = nil, &block)
57
+ super
58
+ instance_eval(&block) if block_given?
59
+
60
+ @mode ||= :copy
61
+
62
+ unless MODES.include?(mode)
63
+ raise Error, "'#{ mode }' is not a valid mode"
64
+ end
65
+
66
+ if mode == :copy && rdb_path.nil?
67
+ raise Error, '`rdb_path` must be set when `mode` is :copy'
68
+ end
69
+ end
70
+
71
+ ##
72
+ # Performs the dump based on {#mode} and stores the Redis dump file
73
+ # to the +dump_path+ using the +dump_filename+.
74
+ #
75
+ # <trigger>/databases/Redis[-<database_id>].rdb[.gz]
76
+ def perform!
77
+ super
78
+
79
+ case mode
80
+ when :sync
81
+ # messages output by `redis-cli --rdb` on $stderr
82
+ Logger.configure do
83
+ ignore_warning(/Transfer finished with success/)
84
+ ignore_warning(/SYNC sent to master/)
85
+ end
86
+ sync!
87
+ when :copy
88
+ save! if invoke_save
89
+ copy!
90
+ end
91
+
92
+ log!(:finished)
93
+ end
94
+
95
+ private
96
+
97
+ def sync!
98
+ pipeline = Pipeline.new
99
+ dump_ext = 'rdb'
100
+
101
+ pipeline << "#{ redis_cli_cmd } --rdb -"
102
+
103
+ model.compressor.compress_with do |command, ext|
104
+ pipeline << command
105
+ dump_ext << ext
106
+ end if model.compressor
107
+
108
+ pipeline << "#{ utility(:cat) } > " +
109
+ "'#{ File.join(dump_path, dump_filename) }.#{ dump_ext }'"
110
+
111
+ pipeline.run
112
+
113
+ unless pipeline.success?
114
+ raise Error, "Dump Failed!\n" + pipeline.error_messages
115
+ end
116
+ end
117
+
118
+ def save!
119
+ resp = run("#{ redis_cli_cmd } SAVE")
120
+ unless resp =~ /OK$/
121
+ raise Error, <<-EOS
122
+ Failed to invoke the `SAVE` command
123
+ Response was: #{ resp }
124
+ EOS
125
+ end
126
+
127
+ rescue Error
128
+ if resp =~ /save already in progress/
129
+ unless (attempts ||= '0').next! == '5'
130
+ sleep 5
131
+ retry
132
+ end
133
+ end
134
+ raise
135
+ end
136
+
137
+ def copy!
138
+ unless File.exist?(rdb_path)
139
+ raise Error, <<-EOS
140
+ Redis database dump not found
141
+ `rdb_path` was '#{ rdb_path }'
142
+ EOS
143
+ end
144
+
145
+ dst_path = File.join(dump_path, dump_filename + '.rdb')
146
+ if model.compressor
147
+ model.compressor.compress_with do |command, ext|
148
+ run("#{ command } -c '#{ rdb_path }' > '#{ dst_path + ext }'")
149
+ end
150
+ else
151
+ FileUtils.cp(rdb_path, dst_path)
152
+ end
153
+ end
154
+
155
+ def redis_cli_cmd
156
+ "#{ utility('redis-cli') } #{ password_option } " +
157
+ "#{ connectivity_options } #{ user_options }"
158
+ end
159
+
160
+ def password_option
161
+ "-a '#{ password }'" if password
162
+ end
163
+
164
+ def connectivity_options
165
+ return "-s '#{ socket }'" if socket
166
+
167
+ opts = []
168
+ opts << "-h '#{ host }'" if host
169
+ opts << "-p '#{ port }'" if port
170
+ opts.join(' ')
171
+ end
172
+
173
+ def user_options
174
+ Array(additional_options).join(' ')
175
+ end
176
+
177
+ end
178
+ end
179
+ end
@@ -0,0 +1,82 @@
1
+ # encoding: utf-8
2
+
3
+ module SlidayBackup
4
+ module Database
5
+ class Riak < Base
6
+
7
+ ##
8
+ # Node is the node from which to perform the backup.
9
+ # Default: riak@127.0.0.1
10
+ attr_accessor :node
11
+
12
+ ##
13
+ # Cookie is the Erlang cookie/shared secret used to connect to the node.
14
+ # Default: riak
15
+ attr_accessor :cookie
16
+
17
+ ##
18
+ # Username for the riak instance
19
+ # Default: riak
20
+ attr_accessor :user
21
+
22
+ def initialize(model, database_id = nil, &block)
23
+ super
24
+ instance_eval(&block) if block_given?
25
+
26
+ @node ||= 'riak@127.0.0.1'
27
+ @cookie ||= 'riak'
28
+ @user ||= 'riak'
29
+ end
30
+
31
+ ##
32
+ # Performs the dump using `riak-admin backup`.
33
+ #
34
+ # This will be stored in the final backup package as
35
+ # <trigger>/databases/<dump_filename>-<node>[.gz]
36
+ def perform!
37
+ super
38
+
39
+ dump_file = File.join(dump_path, dump_filename)
40
+ with_riak_owned_dump_path do
41
+ run("#{ riakadmin } backup #{ node } #{ cookie } '#{ dump_file }' node")
42
+ end
43
+
44
+ model.compressor.compress_with do |command, ext|
45
+ dump_file << "-#{ node }" # `riak-admin` appends `node` to the filename.
46
+ run("#{ command } -c '#{ dump_file }' > '#{ dump_file + ext }'")
47
+ FileUtils.rm_f(dump_file)
48
+ end if model.compressor
49
+
50
+ log!(:finished)
51
+ end
52
+
53
+ private
54
+
55
+ ##
56
+ # The `riak-admin backup` command is run as the riak +user+,
57
+ # so +user+ must have write priviledges to the +dump_path+.
58
+ #
59
+ # Note that the riak +user+ must also have access to +dump_path+.
60
+ # This means SlidayBackup's +tmp_path+ can not be under the home directory of
61
+ # the user running SlidayBackup, since the absence of the execute bit on their
62
+ # home directory would deny +user+ access.
63
+ def with_riak_owned_dump_path
64
+ run("#{ utility(:sudo) } -n #{ utility(:chown) } " +
65
+ "#{ user } '#{ dump_path }'")
66
+ yield
67
+ ensure
68
+ # reclaim ownership
69
+ run("#{ utility(:sudo) } -n #{ utility(:chown) } -R " +
70
+ "#{ Config.user } '#{ dump_path }'")
71
+ end
72
+
73
+ ##
74
+ # `riak-admin` must be run as the riak +user+.
75
+ # It will do this itself, but without `-n` and emits a message on STDERR.
76
+ def riakadmin
77
+ "#{ utility(:sudo) } -n -u #{ user } #{ utility('riak-admin') }"
78
+ end
79
+
80
+ end
81
+ end
82
+ end
@@ -0,0 +1,57 @@
1
+ # encoding: utf-8
2
+
3
+ module SlidayBackup
4
+ module Database
5
+ class SQLite < Base
6
+ class Error < SlidayBackup::Error; end
7
+
8
+ ##
9
+ # Path to the sqlite3 file
10
+ attr_accessor :path
11
+
12
+ ##
13
+ # Path to sqlite utility (optional)
14
+ attr_accessor :sqlitedump_utility
15
+
16
+ ##
17
+ # Creates a new instance of the SQLite adapter object
18
+ def initialize(model, database_id = nil, &block)
19
+ super
20
+ instance_eval(&block) if block_given?
21
+
22
+ @sqlitedump_utility ||= utility(:sqlitedump)
23
+ end
24
+
25
+ ##
26
+ # Performs the sqlitedump command and outputs the
27
+ # data to the specified path based on the 'trigger'
28
+ def perform!
29
+ super
30
+
31
+ dump = "echo '.dump' | #{ sqlitedump_utility } #{ path }"
32
+
33
+ pipeline = Pipeline.new
34
+ dump_ext = 'sql'
35
+
36
+ pipeline << dump
37
+ if model.compressor
38
+ model.compressor.compress_with do |command, ext|
39
+ pipeline << command
40
+ dump_ext << ext
41
+ end
42
+ end
43
+
44
+ pipeline << "cat > '#{ File.join( dump_path , dump_filename) }.#{ dump_ext }'"
45
+
46
+ pipeline.run
47
+
48
+ if pipeline.success?
49
+ log!(:finished)
50
+ else
51
+ raise Error,
52
+ "#{ database_name } Dump Failed!\n" + pipeline.error_messages
53
+ end
54
+ end
55
+ end
56
+ end
57
+ end