sliday_backup 0.1

Sign up to get free protection for your applications and to get access to all the features.
Files changed (135) hide show
  1. checksums.yaml +7 -0
  2. data/README.md +29 -0
  3. data/bin/sliday_backup +5 -0
  4. data/lib/sliday_backup.rb +147 -0
  5. data/lib/sliday_backup/archive.rb +170 -0
  6. data/lib/sliday_backup/binder.rb +22 -0
  7. data/lib/sliday_backup/cleaner.rb +116 -0
  8. data/lib/sliday_backup/cli.rb +374 -0
  9. data/lib/sliday_backup/cloud_io/base.rb +41 -0
  10. data/lib/sliday_backup/cloud_io/cloud_files.rb +298 -0
  11. data/lib/sliday_backup/cloud_io/s3.rb +260 -0
  12. data/lib/sliday_backup/compressor/base.rb +35 -0
  13. data/lib/sliday_backup/compressor/bzip2.rb +39 -0
  14. data/lib/sliday_backup/compressor/custom.rb +53 -0
  15. data/lib/sliday_backup/compressor/gzip.rb +74 -0
  16. data/lib/sliday_backup/config.rb +119 -0
  17. data/lib/sliday_backup/config/dsl.rb +103 -0
  18. data/lib/sliday_backup/config/helpers.rb +143 -0
  19. data/lib/sliday_backup/database/base.rb +86 -0
  20. data/lib/sliday_backup/database/mongodb.rb +187 -0
  21. data/lib/sliday_backup/database/mysql.rb +192 -0
  22. data/lib/sliday_backup/database/openldap.rb +95 -0
  23. data/lib/sliday_backup/database/postgresql.rb +133 -0
  24. data/lib/sliday_backup/database/redis.rb +179 -0
  25. data/lib/sliday_backup/database/riak.rb +82 -0
  26. data/lib/sliday_backup/database/sqlite.rb +57 -0
  27. data/lib/sliday_backup/encryptor/base.rb +29 -0
  28. data/lib/sliday_backup/encryptor/gpg.rb +747 -0
  29. data/lib/sliday_backup/encryptor/open_ssl.rb +77 -0
  30. data/lib/sliday_backup/errors.rb +58 -0
  31. data/lib/sliday_backup/logger.rb +199 -0
  32. data/lib/sliday_backup/logger/console.rb +51 -0
  33. data/lib/sliday_backup/logger/fog_adapter.rb +29 -0
  34. data/lib/sliday_backup/logger/logfile.rb +133 -0
  35. data/lib/sliday_backup/logger/syslog.rb +116 -0
  36. data/lib/sliday_backup/model.rb +479 -0
  37. data/lib/sliday_backup/notifier/base.rb +128 -0
  38. data/lib/sliday_backup/notifier/campfire.rb +63 -0
  39. data/lib/sliday_backup/notifier/command.rb +99 -0
  40. data/lib/sliday_backup/notifier/datadog.rb +107 -0
  41. data/lib/sliday_backup/notifier/flowdock.rb +103 -0
  42. data/lib/sliday_backup/notifier/hipchat.rb +112 -0
  43. data/lib/sliday_backup/notifier/http_post.rb +117 -0
  44. data/lib/sliday_backup/notifier/mail.rb +244 -0
  45. data/lib/sliday_backup/notifier/nagios.rb +69 -0
  46. data/lib/sliday_backup/notifier/pagerduty.rb +81 -0
  47. data/lib/sliday_backup/notifier/prowl.rb +68 -0
  48. data/lib/sliday_backup/notifier/pushover.rb +74 -0
  49. data/lib/sliday_backup/notifier/ses.rb +88 -0
  50. data/lib/sliday_backup/notifier/slack.rb +148 -0
  51. data/lib/sliday_backup/notifier/twitter.rb +58 -0
  52. data/lib/sliday_backup/notifier/zabbix.rb +63 -0
  53. data/lib/sliday_backup/package.rb +55 -0
  54. data/lib/sliday_backup/packager.rb +107 -0
  55. data/lib/sliday_backup/pipeline.rb +124 -0
  56. data/lib/sliday_backup/splitter.rb +76 -0
  57. data/lib/sliday_backup/storage/base.rb +69 -0
  58. data/lib/sliday_backup/storage/cloud_files.rb +158 -0
  59. data/lib/sliday_backup/storage/cycler.rb +75 -0
  60. data/lib/sliday_backup/storage/dropbox.rb +212 -0
  61. data/lib/sliday_backup/storage/ftp.rb +112 -0
  62. data/lib/sliday_backup/storage/local.rb +64 -0
  63. data/lib/sliday_backup/storage/qiniu.rb +65 -0
  64. data/lib/sliday_backup/storage/rsync.rb +248 -0
  65. data/lib/sliday_backup/storage/s3.rb +156 -0
  66. data/lib/sliday_backup/storage/scp.rb +67 -0
  67. data/lib/sliday_backup/storage/sftp.rb +82 -0
  68. data/lib/sliday_backup/storage/sliday_storage.rb +79 -0
  69. data/lib/sliday_backup/syncer/base.rb +70 -0
  70. data/lib/sliday_backup/syncer/cloud/base.rb +179 -0
  71. data/lib/sliday_backup/syncer/cloud/cloud_files.rb +83 -0
  72. data/lib/sliday_backup/syncer/cloud/local_file.rb +100 -0
  73. data/lib/sliday_backup/syncer/cloud/s3.rb +110 -0
  74. data/lib/sliday_backup/syncer/rsync/base.rb +54 -0
  75. data/lib/sliday_backup/syncer/rsync/local.rb +31 -0
  76. data/lib/sliday_backup/syncer/rsync/pull.rb +51 -0
  77. data/lib/sliday_backup/syncer/rsync/push.rb +205 -0
  78. data/lib/sliday_backup/template.rb +46 -0
  79. data/lib/sliday_backup/utilities.rb +224 -0
  80. data/lib/sliday_backup/version.rb +5 -0
  81. data/templates/cli/archive +28 -0
  82. data/templates/cli/compressor/bzip2 +4 -0
  83. data/templates/cli/compressor/custom +7 -0
  84. data/templates/cli/compressor/gzip +4 -0
  85. data/templates/cli/config +123 -0
  86. data/templates/cli/databases/mongodb +15 -0
  87. data/templates/cli/databases/mysql +18 -0
  88. data/templates/cli/databases/openldap +24 -0
  89. data/templates/cli/databases/postgresql +16 -0
  90. data/templates/cli/databases/redis +16 -0
  91. data/templates/cli/databases/riak +17 -0
  92. data/templates/cli/databases/sqlite +11 -0
  93. data/templates/cli/encryptor/gpg +27 -0
  94. data/templates/cli/encryptor/openssl +9 -0
  95. data/templates/cli/model +26 -0
  96. data/templates/cli/notifier/zabbix +15 -0
  97. data/templates/cli/notifiers/campfire +12 -0
  98. data/templates/cli/notifiers/command +32 -0
  99. data/templates/cli/notifiers/datadog +57 -0
  100. data/templates/cli/notifiers/flowdock +16 -0
  101. data/templates/cli/notifiers/hipchat +16 -0
  102. data/templates/cli/notifiers/http_post +32 -0
  103. data/templates/cli/notifiers/mail +24 -0
  104. data/templates/cli/notifiers/nagios +13 -0
  105. data/templates/cli/notifiers/pagerduty +12 -0
  106. data/templates/cli/notifiers/prowl +11 -0
  107. data/templates/cli/notifiers/pushover +11 -0
  108. data/templates/cli/notifiers/ses +15 -0
  109. data/templates/cli/notifiers/slack +22 -0
  110. data/templates/cli/notifiers/twitter +13 -0
  111. data/templates/cli/splitter +7 -0
  112. data/templates/cli/storages/cloud_files +11 -0
  113. data/templates/cli/storages/dropbox +20 -0
  114. data/templates/cli/storages/ftp +13 -0
  115. data/templates/cli/storages/local +8 -0
  116. data/templates/cli/storages/qiniu +12 -0
  117. data/templates/cli/storages/rsync +17 -0
  118. data/templates/cli/storages/s3 +16 -0
  119. data/templates/cli/storages/scp +15 -0
  120. data/templates/cli/storages/sftp +15 -0
  121. data/templates/cli/storages/sliday_storage +6 -0
  122. data/templates/cli/syncers/cloud_files +22 -0
  123. data/templates/cli/syncers/rsync_local +20 -0
  124. data/templates/cli/syncers/rsync_pull +28 -0
  125. data/templates/cli/syncers/rsync_push +28 -0
  126. data/templates/cli/syncers/s3 +27 -0
  127. data/templates/general/links +3 -0
  128. data/templates/general/version.erb +2 -0
  129. data/templates/notifier/mail/failure.erb +16 -0
  130. data/templates/notifier/mail/success.erb +16 -0
  131. data/templates/notifier/mail/warning.erb +16 -0
  132. data/templates/storage/dropbox/authorization_url.erb +6 -0
  133. data/templates/storage/dropbox/authorized.erb +4 -0
  134. data/templates/storage/dropbox/cache_file_written.erb +10 -0
  135. metadata +1079 -0
@@ -0,0 +1,187 @@
1
+ # encoding: utf-8
2
+
3
+ module SlidayBackup
4
+ module Database
5
+ class MongoDB < Base
6
+ class Error < SlidayBackup::Error; end
7
+
8
+ ##
9
+ # Name of the database that needs to get dumped
10
+ attr_accessor :name
11
+
12
+ ##
13
+ # Credentials for the specified database
14
+ attr_accessor :username, :password, :authdb
15
+
16
+ ##
17
+ # Connectivity options
18
+ attr_accessor :host, :port
19
+
20
+ ##
21
+ # IPv6 support (disabled by default)
22
+ attr_accessor :ipv6
23
+
24
+ ##
25
+ # Collections to dump, collections that aren't specified won't get dumped
26
+ attr_accessor :only_collections
27
+
28
+ ##
29
+ # Additional "mongodump" options
30
+ attr_accessor :additional_options
31
+
32
+ ##
33
+ # Forces mongod to flush all pending write operations to the disk and
34
+ # locks the entire mongod instance to prevent additional writes until the
35
+ # dump is complete.
36
+ #
37
+ # Note that if Profiling is enabled, this will disable it and will not
38
+ # re-enable it after the dump is complete.
39
+ attr_accessor :lock
40
+
41
+ ##
42
+ # Creates a dump of the database that includes an oplog, to create a
43
+ # point-in-time snapshot of the state of a mongod instance.
44
+ #
45
+ # If this option is used, you would not use the `lock` option.
46
+ #
47
+ # This will only work against nodes that maintain a oplog.
48
+ # This includes all members of a replica set, as well as master nodes in
49
+ # master/slave replication deployments.
50
+ attr_accessor :oplog
51
+
52
+ def initialize(model, database_id = nil, &block)
53
+ super
54
+ instance_eval(&block) if block_given?
55
+ end
56
+
57
+ def perform!
58
+ super
59
+
60
+ lock_database if @lock
61
+ dump!
62
+ package!
63
+
64
+ ensure
65
+ unlock_database if @lock
66
+ end
67
+
68
+ private
69
+
70
+ ##
71
+ # Performs all required mongodump commands, dumping the output files
72
+ # into the +dump_packaging_path+ directory for packaging.
73
+ def dump!
74
+ FileUtils.mkdir_p dump_packaging_path
75
+
76
+ collections = Array(only_collections)
77
+ if collections.empty?
78
+ run(mongodump)
79
+ else
80
+ collections.each do |collection|
81
+ run("#{ mongodump } --collection='#{ collection }'")
82
+ end
83
+ end
84
+ end
85
+
86
+ ##
87
+ # Creates a tar archive of the +dump_packaging_path+ directory
88
+ # and stores it in the +dump_path+ using +dump_filename+.
89
+ #
90
+ # <trigger>/databases/MongoDB[-<database_id>].tar[.gz]
91
+ #
92
+ # If successful, +dump_packaging_path+ is removed.
93
+ def package!
94
+ pipeline = Pipeline.new
95
+ dump_ext = 'tar'
96
+
97
+ pipeline << "#{ utility(:tar) } -cf - " +
98
+ "-C '#{ dump_path }' '#{ dump_filename }'"
99
+
100
+ model.compressor.compress_with do |command, ext|
101
+ pipeline << command
102
+ dump_ext << ext
103
+ end if model.compressor
104
+
105
+ pipeline << "#{ utility(:cat) } > " +
106
+ "'#{ File.join(dump_path, dump_filename) }.#{ dump_ext }'"
107
+
108
+ pipeline.run
109
+ if pipeline.success?
110
+ FileUtils.rm_rf dump_packaging_path
111
+ log!(:finished)
112
+ else
113
+ raise Error, "Dump Failed!\n" + pipeline.error_messages
114
+ end
115
+ end
116
+
117
+ def dump_packaging_path
118
+ File.join(dump_path, dump_filename)
119
+ end
120
+
121
+ def mongodump
122
+ "#{ utility(:mongodump) } #{ name_option } #{ credential_options } " +
123
+ "#{ connectivity_options } #{ ipv6_option } #{ oplog_option } " +
124
+ "#{ user_options } --out='#{ dump_packaging_path }'"
125
+ end
126
+
127
+ def name_option
128
+ "--db='#{ name }'" if name
129
+ end
130
+
131
+ def credential_options
132
+ opts = []
133
+ opts << "--username='#{ username }'" if username
134
+ opts << "--password='#{ password }'" if password
135
+ opts << "--authenticationDatabase='#{ authdb }'" if authdb
136
+ opts.join(' ')
137
+ end
138
+
139
+ def connectivity_options
140
+ opts = []
141
+ opts << "--host='#{ host }'" if host
142
+ opts << "--port='#{ port }'" if port
143
+ opts.join(' ')
144
+ end
145
+
146
+ def ipv6_option
147
+ '--ipv6' if ipv6
148
+ end
149
+
150
+ def oplog_option
151
+ '--oplog' if oplog
152
+ end
153
+
154
+ def user_options
155
+ Array(additional_options).join(' ')
156
+ end
157
+
158
+ def lock_database
159
+ lock_command = <<-EOS.gsub(/^ +/, '')
160
+ echo 'use admin
161
+ db.setProfilingLevel(0)
162
+ db.fsyncLock()' | #{ mongo_shell }
163
+ EOS
164
+
165
+ run(lock_command)
166
+ end
167
+
168
+ def unlock_database
169
+ unlock_command = <<-EOS.gsub(/^ +/, '')
170
+ echo 'use admin
171
+ db.fsyncUnlock()' | #{ mongo_shell }
172
+ EOS
173
+
174
+ run(unlock_command)
175
+ end
176
+
177
+ def mongo_shell
178
+ cmd = "#{ utility(:mongo) } #{ connectivity_options }".rstrip
179
+ cmd << " #{ credential_options }".rstrip
180
+ cmd << " #{ ipv6_option }".rstrip
181
+ cmd << " '#{ name }'" if name
182
+ cmd
183
+ end
184
+
185
+ end
186
+ end
187
+ end
@@ -0,0 +1,192 @@
1
+ # encoding: utf-8
2
+
3
+ module SlidayBackup
4
+ module Database
5
+ class MySQL < Base
6
+ class Error < SlidayBackup::Error; end
7
+
8
+ ##
9
+ # Name of the database that needs to get dumped
10
+ # To dump all databases, set this to `:all` or leave blank.
11
+ attr_accessor :name
12
+
13
+ ##
14
+ # Credentials for the specified database
15
+ attr_accessor :username, :password
16
+
17
+ ##
18
+ # Connectivity options
19
+ attr_accessor :host, :port, :socket
20
+
21
+ ##
22
+ # Tables to skip while dumping the database
23
+ #
24
+ # If `name` is set to :all (or not specified), these must include
25
+ # a database name. e.g. 'name.table'.
26
+ # If `name` is given, these may simply be table names.
27
+ attr_accessor :skip_tables
28
+
29
+ ##
30
+ # Tables to dump. This in only valid if `name` is specified.
31
+ # If none are given, the entire database will be dumped.
32
+ attr_accessor :only_tables
33
+
34
+ ##
35
+ # Additional "mysqldump" or "innobackupex (backup creation)" options
36
+ attr_accessor :additional_options
37
+
38
+ ##
39
+ # Additional innobackupex log preparation phase ("apply-logs") options
40
+ attr_accessor :prepare_options
41
+
42
+ ##
43
+ # Default is :mysqldump (which is built in MySQL and generates
44
+ # a textual SQL file), but can be changed to :innobackupex, which
45
+ # has more feasible restore times for large databases.
46
+ # See: http://www.percona.com/doc/percona-xtrabackup/
47
+ attr_accessor :backup_engine
48
+
49
+ ##
50
+ # If true (which is the default behaviour), the backup will be prepared
51
+ # after it has been successfuly created. This option is only valid if
52
+ # :backup_engine is set to :innobackupex.
53
+ attr_accessor :prepare_backup
54
+
55
+ ##
56
+ # If set the backup engine command block is executed as the given user
57
+ attr_accessor :sudo_user
58
+
59
+ ##
60
+ # If set, do not suppress innobackupdb output (useful for debugging)
61
+ attr_accessor :verbose
62
+
63
+ def initialize(model, database_id = nil, &block)
64
+ super
65
+ instance_eval(&block) if block_given?
66
+
67
+ @name ||= :all
68
+ @backup_engine ||= :mysqldump
69
+ @prepare_backup = true if @prepare_backup.nil?
70
+ end
71
+
72
+ ##
73
+ # Performs the mysqldump or innobackupex command and outputs
74
+ # the dump file in the +dump_path+ using +dump_filename+.
75
+ #
76
+ # <trigger>/databases/MySQL[-<database_id>].[sql|tar][.gz]
77
+ def perform!
78
+ super
79
+
80
+ pipeline = Pipeline.new
81
+ dump_ext = sql_backup? ? 'sql' : 'tar'
82
+
83
+ pipeline << sudo_option(sql_backup? ? mysqldump : innobackupex)
84
+
85
+ model.compressor.compress_with do |command, ext|
86
+ pipeline << command
87
+ dump_ext << ext
88
+ end if model.compressor
89
+
90
+ pipeline << "#{ utility(:cat) } > " +
91
+ "'#{ File.join(dump_path, dump_filename) }.#{ dump_ext }'"
92
+
93
+ pipeline.run
94
+ if pipeline.success?
95
+ log!(:finished)
96
+ else
97
+ raise Error, "Dump Failed!\n" + pipeline.error_messages
98
+ end
99
+ end
100
+
101
+ private
102
+
103
+ def mysqldump
104
+ "#{ utility(:mysqldump) } #{ user_options } #{ credential_options } " +
105
+ "#{ connectivity_options } #{ name_option } " +
106
+ "#{ tables_to_dump } #{ tables_to_skip }"
107
+ end
108
+
109
+ def credential_options
110
+ opts = []
111
+ opts << "--user=#{ Shellwords.escape(username) }" if username
112
+ opts << "--password=#{ Shellwords.escape(password) }" if password
113
+ opts.join(' ')
114
+ end
115
+
116
+ def connectivity_options
117
+ return "--socket='#{ socket }'" if socket
118
+
119
+ opts = []
120
+ opts << "--host='#{ host }'" if host
121
+ opts << "--port='#{ port }'" if port
122
+ opts.join(' ')
123
+ end
124
+
125
+ def user_options
126
+ Array(additional_options).join(' ')
127
+ end
128
+
129
+ def user_prepare_options
130
+ Array(prepare_options).join(' ')
131
+ end
132
+
133
+ def name_option
134
+ dump_all? ? '--all-databases' : name
135
+ end
136
+
137
+ def tables_to_dump
138
+ Array(only_tables).join(' ') unless dump_all?
139
+ end
140
+
141
+ def tables_to_skip
142
+ Array(skip_tables).map do |table|
143
+ table = (dump_all? || table['.']) ? table : "#{ name }.#{ table }"
144
+ "--ignore-table='#{ table }'"
145
+ end.join(' ')
146
+ end
147
+
148
+ def dump_all?
149
+ name == :all
150
+ end
151
+
152
+ def sql_backup?
153
+ backup_engine.to_sym == :mysqldump
154
+ end
155
+
156
+ def innobackupex
157
+ # Creation phase
158
+ "#{ utility(:innobackupex) } #{ credential_options } " +
159
+ "#{ connectivity_options } #{ user_options } " +
160
+ "--no-timestamp #{ temp_dir } #{ quiet_option } && " +
161
+ innobackupex_prepare +
162
+ # Move files to tar-ed stream on stdout
163
+ "#{ utility(:tar) } --remove-files -cf - " +
164
+ "-C #{ File.dirname(temp_dir) } #{ File.basename(temp_dir) }"
165
+ end
166
+
167
+ def innobackupex_prepare
168
+ return "" unless @prepare_backup
169
+ # Log applying phase (prepare for restore)
170
+ "#{ utility(:innobackupex) } --apply-log #{ temp_dir } " +
171
+ "#{ user_prepare_options } #{ quiet_option } && "
172
+ end
173
+
174
+ def sudo_option(command_block)
175
+ return command_block unless sudo_user
176
+
177
+ "sudo -s -u #{ sudo_user } -- <<END_OF_SUDO\n" +
178
+ "#{command_block}\n" +
179
+ "END_OF_SUDO\n"
180
+ end
181
+
182
+ def quiet_option
183
+ verbose ? "" : " 2> /dev/null "
184
+ end
185
+
186
+ def temp_dir
187
+ File.join(dump_path, dump_filename + ".bkpdir")
188
+ end
189
+
190
+ end
191
+ end
192
+ end
@@ -0,0 +1,95 @@
1
+ # encoding: utf-8
2
+
3
+ module SlidayBackup
4
+ module Database
5
+ class OpenLDAP < Base
6
+ class Error < SlidayBackup::Error; end
7
+
8
+ ##
9
+ # Name of the ldap backup
10
+ attr_accessor :name
11
+
12
+ ##
13
+ # run slapcat under sudo if needed
14
+ # make sure to set SUID on a file, to let you run the file with permissions of file owner
15
+ # eg. sudo chmod u+s /usr/sbin/slapcat
16
+ attr_accessor :use_sudo
17
+
18
+ ##
19
+ # Stores the location of the slapd.conf or slapcat confdir
20
+ attr_accessor :slapcat_conf
21
+
22
+ ##
23
+ # Additional slapcat options
24
+ attr_accessor :slapcat_args
25
+
26
+ ##
27
+ # Path to slapcat utility (optional)
28
+ attr_accessor :slapcat_utility
29
+
30
+ ##
31
+ # Takes the name of the archive and the configuration block
32
+ def initialize(model, database_id = nil, &block)
33
+ super
34
+ instance_eval(&block) if block_given?
35
+
36
+ @name ||= 'ldap_backup'
37
+ @use_sudo ||= false
38
+ @slapcat_args ||= Array.new
39
+ @slapcat_utility ||= utility(:slapcat)
40
+ @slapcat_conf ||= '/etc/ldap/slapd.d'
41
+ end
42
+
43
+ ##
44
+ # Performs the slapcat command and outputs the
45
+ # data to the specified path based on the 'trigger'
46
+ def perform!
47
+ super
48
+
49
+ pipeline = Pipeline.new
50
+ dump_ext = 'ldif'
51
+
52
+ pipeline << slapcat
53
+ if @model.compressor
54
+ @model.compressor.compress_with do |command, ext|
55
+ pipeline << command
56
+ dump_ext << ext
57
+ end
58
+ end
59
+
60
+ pipeline << "#{ utility(:cat) } > " +
61
+ "'#{ File.join(dump_path, dump_filename) }.#{ dump_ext }'"
62
+
63
+ pipeline.run
64
+ if pipeline.success?
65
+ log!(:finished)
66
+ else
67
+ raise Error, "Dump Failed!\n" + pipeline.error_messages
68
+ end
69
+ end
70
+
71
+ private
72
+
73
+ ##
74
+ # Builds the full slapcat string based on all attributes
75
+ def slapcat
76
+ command = "#{ slapcat_utility } #{ slapcat_conf_option } #{ slapcat_conf } #{ user_options }"
77
+ command.prepend("sudo ") if use_sudo
78
+ command
79
+ end
80
+
81
+ ##
82
+ # Uses different slapcat switch depending on confdir or conffile set
83
+ def slapcat_conf_option
84
+ @slapcat_conf.include?(".d") ? "-F" : "-f"
85
+ end
86
+
87
+ ##
88
+ # Builds a compatible string for the additional options
89
+ # specified by the user
90
+ def user_options
91
+ slapcat_args.join(' ')
92
+ end
93
+ end
94
+ end
95
+ end