ll-innobackup 0.0.1
Sign up to get free protection for your applications and to get access to all the features.
- data/bin/ll-innobackup +6 -0
- data/lib/ll-innobackup.rb +307 -0
- metadata +64 -0
data/bin/ll-innobackup
ADDED
@@ -0,0 +1,307 @@
|
|
1
|
+
require 'json'
|
2
|
+
require 'date'
|
3
|
+
|
4
|
+
# Run Inmental or Full backups on MySQL
|
5
|
+
module LL
|
6
|
+
class InnoBackup
|
7
|
+
class << self
|
8
|
+
# Use this in case the log file is massive
|
9
|
+
def options
|
10
|
+
JSON.parse(File.read('/etc/mysql/innobackupex.json'))
|
11
|
+
rescue Errno::ENOENT
|
12
|
+
{}
|
13
|
+
end
|
14
|
+
|
15
|
+
def tail_file(path, n)
|
16
|
+
file = File.open(path, 'r')
|
17
|
+
buffer_s = 512
|
18
|
+
line_count = 0
|
19
|
+
file.seek(0, IO::SEEK_END)
|
20
|
+
|
21
|
+
offset = file.pos # we start at the end
|
22
|
+
|
23
|
+
while line_count <= n && offset > 0
|
24
|
+
to_read = if (offset - buffer_s) < 0
|
25
|
+
offset
|
26
|
+
else
|
27
|
+
buffer_s
|
28
|
+
end
|
29
|
+
|
30
|
+
file.seek(offset - to_read)
|
31
|
+
data = file.read(to_read)
|
32
|
+
|
33
|
+
data.reverse.each_char do |c|
|
34
|
+
if line_count > n
|
35
|
+
offset += 1
|
36
|
+
break
|
37
|
+
end
|
38
|
+
offset -= 1
|
39
|
+
line_count += 1 if c == "\n|"
|
40
|
+
end
|
41
|
+
end
|
42
|
+
|
43
|
+
file.seek(offset)
|
44
|
+
file.read
|
45
|
+
end
|
46
|
+
|
47
|
+
def state_file(t)
|
48
|
+
"/tmp/backup_#{t}_state"
|
49
|
+
end
|
50
|
+
|
51
|
+
def lock_file(type)
|
52
|
+
"/tmp/backup_#{type}.lock"
|
53
|
+
end
|
54
|
+
|
55
|
+
def innobackup_log(t)
|
56
|
+
"/tmp/backup_#{t}_innobackup_log"
|
57
|
+
end
|
58
|
+
end
|
59
|
+
|
60
|
+
attr_reader :type,
|
61
|
+
:now,
|
62
|
+
:date,
|
63
|
+
:state_files,
|
64
|
+
:lock_files,
|
65
|
+
:options
|
66
|
+
|
67
|
+
def initialize(options = {})
|
68
|
+
@now = Time.now
|
69
|
+
@date = @now.to_date
|
70
|
+
@options = options
|
71
|
+
@lock_files = {}
|
72
|
+
@state_files = {}
|
73
|
+
@type = backup_type
|
74
|
+
end
|
75
|
+
|
76
|
+
def aws_log
|
77
|
+
"/tmp/backup_#{type}_aws_log"
|
78
|
+
end
|
79
|
+
|
80
|
+
def innobackup_log
|
81
|
+
"/tmp/backup_#{type}_innobackup_log"
|
82
|
+
end
|
83
|
+
|
84
|
+
def lock?(t = type)
|
85
|
+
lock_files[t] ||= File.new(InnoBackup.lock_file(t), File::CREAT)
|
86
|
+
lock_files[t].flock(File::LOCK_NB | File::LOCK_EX).zero?
|
87
|
+
end
|
88
|
+
|
89
|
+
def state(t)
|
90
|
+
state_files[t] ||= JSON.parse(File.read(InnoBackup.state_file(t)))
|
91
|
+
rescue JSON::ParserError
|
92
|
+
puts 'unable to stat state file'
|
93
|
+
{}
|
94
|
+
end
|
95
|
+
|
96
|
+
def fully_backed_up_today?
|
97
|
+
require 'active_support/all'
|
98
|
+
date = state('full')['date']
|
99
|
+
Time.parse(date).today?
|
100
|
+
rescue Errno::ENOENT
|
101
|
+
puts 'unable to obtain last full backup state'
|
102
|
+
false
|
103
|
+
rescue NoMethodError
|
104
|
+
puts 'unable to obtain last backup state'
|
105
|
+
false
|
106
|
+
end
|
107
|
+
|
108
|
+
def can_full_backup?
|
109
|
+
!fully_backed_up_today? && lock?('full')
|
110
|
+
end
|
111
|
+
|
112
|
+
def full_backup_running?
|
113
|
+
!lock?('full')
|
114
|
+
end
|
115
|
+
|
116
|
+
def incremental_backup_running?
|
117
|
+
!lock?('incremental')
|
118
|
+
end
|
119
|
+
|
120
|
+
def backup_type
|
121
|
+
return 'full' unless fully_backed_up_today? || full_backup_running?
|
122
|
+
return 'incremental' unless incremental_backup_running?
|
123
|
+
raise 'Unable to backup as backups are running'
|
124
|
+
end
|
125
|
+
|
126
|
+
def backup_bin
|
127
|
+
@backup_bin = options['backup_bin'] ||= '/usr/bin/innobackupex'
|
128
|
+
end
|
129
|
+
|
130
|
+
def backup_parallel
|
131
|
+
@backup_parallel = options['backup_parallel'] ||= 4
|
132
|
+
end
|
133
|
+
|
134
|
+
def backup_compress_threads
|
135
|
+
@backup_compress_threads = options['backup_compress_threads'] ||= 4
|
136
|
+
end
|
137
|
+
|
138
|
+
def sql_backup_user
|
139
|
+
@sql_backup_user ||= options['sql_backup_user']
|
140
|
+
end
|
141
|
+
|
142
|
+
def sql_backup_password
|
143
|
+
@sql_backup_password ||= options['sql_backup_password']
|
144
|
+
end
|
145
|
+
|
146
|
+
def aws_bin
|
147
|
+
@aws_bin = options['aws_bin'] ||= '/usr/local/bin/aws'
|
148
|
+
end
|
149
|
+
|
150
|
+
def aws_bucket
|
151
|
+
@aws_bucket = options['aws_bucket'] ||= 'jim-test-mysql-stream1'
|
152
|
+
end
|
153
|
+
|
154
|
+
def expected_full_size
|
155
|
+
@expected_full_size = options['expected_full_size'] ||= 1_600_000_000
|
156
|
+
end
|
157
|
+
|
158
|
+
def sql_authentication
|
159
|
+
"--user=#{sql_backup_user} --password=#{sql_backup_password}"
|
160
|
+
end
|
161
|
+
|
162
|
+
def innobackup_options
|
163
|
+
"--parallel=#{backup_parallel} "\
|
164
|
+
"--compress-threads=#{backup_compress_threads} "\
|
165
|
+
'--stream=xbstream --compress'
|
166
|
+
end
|
167
|
+
|
168
|
+
def innobackup_command
|
169
|
+
"#{backup_bin} #{sql_authentication} "\
|
170
|
+
"#{incremental} #{innobackup_options} /tmp/sql"
|
171
|
+
end
|
172
|
+
|
173
|
+
def expires_date
|
174
|
+
require 'active_support/all'
|
175
|
+
# Keep incrementals for 2 days
|
176
|
+
return (@now + 2.days).iso8601 if type == 'incremental'
|
177
|
+
# Keep first backup of month for 180 days
|
178
|
+
return (@now + 6.months).iso8601 if @date.yesterday.month != @date.month
|
179
|
+
# Keep first backup of week for 31 days (monday)
|
180
|
+
return (@now + 1.month).iso8601 if @date.cwday == 1
|
181
|
+
# Keep daily backups for 14 days
|
182
|
+
(@now + 2.weeks).iso8601
|
183
|
+
end
|
184
|
+
|
185
|
+
def expires
|
186
|
+
ed = expires_date
|
187
|
+
"--expires=#{ed}" if ed
|
188
|
+
end
|
189
|
+
|
190
|
+
def expected_size
|
191
|
+
"--expected-size=#{expected_full_size}" if type == 'full'
|
192
|
+
end
|
193
|
+
|
194
|
+
def aws_command
|
195
|
+
"#{aws_bin} s3 cp - s3://#{aws_bucket}/#{aws_backup_file}"\
|
196
|
+
" #{expected_size} #{expires}"
|
197
|
+
end
|
198
|
+
|
199
|
+
def valid_commands?
|
200
|
+
File.exist?(backup_bin) && File.exist?(aws_bin)
|
201
|
+
end
|
202
|
+
|
203
|
+
def backup
|
204
|
+
require 'English'
|
205
|
+
exc = "#{innobackup_command} 2> #{innobackup_log} |
|
206
|
+
#{aws_command} 2> #{aws_log} >> #{aws_log}"
|
207
|
+
`#{exc}`if valid_commands?
|
208
|
+
@completed = $CHILD_STATUS == 0
|
209
|
+
return record if success? && completed?
|
210
|
+
revert_aws if valid_commands?
|
211
|
+
rescue InnoBackup::NoStateError => e
|
212
|
+
STDERR.puts e.message
|
213
|
+
ensure
|
214
|
+
report
|
215
|
+
end
|
216
|
+
|
217
|
+
def revert_aws
|
218
|
+
exc = "#{aws_bin} s3 rm s3://#{aws_bucket}/#{aws_backup_file} > /dev/null 2>/dev/null"
|
219
|
+
`#{exc}`
|
220
|
+
end
|
221
|
+
|
222
|
+
def success?
|
223
|
+
InnoBackup.tail_file(
|
224
|
+
innobackup_log,
|
225
|
+
1
|
226
|
+
) =~ /: completed OK/
|
227
|
+
rescue Errno::ENOENT
|
228
|
+
false
|
229
|
+
end
|
230
|
+
|
231
|
+
def record
|
232
|
+
File.write(
|
233
|
+
InnoBackup.state_file(type),
|
234
|
+
{
|
235
|
+
date: now,
|
236
|
+
lsn: lsn_from_backup_log,
|
237
|
+
file: aws_backup_file
|
238
|
+
}.to_json
|
239
|
+
)
|
240
|
+
end
|
241
|
+
|
242
|
+
def incremental
|
243
|
+
return unless backup_type == 'incremental'
|
244
|
+
"--incremental --incremental-lsn=#{lsn_from_state}"
|
245
|
+
end
|
246
|
+
|
247
|
+
def lsn_from_full_backup_state?
|
248
|
+
Time.parse(state('full')['date']) > Time.parse(state('incremental')['date'])
|
249
|
+
rescue Errno::ENOENT
|
250
|
+
true
|
251
|
+
end
|
252
|
+
|
253
|
+
def lsn_from_state
|
254
|
+
return state('full')['lsn'] if lsn_from_full_backup_state?
|
255
|
+
state('incremental')['lsn']
|
256
|
+
rescue NoMethodError
|
257
|
+
raise NoStateError, 'no state file for incremental backup'
|
258
|
+
end
|
259
|
+
|
260
|
+
def lsn_from_backup_log
|
261
|
+
matches = InnoBackup.tail_file(
|
262
|
+
InnoBackup.innobackup_log(type),
|
263
|
+
30
|
264
|
+
).match(/The latest check point \(for incremental\): '(\d+)'/)
|
265
|
+
matches[1] if matches
|
266
|
+
end
|
267
|
+
|
268
|
+
def hostname
|
269
|
+
return options[:hostname] if options[:hostname]
|
270
|
+
require 'socket'
|
271
|
+
Socket.gethostbyname(Socket.gethostname).first
|
272
|
+
end
|
273
|
+
|
274
|
+
def aws_backup_file
|
275
|
+
return "#{hostname}/#{now.to_time.utc.to_i}/percona_full_backup" if type == 'full'
|
276
|
+
"#{hostname}/#{Time.parse(state('full')['date']).to_i}/percona_incremental_#{now.to_time.utc.to_i}"
|
277
|
+
rescue NoMethodError
|
278
|
+
raise NoStateError, 'incremental state missing or corrupt'
|
279
|
+
end
|
280
|
+
|
281
|
+
def completed?
|
282
|
+
@completed == true
|
283
|
+
end
|
284
|
+
|
285
|
+
def report
|
286
|
+
# Eventually Tell Zabbix
|
287
|
+
if success? && completed?
|
288
|
+
STDERR.puts "#{$PROGRAM_NAME}: success: completed #{type} backup"
|
289
|
+
return
|
290
|
+
end
|
291
|
+
STDERR.puts "#{$PROGRAM_NAME}: failed"
|
292
|
+
STDERR.puts 'missing binaries' unless valid_commands?
|
293
|
+
inno_tail = InnoBackup.tail_file(innobackup_log, 10)
|
294
|
+
STDERR.puts 'invalid sql user' if inno_tail =~ /Option user requires an argument/
|
295
|
+
STDERR.puts 'unable to connect to DB' if inno_tail =~ /Access denied for user/
|
296
|
+
STDERR.puts 'insufficient file access' if inno_tail =~ /Can't change dir to/
|
297
|
+
aws_tail = InnoBackup.tail_file(aws_log, 10)
|
298
|
+
STDERR.puts 'bucket incorrect' if aws_tail =~ /The specified bucket does not exist/
|
299
|
+
STDERR.puts 'invalid AWS key' if aws_tail =~ /The AWS Access Key Id you/
|
300
|
+
STDERR.puts 'invalid Secret key' if aws_tail =~ /The request signature we calculated/
|
301
|
+
end
|
302
|
+
|
303
|
+
class NoStateError < StandardError
|
304
|
+
end
|
305
|
+
end
|
306
|
+
end
|
307
|
+
InnoBackup.new(InnoBackup.options).backup if $PROGRAM_NAME == __FILE__
|
metadata
ADDED
@@ -0,0 +1,64 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: ll-innobackup
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.0.1
|
5
|
+
prerelease:
|
6
|
+
platform: ruby
|
7
|
+
authors:
|
8
|
+
- Stuart Harland, LiveLink Technology Ltd
|
9
|
+
autorequire:
|
10
|
+
bindir: bin
|
11
|
+
cert_chain: []
|
12
|
+
date: 2018-03-20 00:00:00.000000000 Z
|
13
|
+
dependencies:
|
14
|
+
- !ruby/object:Gem::Dependency
|
15
|
+
name: activesupport
|
16
|
+
requirement: !ruby/object:Gem::Requirement
|
17
|
+
none: false
|
18
|
+
requirements:
|
19
|
+
- - '='
|
20
|
+
- !ruby/object:Gem::Version
|
21
|
+
version: 4.2.6
|
22
|
+
type: :runtime
|
23
|
+
prerelease: false
|
24
|
+
version_requirements: !ruby/object:Gem::Requirement
|
25
|
+
none: false
|
26
|
+
requirements:
|
27
|
+
- - '='
|
28
|
+
- !ruby/object:Gem::Version
|
29
|
+
version: 4.2.6
|
30
|
+
description: A program to conduct innobackup
|
31
|
+
email: essjayhch@gmail.com, infra@livelinktechnology.net
|
32
|
+
executables:
|
33
|
+
- ll-innobackup
|
34
|
+
extensions: []
|
35
|
+
extra_rdoc_files: []
|
36
|
+
files:
|
37
|
+
- lib/ll-innobackup.rb
|
38
|
+
- bin/ll-innobackup
|
39
|
+
homepage: http://rubygems.org/gems/hola
|
40
|
+
licenses:
|
41
|
+
- MIT
|
42
|
+
post_install_message:
|
43
|
+
rdoc_options: []
|
44
|
+
require_paths:
|
45
|
+
- lib
|
46
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
47
|
+
none: false
|
48
|
+
requirements:
|
49
|
+
- - ! '>='
|
50
|
+
- !ruby/object:Gem::Version
|
51
|
+
version: '0'
|
52
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
53
|
+
none: false
|
54
|
+
requirements:
|
55
|
+
- - ! '>='
|
56
|
+
- !ruby/object:Gem::Version
|
57
|
+
version: '0'
|
58
|
+
requirements: []
|
59
|
+
rubyforge_project:
|
60
|
+
rubygems_version: 1.8.23
|
61
|
+
signing_key:
|
62
|
+
specification_version: 3
|
63
|
+
summary: Livelink Innobackup Script
|
64
|
+
test_files: []
|