bibliotech 0.7.1 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/default_configuration/config.yaml +14 -8
- data/lib/bibliotech/backups/calendar_scheduler.rb +91 -0
- data/lib/bibliotech/backups/prune_list.rb +7 -1
- data/lib/bibliotech/backups/scheduler.rb +15 -2
- data/lib/bibliotech/cli.rb +5 -1
- data/lib/bibliotech/command_generator.rb +18 -2
- data/lib/bibliotech/config/schedule.rb +130 -0
- data/lib/bibliotech/config.rb +6 -46
- data/spec/bibliotech/backup_pruner_spec.rb +1 -2
- data/spec/bibliotech/backup_scheduler_spec.rb +6 -6
- data/spec/bibliotech/command_generator/mysql_spec.rb +37 -22
- data/spec/bibliotech/command_generator/postgres_spec.rb +37 -21
- data/spec/bibliotech/config_spec.rb +117 -20
- metadata +31 -29
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: edbcdd1d8fee2ac6fbfbf3c9dcb2d95e6e8ab8dd
|
4
|
+
data.tar.gz: fe1f5a3328dc2b83d7ba317167d4d5dcc56fae71
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 166cda03abf645d429194b382988be7bfac95e03b96182de7711a3e96891d28b372122971634dd356dc31423baf65886b4b129f7b66c86960352e21d88e83b28
|
7
|
+
data.tar.gz: 37e1d3bb553b164f15e9a8143b58f2e3884a8f9f937a40c7a7c2e35088b70550cf8d64c5482ab1202d204a0aaf27a532ba1a86a74c70bb9bb98ac56749f2769d
|
@@ -7,21 +7,25 @@ backups:
|
|
7
7
|
dir: db_backups
|
8
8
|
compress: gzip # [ none, gzip, bzip2, 7zip ]
|
9
9
|
prefix: backup
|
10
|
-
|
11
|
-
|
10
|
+
retain:
|
11
|
+
periodic:
|
12
|
+
hourlies: 1
|
12
13
|
frequency: hourly
|
13
14
|
|
14
15
|
log:
|
15
|
-
target:
|
16
|
+
target: log/backups.log
|
16
17
|
level: warn
|
17
18
|
|
18
19
|
production:
|
19
20
|
backups:
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
21
|
+
retain:
|
22
|
+
periodic:
|
23
|
+
hourlies: 48
|
24
|
+
dailies: 14
|
25
|
+
weeklies: 8
|
26
|
+
calendar:
|
27
|
+
monthlies: 12
|
28
|
+
quarterly: all
|
25
29
|
|
26
30
|
database_config_env: production
|
27
31
|
|
@@ -38,6 +42,8 @@ staging:
|
|
38
42
|
host: some.server.com
|
39
43
|
|
40
44
|
development:
|
45
|
+
log:
|
46
|
+
target: stderr
|
41
47
|
database_config_env: development
|
42
48
|
path: "."
|
43
49
|
rsa_files:
|
@@ -0,0 +1,91 @@
|
|
1
|
+
require 'bibliotech/backups/scheduler'
|
2
|
+
|
3
|
+
module BiblioTech
|
4
|
+
module Backups
|
5
|
+
class CalendarScheduler < Scheduler
|
6
|
+
# pattern is an array with minutes rightmost
|
7
|
+
def initialize(pattern, limit)
|
8
|
+
@pattern = pattern
|
9
|
+
super(name_for(pattern), freq_for(pattern), limit)
|
10
|
+
end
|
11
|
+
attr_accessor :pattern
|
12
|
+
|
13
|
+
def name_for(pattern)
|
14
|
+
example_time = coerce(Time.new, pattern)
|
15
|
+
case pattern.length
|
16
|
+
when 0
|
17
|
+
"every-minute"
|
18
|
+
when 1
|
19
|
+
example_time.strftime('Hourly at :%M')
|
20
|
+
when 2
|
21
|
+
example_time.strftime('Daily at %H:%M')
|
22
|
+
when 3
|
23
|
+
example_time.strftime('Monthly on day %d, at %H:%M')
|
24
|
+
when 4
|
25
|
+
example_time.strftime('Yearly: %b %d, at %H:%M')
|
26
|
+
else
|
27
|
+
raise ArgumentError, "argument out of range"
|
28
|
+
end
|
29
|
+
|
30
|
+
end
|
31
|
+
|
32
|
+
def freq_for(pattern)
|
33
|
+
case pattern.length
|
34
|
+
when 0
|
35
|
+
1
|
36
|
+
when 1
|
37
|
+
60
|
38
|
+
when 2
|
39
|
+
60 * 24
|
40
|
+
when 3
|
41
|
+
60 * 24 * 28
|
42
|
+
when 4
|
43
|
+
60 * 24 * 365
|
44
|
+
else
|
45
|
+
raise ArgumentError, "argument out of range"
|
46
|
+
end
|
47
|
+
end
|
48
|
+
|
49
|
+
def adjustment_index
|
50
|
+
5 - pattern.length
|
51
|
+
end
|
52
|
+
|
53
|
+
# [sec, min, hour, day, month, year, wday, yday, isdst, zone]
|
54
|
+
# new(year, month, day, hour, min, sec, utc_offset)
|
55
|
+
def coerce(time, pat=pattern)
|
56
|
+
existing = time.to_a
|
57
|
+
changeable = existing[((pat.length+1)..5)].reverse
|
58
|
+
values = changeable + pat + [0, time.utc_offset]
|
59
|
+
Time.new(*values) + 1
|
60
|
+
end
|
61
|
+
|
62
|
+
def compute_earliest_time(file_list)
|
63
|
+
exact_time = super
|
64
|
+
time = coerce(exact_time)
|
65
|
+
if time < exact_time
|
66
|
+
time
|
67
|
+
else
|
68
|
+
step_back(time)
|
69
|
+
end
|
70
|
+
end
|
71
|
+
|
72
|
+
def latest_time(file_list)
|
73
|
+
exact_time = super
|
74
|
+
time = coerce(exact_time)
|
75
|
+
if time > exact_time
|
76
|
+
time
|
77
|
+
else
|
78
|
+
step_forward(time)
|
79
|
+
end
|
80
|
+
end
|
81
|
+
|
82
|
+
def step_back(time)
|
83
|
+
coerce(super(time))
|
84
|
+
end
|
85
|
+
|
86
|
+
def step_forward(time)
|
87
|
+
coerce(super(time))
|
88
|
+
end
|
89
|
+
end
|
90
|
+
end
|
91
|
+
end
|
@@ -11,7 +11,13 @@ module BiblioTech
|
|
11
11
|
|
12
12
|
def list
|
13
13
|
files = []
|
14
|
-
|
14
|
+
begin
|
15
|
+
dir = Dir.new(path)
|
16
|
+
rescue Errno::ENOENT
|
17
|
+
return []
|
18
|
+
end
|
19
|
+
|
20
|
+
dir.each do |file|
|
15
21
|
next if %w{. ..}.include?(file)
|
16
22
|
file_record = build_record(file)
|
17
23
|
if file_record.nil?
|
@@ -35,6 +35,14 @@ module BiblioTech
|
|
35
35
|
file_list.first.timestamp
|
36
36
|
end
|
37
37
|
|
38
|
+
def step_back(time)
|
39
|
+
time - freq_seconds
|
40
|
+
end
|
41
|
+
|
42
|
+
def step_forward(time)
|
43
|
+
time + freq_seconds
|
44
|
+
end
|
45
|
+
|
38
46
|
# Working from the latest time backwards, mark the closest file to the
|
39
47
|
# appropriate frequencies as keepable
|
40
48
|
def mark(original_file_list)
|
@@ -42,19 +50,24 @@ module BiblioTech
|
|
42
50
|
|
43
51
|
time = latest_time(file_list)
|
44
52
|
earliest_time = compute_earliest_time(file_list)
|
53
|
+
oldest_file = file_list.last
|
54
|
+
|
45
55
|
while time > earliest_time do
|
46
56
|
file_list.delete_if do |record|
|
47
57
|
record.timestamp > time
|
48
58
|
end
|
49
59
|
|
50
|
-
|
60
|
+
if file_list.empty?
|
61
|
+
oldest_file.in_schedule(name)
|
62
|
+
break
|
63
|
+
end
|
51
64
|
|
52
65
|
closest = file_list.first
|
53
66
|
|
54
67
|
if (time - closest.timestamp) < freq_seconds
|
55
68
|
closest.in_schedule(name)
|
56
69
|
end
|
57
|
-
time
|
70
|
+
time = step_back(time)
|
58
71
|
end
|
59
72
|
return original_file_list
|
60
73
|
end
|
data/lib/bibliotech/cli.rb
CHANGED
@@ -3,6 +3,10 @@ require 'bibliotech/application'
|
|
3
3
|
|
4
4
|
module BiblioTech
|
5
5
|
class CLI < Thor
|
6
|
+
def self.exit_on_failure?
|
7
|
+
true
|
8
|
+
end
|
9
|
+
|
6
10
|
desc "latest", "Outputs the latest DB dump available locally"
|
7
11
|
def latest
|
8
12
|
app = App.new(:log => { :target => "/dev/null" })
|
@@ -18,7 +22,7 @@ module BiblioTech
|
|
18
22
|
desc "load FILENAME", "Load a database file from FILE"
|
19
23
|
def load(file)
|
20
24
|
app = App.new
|
21
|
-
app.import(:backups => { :
|
25
|
+
app.import(:backups => { :file => file })
|
22
26
|
end
|
23
27
|
|
24
28
|
desc "backup", "Create a database backup if needed"
|
@@ -26,8 +26,24 @@ module BiblioTech
|
|
26
26
|
end
|
27
27
|
end
|
28
28
|
|
29
|
-
def import(
|
30
|
-
|
29
|
+
def import(opts = nil)
|
30
|
+
opts ||= {}
|
31
|
+
options = config.merge(opts)
|
32
|
+
unless File.exists?(options.backup_file)
|
33
|
+
filename = options.backup_file
|
34
|
+
opts[:backups] = opts.fetch(:backups){ opts.fetch("backups", {})}
|
35
|
+
opts[:filename] = filename
|
36
|
+
|
37
|
+
options = config.merge(opts)
|
38
|
+
|
39
|
+
if File.exists?(options.backup_file)
|
40
|
+
log.warn { "Actually restoring from #{options.backup_file} - this behavior is deprecated. In future, use explicit path."}
|
41
|
+
else
|
42
|
+
log.fatal{ "Cannot restore from database from missing file #{filename} !"}
|
43
|
+
raise "Missing #{filename}"
|
44
|
+
end
|
45
|
+
end
|
46
|
+
|
31
47
|
command = cmd()
|
32
48
|
command = Builders::Import.for(options).go(command)
|
33
49
|
Builders::FileInput.for(options).go(command).tap do |cmd|
|
@@ -0,0 +1,130 @@
|
|
1
|
+
require 'bibliotech/backups/scheduler'
|
2
|
+
require 'bibliotech/backups/calendar_scheduler'
|
3
|
+
|
4
|
+
module BiblioTech
|
5
|
+
class Config
|
6
|
+
class Schedules
|
7
|
+
def initialize(config)
|
8
|
+
@config = config
|
9
|
+
end
|
10
|
+
attr_reader :config
|
11
|
+
|
12
|
+
SCHEDULE_SHORTHANDS = {
|
13
|
+
"hourly" => 60,
|
14
|
+
"hourlies" => 60,
|
15
|
+
"daily" => 60 * 24,
|
16
|
+
"dailies" => 60 * 24,
|
17
|
+
"weekly" => 60 * 24 * 7,
|
18
|
+
"weeklies" => 60 * 24 * 7,
|
19
|
+
"monthly" => 60 * 24 * 30,
|
20
|
+
"monthlies" => 60 * 24 * 30,
|
21
|
+
"quarterly" => 60 * 24 * 120,
|
22
|
+
"quarterlies" => 60 * 24 * 120,
|
23
|
+
"yearly" => 60 * 24 * 365,
|
24
|
+
"yearlies" => 60 * 24 * 365,
|
25
|
+
}.freeze
|
26
|
+
def regularize_frequency(frequency)
|
27
|
+
Integer( SCHEDULE_SHORTHANDS.fetch(frequency){ frequency } )
|
28
|
+
rescue ArgumentError
|
29
|
+
raise "#{frequency.inspect} is neither a number of minutes or a shorthand. Try:\n #{SCHEDULE_SHORTHANDS.keys.join(" ")}"
|
30
|
+
end
|
31
|
+
|
32
|
+
def backup_frequency
|
33
|
+
@backup_frequency ||= regularize_frequency(config.local_get(:backup_frequency))
|
34
|
+
end
|
35
|
+
|
36
|
+
def regularize_schedule(freq)
|
37
|
+
regularize_frequency(freq)
|
38
|
+
end
|
39
|
+
|
40
|
+
def schedules
|
41
|
+
config_hash.map do |frequency, limit|
|
42
|
+
next if limit == "none"
|
43
|
+
real_frequency = regularize_schedule(frequency)
|
44
|
+
limit =
|
45
|
+
case limit
|
46
|
+
when "all"
|
47
|
+
nil
|
48
|
+
else
|
49
|
+
Integer(limit)
|
50
|
+
end
|
51
|
+
[frequency, real_frequency, limit]
|
52
|
+
end.compact.map do |config_frequency, regularized, limit|
|
53
|
+
build_scheduler(config_frequency, regularized, limit).tap do |sched|
|
54
|
+
unless sched.frequency % backup_frequency == 0
|
55
|
+
raise "Pruning frequency #{sched.frequency}:#{config_frequency} is not a multiple " +
|
56
|
+
"of backup frequency: #{backup_frequency}:#{config.local_get(:backup_frequency)}"
|
57
|
+
end
|
58
|
+
end
|
59
|
+
end.compact.sort_by do |schedule|
|
60
|
+
schedule.frequency
|
61
|
+
end
|
62
|
+
end
|
63
|
+
|
64
|
+
def get_config_hash(name)
|
65
|
+
value = config.local_get(name)
|
66
|
+
if value.to_s == "none"
|
67
|
+
return {}
|
68
|
+
else
|
69
|
+
return value
|
70
|
+
end
|
71
|
+
end
|
72
|
+
end
|
73
|
+
|
74
|
+
class Periods < Schedules
|
75
|
+
def config_hash
|
76
|
+
hash = {}
|
77
|
+
|
78
|
+
config.optionally{ hash.merge! get_config_hash(:prune_schedule) }
|
79
|
+
config.optionally{ hash.merge! get_config_hash(:legacy_prune_schedule) }
|
80
|
+
hash
|
81
|
+
end
|
82
|
+
|
83
|
+
def build_scheduler(frequency, real_frequency, limit)
|
84
|
+
Backups::Scheduler.new(frequency, real_frequency, limit)
|
85
|
+
end
|
86
|
+
end
|
87
|
+
|
88
|
+
class Calendars < Schedules
|
89
|
+
def config_hash
|
90
|
+
hash = {}
|
91
|
+
config.optionally{ hash = get_config_hash(:prune_calendar) }
|
92
|
+
[:quarterly, :quarterlies, "quarterly", "quarterlies"].each do |qkey|
|
93
|
+
limit = hash.delete(qkey)
|
94
|
+
if limit
|
95
|
+
hash.merge!(
|
96
|
+
:first_quarter => limit,
|
97
|
+
:second_quarter => limit,
|
98
|
+
:third_quarter => limit,
|
99
|
+
:fourth_quarter => limit
|
100
|
+
)
|
101
|
+
end
|
102
|
+
end
|
103
|
+
hash
|
104
|
+
end
|
105
|
+
|
106
|
+
def regularize_schedule(freq)
|
107
|
+
case freq
|
108
|
+
when Array
|
109
|
+
freq
|
110
|
+
when "daily", "dailies", :daily, :dailies
|
111
|
+
[ 0, 00 ]
|
112
|
+
when "monthly", "monthlies", :monthly, :monthlies
|
113
|
+
[ 1, 0, 00 ] # on the 1st, at 0:00
|
114
|
+
when "first_quarter", "first-quarter", :first_quarter
|
115
|
+
[ 1, 1, 0, 00 ] # on the 1st of Jan, at 0:00
|
116
|
+
when "second_quarter", "second-quarter", :second_quarter
|
117
|
+
[ 4, 1, 0, 00 ] # on the 1st of Apr, at 0:00
|
118
|
+
when "third_quarter", "third-quarter", :third_quarter
|
119
|
+
[ 7, 1, 0, 00 ] # on the 1st of Jul, at 0:00
|
120
|
+
when "fourth_quarter", "fourth-quarter", :fourth_quarter
|
121
|
+
[ 10, 1, 0, 00 ] # on the 1st of Oct, at 0:00
|
122
|
+
end
|
123
|
+
end
|
124
|
+
|
125
|
+
def build_scheduler(frequency, schedule_pattern, limit)
|
126
|
+
Backups::CalendarScheduler.new(schedule_pattern, limit)
|
127
|
+
end
|
128
|
+
end
|
129
|
+
end
|
130
|
+
end
|
data/lib/bibliotech/config.rb
CHANGED
@@ -1,4 +1,4 @@
|
|
1
|
-
require 'bibliotech/
|
1
|
+
require 'bibliotech/config/schedule'
|
2
2
|
|
3
3
|
module BiblioTech
|
4
4
|
class Config
|
@@ -20,7 +20,9 @@ module BiblioTech
|
|
20
20
|
:filename => [ "backups" , "filename" ] ,
|
21
21
|
:backup_path => [ "backups" , "dir" ] ,
|
22
22
|
:compressor => [ "backups" , "compress" ] ,
|
23
|
-
:
|
23
|
+
:legacy_prune_schedule => [ "backups" , "keep" ] ,
|
24
|
+
:prune_schedule => [ "backups" , "retain" , "periodic" ] ,
|
25
|
+
:prune_calendar => [ "backups" , "retain" , "calendar" ] ,
|
24
26
|
:backup_name => [ "backups" , "prefix" ] ,
|
25
27
|
:backup_frequency => [ "backups" , "frequency" ] ,
|
26
28
|
:db_adapter => [ "database_config" , "adapter" ] ,
|
@@ -204,60 +206,18 @@ module BiblioTech
|
|
204
206
|
File::join(remote_path(remote), filename)
|
205
207
|
end
|
206
208
|
|
207
|
-
SCHEDULE_SHORTHANDS = {
|
208
|
-
"hourly" => 60,
|
209
|
-
"hourlies" => 60,
|
210
|
-
"daily" => 60 * 24,
|
211
|
-
"dailies" => 60 * 24,
|
212
|
-
"weekly" => 60 * 24 * 7,
|
213
|
-
"weeklies" => 60 * 24 * 7,
|
214
|
-
"monthly" => 60 * 24 * 30,
|
215
|
-
"monthlies" => 60 * 24 * 30,
|
216
|
-
"quarterly" => 60 * 24 * 120,
|
217
|
-
"quarterlies" => 60 * 24 * 120,
|
218
|
-
"yearly" => 60 * 24 * 365,
|
219
|
-
"yearlies" => 60 * 24 * 365,
|
220
|
-
}
|
221
|
-
def regularize_frequency(frequency)
|
222
|
-
Integer( SCHEDULE_SHORTHANDS.fetch(frequency){ frequency } )
|
223
|
-
rescue ArgumentError
|
224
|
-
raise "#{frequency.inspect} is neither a number of minutes or a shorthand. Try:\n #{SCHEDULE_SHORTHANDS.keys.join(" ")}"
|
225
|
-
end
|
226
|
-
|
227
209
|
def backup_name
|
228
210
|
local_get(:backup_name)
|
229
211
|
end
|
230
212
|
|
231
|
-
def backup_frequency
|
232
|
-
@backup_frequency ||= regularize_frequency(local_get(:backup_frequency))
|
233
|
-
end
|
234
213
|
|
235
214
|
def prune_schedules
|
236
|
-
|
237
|
-
prune_hash.map do |frequency, limit|
|
238
|
-
next if limit == "none"
|
239
|
-
real_frequency = regularize_frequency(frequency)
|
240
|
-
unless real_frequency % backup_frequency == 0
|
241
|
-
raise "Pruning frequency #{real_frequency}:#{frequency} is not a multiple of backup frequency: #{backup_frequency}:#{local_get(:backup_frequency)}"
|
242
|
-
end
|
243
|
-
limit =
|
244
|
-
case limit
|
245
|
-
when "all"
|
246
|
-
nil
|
247
|
-
else
|
248
|
-
Integer(limit)
|
249
|
-
end
|
250
|
-
[frequency, real_frequency, limit]
|
251
|
-
end.compact.sort_by do |freq_name, frequency, limit|
|
252
|
-
frequency
|
253
|
-
end.tap do |list|
|
215
|
+
list = Periods.new(self).schedules + Calendars.new(self).schedules
|
254
216
|
if list.empty?
|
255
217
|
require 'pp'
|
256
218
|
raise "No backups will be kept by prune schedule: #{prune_hash.pretty_inspect}"
|
257
219
|
end
|
258
|
-
|
259
|
-
Backups::Scheduler.new(freq_name, frequency, limit)
|
260
|
-
end
|
220
|
+
list
|
261
221
|
end
|
262
222
|
|
263
223
|
def database_config
|
@@ -103,7 +103,6 @@ module BiblioTech
|
|
103
103
|
end
|
104
104
|
|
105
105
|
it "should keep 48 hours of backup" do
|
106
|
-
now = Time.now.utc
|
107
106
|
(0..47).each do |interval|
|
108
107
|
sandbox.new :file => "db_backups/#{pruner.filename_for(Time.now.utc - interval * 60 * 60)}"
|
109
108
|
end
|
@@ -122,7 +121,7 @@ module BiblioTech
|
|
122
121
|
sandbox.new :file => "db_backups/#{pruner.filename_for(now - interval * 60 * 60)}"
|
123
122
|
end
|
124
123
|
|
125
|
-
expect(pruner.pruneable.length).to eq(471 - 48 - (14 - 2) -
|
124
|
+
expect(pruner.pruneable.length).to eq(471 - 48 - (14 - 2) - 2) # 2 dailies hourly etc.
|
126
125
|
|
127
126
|
expect(pruner.list.length).to eq(471)
|
128
127
|
end
|
@@ -26,13 +26,13 @@ module BiblioTech::Backups
|
|
26
26
|
Scheduler.new("hourly", 60, nil)
|
27
27
|
end
|
28
28
|
|
29
|
-
context "when there's
|
29
|
+
context "when there's superfrequent backups over 12 hours" do
|
30
30
|
let(:interval){ 60*60*12 - test_jitter}
|
31
31
|
let(:frequency) { 15 }
|
32
32
|
let(:test_jitter){ 60 }
|
33
33
|
|
34
|
-
it "should mark
|
35
|
-
expect(kept_files.count).to eql
|
34
|
+
it "should mark 13 files kept" do
|
35
|
+
expect(kept_files.count).to eql 13
|
36
36
|
end
|
37
37
|
end
|
38
38
|
end
|
@@ -84,13 +84,13 @@ module BiblioTech::Backups
|
|
84
84
|
end
|
85
85
|
end
|
86
86
|
|
87
|
-
context "when
|
87
|
+
context "when the total backup interval is too short" do
|
88
88
|
let(:interval){ 60*60*4 - test_jitter }
|
89
89
|
let(:frequency){ 60*8 }
|
90
90
|
let(:test_jitter){ 60 }
|
91
91
|
|
92
|
-
it "should mark
|
93
|
-
expect(kept_files.count).to eql
|
92
|
+
it "should mark 5 files kept" do
|
93
|
+
expect(kept_files.count).to eql 5
|
94
94
|
end
|
95
95
|
end
|
96
96
|
|
@@ -1,5 +1,6 @@
|
|
1
1
|
require 'spec_helper'
|
2
2
|
require 'pry'
|
3
|
+
require 'file-sandbox'
|
3
4
|
|
4
5
|
module BiblioTech
|
5
6
|
describe CommandGenerator, "for mysql" do
|
@@ -16,7 +17,7 @@ module BiblioTech
|
|
16
17
|
let( :password ) { "password123" }
|
17
18
|
let( :host ) { "127.0.0.1" }
|
18
19
|
let( :filename ) { "export.sql" }
|
19
|
-
let( :path ) { "
|
20
|
+
let( :path ) { "some/path" }
|
20
21
|
|
21
22
|
let :base_config_hash do
|
22
23
|
{ "database_config" => {
|
@@ -119,6 +120,8 @@ module BiblioTech
|
|
119
120
|
|
120
121
|
|
121
122
|
describe :import do
|
123
|
+
include FileSandbox
|
124
|
+
|
122
125
|
let :command do
|
123
126
|
generator.import(options)
|
124
127
|
end
|
@@ -127,38 +130,50 @@ module BiblioTech
|
|
127
130
|
command
|
128
131
|
end
|
129
132
|
|
133
|
+
|
130
134
|
context 'with username, database, file, and path' do
|
131
135
|
let :options do
|
132
136
|
{ :backups => { :filename => filename, :dir => path }}
|
133
137
|
end
|
134
138
|
|
135
|
-
it
|
136
|
-
|
137
|
-
|
138
|
-
it { expect(command.executable).to eq('mysql')}
|
139
|
-
it { expect(command.options).to eq(["-u #{username}", db_name ]) }
|
139
|
+
it "should raise an exception if load file is missing" do
|
140
|
+
expect{ command }.to raise_error(/Missing/i)
|
141
|
+
end
|
140
142
|
|
141
|
-
context
|
142
|
-
|
143
|
-
|
144
|
-
|
145
|
-
end
|
143
|
+
context 'with existent backup file' do
|
144
|
+
before :each do
|
145
|
+
sandbox.new :file => File.join(path, filename)
|
146
|
+
sandbox.new :file => File.join(path, filename + '.gz')
|
146
147
|
end
|
147
148
|
|
148
|
-
it { expect(command
|
149
|
+
it { expect(command).to be_a(Caliph::CommandLine) }
|
149
150
|
|
150
|
-
|
151
|
-
|
152
|
-
|
153
|
-
|
154
|
-
|
155
|
-
|
156
|
-
|
151
|
+
it { expect(command.redirections).to eq(["0<#{path}/#{filename}"]) }
|
152
|
+
it { expect(command.executable).to eq('mysql')}
|
153
|
+
it { expect(command.options).to eq(["-u #{username}", db_name ]) }
|
154
|
+
|
155
|
+
context "plus password" do
|
156
|
+
let :config_hash do
|
157
|
+
base_config_hash.tap do |hash|
|
158
|
+
hash["database_config"]["password"] = password
|
159
|
+
end
|
157
160
|
end
|
158
161
|
|
159
|
-
it { expect(command).to
|
160
|
-
|
161
|
-
|
162
|
+
it { expect(command.options).to eq(["-u #{username}","--password='#{password}'", "#{db_name}"]) }
|
163
|
+
|
164
|
+
context 'and compressor' do
|
165
|
+
let :options do
|
166
|
+
{ :backups => {
|
167
|
+
:filename => filename + '.gz',
|
168
|
+
:dir => path,
|
169
|
+
:compress => :gzip
|
170
|
+
} }
|
171
|
+
end
|
172
|
+
|
173
|
+
it { expect(command).to be_a(Caliph::PipelineChain) }
|
174
|
+
it { expect(first_cmd.executable).to eq('gunzip') }
|
175
|
+
it { expect(first_cmd.options).to eq(["-c", "#{path}/#{filename}.gz"]) }
|
176
|
+
end
|
162
177
|
end
|
163
178
|
end
|
164
179
|
end
|
@@ -1,6 +1,9 @@
|
|
1
1
|
require 'spec_helper'
|
2
|
+
require 'file-sandbox'
|
2
3
|
|
3
4
|
module BiblioTech
|
5
|
+
include FileSandbox
|
6
|
+
|
4
7
|
describe CommandGenerator do
|
5
8
|
let :generator do
|
6
9
|
CommandGenerator.new(config)
|
@@ -11,7 +14,7 @@ module BiblioTech
|
|
11
14
|
let (:password){ "password123" }
|
12
15
|
let (:host){ "127.0.0.1" }
|
13
16
|
let (:filename){ "export.pg" }
|
14
|
-
let (:path){ "
|
17
|
+
let (:path){ "some/path" }
|
15
18
|
|
16
19
|
|
17
20
|
let (:base_options){{}}
|
@@ -135,6 +138,8 @@ module BiblioTech
|
|
135
138
|
|
136
139
|
|
137
140
|
describe :import do
|
141
|
+
include FileSandbox
|
142
|
+
|
138
143
|
let :command do
|
139
144
|
generator.import(options)
|
140
145
|
end
|
@@ -148,32 +153,43 @@ module BiblioTech
|
|
148
153
|
base_options.merge(:backups => { :filename => filename, :dir => path })
|
149
154
|
end
|
150
155
|
|
151
|
-
it
|
152
|
-
|
153
|
-
|
156
|
+
it "should raise an exception if load file is missing" do
|
157
|
+
expect{ command }.to raise_error(/Missing/i)
|
158
|
+
end
|
154
159
|
|
155
|
-
context "
|
156
|
-
|
157
|
-
|
158
|
-
|
159
|
-
end
|
160
|
+
context "where file exists" do
|
161
|
+
before :each do
|
162
|
+
sandbox.new :file => File.join(path, filename)
|
163
|
+
sandbox.new :file => File.join(path, filename + '.gz')
|
160
164
|
end
|
161
165
|
|
162
|
-
it { expect(command.
|
163
|
-
it { expect(command.
|
166
|
+
it { expect(command.redirections).to eq(["0<#{path}/#{filename}"]) }
|
167
|
+
it { expect(command.executable).to eq('pg_restore')}
|
168
|
+
it { expect(command.options).to eq(["-Oc", "-U #{username}", "-d #{db_name}" ]) }
|
164
169
|
|
165
|
-
context
|
166
|
-
let :
|
167
|
-
|
168
|
-
|
169
|
-
|
170
|
-
:compressor => :gzip
|
171
|
-
})
|
170
|
+
context "plus password" do
|
171
|
+
let :config_hash do
|
172
|
+
base_config_hash.tap do |hash|
|
173
|
+
hash["database_config"]["password"] = password
|
174
|
+
end
|
172
175
|
end
|
173
176
|
|
174
|
-
it { expect(command).to
|
175
|
-
it { expect(
|
176
|
-
|
177
|
+
it { expect(command.options).to eq(["-Oc", "-U #{username}", "-d #{db_name}"]) }
|
178
|
+
it { expect(command.env['PGPASSWORD']).to eq(password) }
|
179
|
+
|
180
|
+
context 'and compressor' do
|
181
|
+
let :options do
|
182
|
+
base_options.merge(:backups => {
|
183
|
+
:filename => filename + '.gz',
|
184
|
+
:dir => path,
|
185
|
+
:compressor => :gzip
|
186
|
+
})
|
187
|
+
end
|
188
|
+
|
189
|
+
it { expect(command).to be_a(Caliph::PipelineChain) }
|
190
|
+
it { expect(first_cmd.executable).to eq('gunzip') }
|
191
|
+
it { expect(first_cmd.options).to eq(["-c", "#{path}/#{filename}.gz"]) }
|
192
|
+
end
|
177
193
|
end
|
178
194
|
end
|
179
195
|
end
|
@@ -101,7 +101,7 @@ module BiblioTech
|
|
101
101
|
end
|
102
102
|
end
|
103
103
|
|
104
|
-
describe "schedule
|
104
|
+
describe "schedule configuration" do
|
105
105
|
let :config do
|
106
106
|
Config.new(nil).tap do |config|
|
107
107
|
config.hash = config_hash
|
@@ -111,21 +111,108 @@ module BiblioTech
|
|
111
111
|
let :config_hash do
|
112
112
|
{ "backups" => {
|
113
113
|
"frequency" => 60,
|
114
|
-
"
|
115
|
-
|
116
|
-
|
117
|
-
|
114
|
+
"retain" => {
|
115
|
+
"periodic" => {
|
116
|
+
60 => 24,
|
117
|
+
1440 => 7
|
118
|
+
},
|
119
|
+
"calendar" => {
|
120
|
+
[ 1,1,1 ] => 10
|
121
|
+
}
|
122
|
+
}
|
123
|
+
}}
|
124
|
+
end
|
125
|
+
|
126
|
+
let :all_schedules do
|
127
|
+
config.prune_schedules
|
118
128
|
end
|
119
129
|
|
120
|
-
let :
|
121
|
-
|
130
|
+
let :periodic_array do
|
131
|
+
all_schedules.find_all{|sch| sch.class == Backups::Scheduler}.map do |sched|
|
122
132
|
[sched.frequency, sched.limit]
|
123
133
|
end
|
124
134
|
end
|
125
135
|
|
136
|
+
let :calendar_array do
|
137
|
+
all_schedules.find_all{|sch| sch.is_a? Backups::CalendarScheduler}.map do |sched|
|
138
|
+
[sched.pattern, sched.limit]
|
139
|
+
end
|
140
|
+
end
|
141
|
+
|
126
142
|
context "simple numerics" do
|
127
143
|
it "should produce correct schedule" do
|
128
|
-
expect(
|
144
|
+
expect(periodic_array).to contain_exactly([60, 24], [1440, 7])
|
145
|
+
expect(calendar_array).to contain_exactly([[1,1,1],10])
|
146
|
+
end
|
147
|
+
end
|
148
|
+
|
149
|
+
context "legacy overrides" do
|
150
|
+
let :config_hash do
|
151
|
+
{ "backups" => {
|
152
|
+
"frequency" => 60,
|
153
|
+
"keep" => {
|
154
|
+
60 => 12,
|
155
|
+
2400 => 6
|
156
|
+
},
|
157
|
+
"calendar" => {
|
158
|
+
[ 1,1,1 ] => 8,
|
159
|
+
[ 2,1,1 ] => 8
|
160
|
+
},
|
161
|
+
"retain" => {
|
162
|
+
"periodic" => {
|
163
|
+
60 => 24,
|
164
|
+
1440 => 7
|
165
|
+
},
|
166
|
+
"calendar" => {
|
167
|
+
[ 1,1,1 ] => 10
|
168
|
+
}
|
169
|
+
}
|
170
|
+
}}
|
171
|
+
end
|
172
|
+
|
173
|
+
# because it exists in deployment, and wants to be fixed
|
174
|
+
it "should prefer the legacy config" do
|
175
|
+
expect(periodic_array).to contain_exactly([60, 12], [1440, 7], [2400, 6])
|
176
|
+
expect(calendar_array).to contain_exactly([[1,1,1],10])
|
177
|
+
end
|
178
|
+
end
|
179
|
+
|
180
|
+
context "complete removal of calendar scedule" do
|
181
|
+
let :config_hash do
|
182
|
+
{ "backups" => {
|
183
|
+
"frequency" => 60,
|
184
|
+
"retain" => {
|
185
|
+
"periodic" => {
|
186
|
+
60 => 24,
|
187
|
+
1440 => 7
|
188
|
+
},
|
189
|
+
"calendar" => "none"
|
190
|
+
}
|
191
|
+
}}
|
192
|
+
end
|
193
|
+
|
194
|
+
it "should prefer the modern config" do
|
195
|
+
expect(periodic_array).to contain_exactly([60, 24], [1440, 7])
|
196
|
+
expect(calendar_array).to be_empty
|
197
|
+
end
|
198
|
+
end
|
199
|
+
|
200
|
+
context "complete removal of periodic scedule" do
|
201
|
+
let :config_hash do
|
202
|
+
{ "backups" => {
|
203
|
+
"frequency" => 60,
|
204
|
+
"retain" => {
|
205
|
+
"periodic" => "none",
|
206
|
+
"calendar" => {
|
207
|
+
[ 1,1,1 ] => 10
|
208
|
+
}
|
209
|
+
}
|
210
|
+
}}
|
211
|
+
end
|
212
|
+
|
213
|
+
it "should prefer the modern config" do
|
214
|
+
expect(periodic_array).to be_empty
|
215
|
+
expect(calendar_array).to contain_exactly([[1,1,1],10])
|
129
216
|
end
|
130
217
|
end
|
131
218
|
|
@@ -148,7 +235,7 @@ module BiblioTech
|
|
148
235
|
end
|
149
236
|
|
150
237
|
it "should prefer the local config" do
|
151
|
-
expect(
|
238
|
+
expect(periodic_array).to contain_exactly([60, 12])
|
152
239
|
end
|
153
240
|
end
|
154
241
|
|
@@ -164,7 +251,7 @@ module BiblioTech
|
|
164
251
|
|
165
252
|
it "should raise an error" do
|
166
253
|
expect do
|
167
|
-
|
254
|
+
periodic_array
|
168
255
|
end.to raise_error(/59/)
|
169
256
|
end
|
170
257
|
end
|
@@ -182,7 +269,7 @@ module BiblioTech
|
|
182
269
|
|
183
270
|
it "should raise an error" do
|
184
271
|
expect do
|
185
|
-
|
272
|
+
periodic_array
|
186
273
|
end.to raise_error
|
187
274
|
end
|
188
275
|
end
|
@@ -200,7 +287,7 @@ module BiblioTech
|
|
200
287
|
end
|
201
288
|
|
202
289
|
it "should not raise error" do
|
203
|
-
expect(
|
290
|
+
expect(periodic_array).to contain_exactly([60*24, nil])
|
204
291
|
end
|
205
292
|
end
|
206
293
|
|
@@ -217,7 +304,7 @@ module BiblioTech
|
|
217
304
|
|
218
305
|
it "should raise an error" do
|
219
306
|
expect do
|
220
|
-
|
307
|
+
periodic_array
|
221
308
|
end.to raise_error
|
222
309
|
end
|
223
310
|
end
|
@@ -226,16 +313,26 @@ module BiblioTech
|
|
226
313
|
let :config_hash do
|
227
314
|
{ "backups" => {
|
228
315
|
"frequency" => "hourly",
|
229
|
-
"
|
230
|
-
"
|
231
|
-
|
232
|
-
|
233
|
-
|
234
|
-
|
316
|
+
"retain" => {
|
317
|
+
"periodic" => {
|
318
|
+
"hourlies" => 24,
|
319
|
+
"daily" => 7,
|
320
|
+
"weeklies" => 4,
|
321
|
+
"monthly" => "all"
|
322
|
+
},
|
323
|
+
"calendar" => {
|
324
|
+
"monthly" => 12,
|
325
|
+
"quarterly" => "all"
|
326
|
+
}
|
327
|
+
}
|
328
|
+
}}
|
235
329
|
end
|
236
330
|
|
237
331
|
it "should produce correct schedule" do
|
238
|
-
expect(
|
332
|
+
expect(periodic_array).to contain_exactly([60, 24], [60*24, 7], [60*24*7, 4], [60*24*30, nil])
|
333
|
+
expect(calendar_array).to contain_exactly(
|
334
|
+
[[1,0,0],12], [[1,1,0,0],nil],[[4,1,0,0],nil],[[7,1,0,0],nil], [[10,1,0,0],nil]
|
335
|
+
)
|
239
336
|
end
|
240
337
|
end
|
241
338
|
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: bibliotech
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.8.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Evan Dorn
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2015-
|
12
|
+
date: 2015-12-19 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: caliph
|
@@ -79,47 +79,49 @@ extra_rdoc_files:
|
|
79
79
|
- doc/todo.txt
|
80
80
|
- doc/example_config_file.yml
|
81
81
|
files:
|
82
|
+
- bin/bibliotech
|
82
83
|
- default_configuration/config.yaml
|
84
|
+
- doc/example_config_file.yml
|
85
|
+
- doc/todo.txt
|
86
|
+
- lib/bibliotech.rb
|
87
|
+
- lib/bibliotech/application.rb
|
88
|
+
- lib/bibliotech/backups/calendar_scheduler.rb
|
89
|
+
- lib/bibliotech/backups/file_record.rb
|
90
|
+
- lib/bibliotech/backups/prune_list.rb
|
91
|
+
- lib/bibliotech/backups/pruner.rb
|
92
|
+
- lib/bibliotech/backups/scheduler.rb
|
83
93
|
- lib/bibliotech/builders.rb
|
84
94
|
- lib/bibliotech/builders/database.rb
|
85
|
-
- lib/bibliotech/builders/gzip.rb
|
86
95
|
- lib/bibliotech/builders/file.rb
|
87
|
-
- lib/bibliotech/builders/
|
96
|
+
- lib/bibliotech/builders/gzip.rb
|
88
97
|
- lib/bibliotech/builders/mysql.rb
|
89
|
-
- lib/bibliotech/
|
90
|
-
- lib/bibliotech/
|
91
|
-
- lib/bibliotech/command_runner.rb
|
98
|
+
- lib/bibliotech/builders/postgres.rb
|
99
|
+
- lib/bibliotech/cli.rb
|
92
100
|
- lib/bibliotech/command_generator.rb
|
93
|
-
- lib/bibliotech/
|
101
|
+
- lib/bibliotech/command_runner.rb
|
94
102
|
- lib/bibliotech/compression.rb
|
95
103
|
- lib/bibliotech/compression/bzip2.rb
|
96
104
|
- lib/bibliotech/compression/gzip.rb
|
97
105
|
- lib/bibliotech/compression/sevenzip.rb
|
98
|
-
- lib/bibliotech/
|
99
|
-
- lib/bibliotech/
|
100
|
-
- lib/bibliotech/
|
101
|
-
- lib/bibliotech/backups/file_record.rb
|
102
|
-
- lib/bibliotech/application.rb
|
103
|
-
- lib/bibliotech/cli.rb
|
106
|
+
- lib/bibliotech/config.rb
|
107
|
+
- lib/bibliotech/config/schedule.rb
|
108
|
+
- lib/bibliotech/logger.rb
|
104
109
|
- lib/bibliotech/railtie.rb
|
105
|
-
- lib/bibliotech.rb
|
106
|
-
- bin/bibliotech
|
107
|
-
- spec/spec_helper.rb
|
108
|
-
- spec/bibliotech/config_spec.rb
|
109
|
-
- spec/bibliotech/command_generator/postgres_spec.rb
|
110
|
-
- spec/bibliotech/command_generator/mysql_spec.rb
|
110
|
+
- lib/bibliotech/rake_lib.rb
|
111
111
|
- spec/bibliotech/backup_pruner_spec.rb
|
112
|
-
- spec/bibliotech/compression_spec.rb
|
113
|
-
- spec/bibliotech/compression/bzip2_spec.rb
|
114
|
-
- spec/bibliotech/compression/sevenzip_spec.rb
|
115
|
-
- spec/bibliotech/compression/bunzip2_spec.rb
|
116
|
-
- spec/bibliotech/compression/gzip_spec.rb
|
117
112
|
- spec/bibliotech/backup_scheduler_spec.rb
|
113
|
+
- spec/bibliotech/command_generator/mysql_spec.rb
|
114
|
+
- spec/bibliotech/command_generator/postgres_spec.rb
|
118
115
|
- spec/bibliotech/command_generator_spec.rb
|
119
116
|
- spec/bibliotech/command_runner_spec.rb
|
117
|
+
- spec/bibliotech/compression/bunzip2_spec.rb
|
118
|
+
- spec/bibliotech/compression/bzip2_spec.rb
|
119
|
+
- spec/bibliotech/compression/gzip_spec.rb
|
120
|
+
- spec/bibliotech/compression/sevenzip_spec.rb
|
121
|
+
- spec/bibliotech/compression_spec.rb
|
122
|
+
- spec/bibliotech/config_spec.rb
|
120
123
|
- spec/gem_test_suite.rb
|
121
|
-
-
|
122
|
-
- doc/example_config_file.yml
|
124
|
+
- spec/spec_helper.rb
|
123
125
|
homepage: ''
|
124
126
|
licenses:
|
125
127
|
- MIT
|
@@ -130,7 +132,7 @@ rdoc_options:
|
|
130
132
|
- --main
|
131
133
|
- doc/README
|
132
134
|
- --title
|
133
|
-
- bibliotech-0.
|
135
|
+
- bibliotech-0.8.0 Documentation
|
134
136
|
require_paths:
|
135
137
|
- lib/
|
136
138
|
required_ruby_version: !ruby/object:Gem::Requirement
|
@@ -145,7 +147,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
145
147
|
version: '0'
|
146
148
|
requirements: []
|
147
149
|
rubyforge_project: bibliotech
|
148
|
-
rubygems_version: 2.
|
150
|
+
rubygems_version: 2.4.8
|
149
151
|
signing_key:
|
150
152
|
specification_version: 4
|
151
153
|
summary: ''
|