boring-backup 0.4.0 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +16 -0
- data/README.md +24 -4
- data/lib/boring_backup/cli.rb +322 -3
- data/lib/boring_backup/commands/backup.rb +8 -7
- data/lib/boring_backup/commands/doctor.rb +133 -0
- data/lib/boring_backup/configuration.rb +32 -9
- data/lib/boring_backup/notifiers/sentinel.rb +3 -3
- data/lib/boring_backup/stores/r2.rb +20 -0
- data/lib/boring_backup/stores/s3.rb +13 -7
- data/lib/boring_backup/stores/store.rb +8 -0
- data/lib/boring_backup/stores.rb +0 -1
- data/lib/boring_backup/tee.rb +10 -1
- data/lib/boring_backup/version.rb +1 -1
- data/lib/boring_backup.rb +11 -0
- metadata +36 -6
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 851d7814c77a411800884379a85ae1feeffb5c8183d3b524ab3851f1fc7d1dde
|
|
4
|
+
data.tar.gz: e54bf1a64c28d237753b4eaa5c4bd7114c067a6b6f957b73ad4526f5332fc393
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 682eb6a513c065b1078ca528bebb9ea5cbc83924a873d47965e5ca463c8d94210a8f6e020242e45ea6fb8817e8c23e8c51a999070c1d5f2c5c63cf0b9ecc605a
|
|
7
|
+
data.tar.gz: 7afd54ae737169eb4f44d4bfd575acd64b13177b8c1d77d33260d217742934cbe053822cd5ebf4a64aa00a92cbfa16751553042f8a4ff1e7da6861f245f99045
|
data/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,21 @@
|
|
|
1
1
|
## [Unreleased]
|
|
2
2
|
|
|
3
|
+
## [0.6.0] - 2026-07-23
|
|
4
|
+
|
|
5
|
+
- `bb doctor` - improved scheduler checks (now supports `sidekiq-crosidekiq-cron`)
|
|
6
|
+
- support `PG*` env vars
|
|
7
|
+
- support for Cloudflare R2 using their S3-compatible API
|
|
8
|
+
|
|
9
|
+
## [0.5.0] - 2026-07-14
|
|
10
|
+
|
|
11
|
+
- **`aws-sdk-s3` is no longer a dependency of the gem.** apps using the `:s3` store must add `gem
|
|
12
|
+
"aws-sdk-s3"` to their own `Gemfile`.
|
|
13
|
+
- New `bb install` command - wires up the initializer, the Solid Queue schedule and a `bin/bb` binstub
|
|
14
|
+
- New `bb doctor` command - checks pg_dump, versions, database, stores and schedule; exits non-zero on failure
|
|
15
|
+
- `bb backup` now shows a boot spinner, a live progress meter and a coloured summary
|
|
16
|
+
- `config.register(:s3)` no longer requires a block - the store configures itself from the environment
|
|
17
|
+
- `Commands::Backup.execute` accepts a `progress:` callback with the running byte count
|
|
18
|
+
|
|
3
19
|
## [0.4.0] - 2026-07-13
|
|
4
20
|
|
|
5
21
|
- **Renamed the gem from `noop-backup` to `boring-backup`.** Breaking, with no shims:
|
data/README.md
CHANGED
|
@@ -20,6 +20,19 @@ gem install boring-backup
|
|
|
20
20
|
|
|
21
21
|
The gem requires `pg_dump` to be installed on the machine that is running it.
|
|
22
22
|
|
|
23
|
+
Storage backends bring their own dependencies, which are not installed by default. To use
|
|
24
|
+
the `:s3` and `:r2` stores, add the AWS SDK to your `Gemfile` as well:
|
|
25
|
+
|
|
26
|
+
```bash
|
|
27
|
+
bundle add aws-sdk-s3
|
|
28
|
+
```
|
|
29
|
+
|
|
30
|
+
Run the install command to set up the initializer:
|
|
31
|
+
|
|
32
|
+
```bash
|
|
33
|
+
bundle exec bb install
|
|
34
|
+
```
|
|
35
|
+
|
|
23
36
|
## Usage
|
|
24
37
|
|
|
25
38
|
### Automatic
|
|
@@ -62,6 +75,8 @@ All configuration options can be edited in the initializer:
|
|
|
62
75
|
# config/initializers/boring-backup.rb
|
|
63
76
|
|
|
64
77
|
BoringBackup.configure do |config|
|
|
78
|
+
config.sentinel_key = '12345678-abcd-1234-abcd-abcdef123456'
|
|
79
|
+
|
|
65
80
|
config.register(:s3) do |store|
|
|
66
81
|
store.bucket = Settings.aws.bucket
|
|
67
82
|
store.region = Settings.aws.region
|
|
@@ -69,6 +84,13 @@ BoringBackup.configure do |config|
|
|
|
69
84
|
store.secret_access_key = Settings.aws.secret_access_key
|
|
70
85
|
end
|
|
71
86
|
|
|
87
|
+
config.register(:r2) do |store|
|
|
88
|
+
store.endpoint = Settings.r2.endpoint
|
|
89
|
+
store.bucket = Settings.r2.bucket
|
|
90
|
+
store.access_key_id = Settings.r2.access_key_id
|
|
91
|
+
store.secret_access_key = Settings.r2.secret_access_key
|
|
92
|
+
end
|
|
93
|
+
|
|
72
94
|
config.notifier :slack do |slack|
|
|
73
95
|
slack.webhook_url = 'https://hooks.slack.com/services/whatever'
|
|
74
96
|
end
|
|
@@ -101,12 +123,10 @@ Bug reports and pull requests are welcome on GitHub at https://github.com/gmitre
|
|
|
101
123
|
|
|
102
124
|
## Wishlist
|
|
103
125
|
|
|
104
|
-
- [
|
|
126
|
+
- [X] S3-compatible backends
|
|
127
|
+
- [X] doctor command
|
|
105
128
|
- [ ] file backend
|
|
106
|
-
- [ ] email notifier
|
|
107
|
-
- [ ] SMS notifier
|
|
108
129
|
- [ ] restore command
|
|
109
|
-
- [ ] test command
|
|
110
130
|
- [ ] encryption
|
|
111
131
|
- [ ] delete stale
|
|
112
132
|
|
data/lib/boring_backup/cli.rb
CHANGED
|
@@ -1,17 +1,336 @@
|
|
|
1
1
|
require "thor"
|
|
2
|
+
require "pastel"
|
|
3
|
+
require "tty-prompt"
|
|
4
|
+
require "tty-spinner"
|
|
2
5
|
|
|
3
6
|
module BoringBackup
|
|
4
7
|
class CLI < Thor
|
|
8
|
+
include Thor::Actions
|
|
9
|
+
|
|
10
|
+
INITIALIZER = "config/initializers/boring_backup.rb"
|
|
11
|
+
RECURRING = "config/recurring.yml"
|
|
12
|
+
BINSTUB = "bin/bb"
|
|
13
|
+
|
|
14
|
+
DEFAULT_SCHEDULE = "every day at 3am"
|
|
15
|
+
PRODUCTION_ANCHOR = /^production:[ \t]*\r?\n/
|
|
16
|
+
|
|
17
|
+
SPINNER = %w[⠋ ⠙ ⠹ ⠸ ⠼ ⠴ ⠦ ⠧ ⠇ ⠏].freeze
|
|
18
|
+
REDRAW_INTERVAL = 0.1
|
|
19
|
+
|
|
20
|
+
add_runtime_options!
|
|
21
|
+
|
|
5
22
|
def self.exit_on_failure? = true
|
|
6
23
|
|
|
24
|
+
def self.source_root = File.expand_path("templates", __dir__)
|
|
25
|
+
|
|
7
26
|
desc "backup", "Create and store a new backup"
|
|
8
27
|
def backup
|
|
9
|
-
BoringBackup.prepare!
|
|
10
|
-
|
|
28
|
+
spinner("Booting") { BoringBackup.prepare! }
|
|
29
|
+
|
|
30
|
+
say " #{pastel.bold("Boring Backup")} #{pastel.dim(BoringBackup.config.pg_env["PGDATABASE"])}"
|
|
31
|
+
|
|
32
|
+
result = stream_backup
|
|
33
|
+
|
|
34
|
+
result.store_results.each { |store_result| say " #{store_line(store_result)}" }
|
|
35
|
+
say "\n #{verdict(result)}\n"
|
|
36
|
+
|
|
11
37
|
exit 1 unless result.success?
|
|
12
38
|
rescue => e
|
|
13
|
-
|
|
39
|
+
say " #{pastel.red("✗")} #{e.message}\n"
|
|
14
40
|
exit 1
|
|
15
41
|
end
|
|
42
|
+
|
|
43
|
+
desc "doctor", "Check that backups are set up correctly"
|
|
44
|
+
def doctor
|
|
45
|
+
spinner("Booting") { BoringBackup.prepare! }
|
|
46
|
+
|
|
47
|
+
say " #{pastel.bold("Boring Backup")} #{pastel.dim("doctor · #{BoringBackup.environment}")}"
|
|
48
|
+
|
|
49
|
+
result = spinner("Running checks") { BoringBackup::Commands::Doctor.execute }
|
|
50
|
+
|
|
51
|
+
width = result.checks.map { |check| check.name.length }.max
|
|
52
|
+
|
|
53
|
+
result.checks.each { |check| say " #{check_line(check, width)}" }
|
|
54
|
+
say "\n #{doctor_verdict(result)}\n"
|
|
55
|
+
|
|
56
|
+
exit 1 unless result.success?
|
|
57
|
+
rescue => e
|
|
58
|
+
say " #{pastel.red("✗")} #{e.message}\n"
|
|
59
|
+
exit 1
|
|
60
|
+
end
|
|
61
|
+
|
|
62
|
+
desc "install", "Wire up recurring backups in this app"
|
|
63
|
+
def install
|
|
64
|
+
say "\n #{pastel.bold("Boring Backup")}\n\n"
|
|
65
|
+
|
|
66
|
+
preflight
|
|
67
|
+
|
|
68
|
+
store = configure_store
|
|
69
|
+
write_initializer(store)
|
|
70
|
+
install_schedule
|
|
71
|
+
install_binstub
|
|
72
|
+
|
|
73
|
+
say_next_steps
|
|
74
|
+
end
|
|
75
|
+
|
|
76
|
+
private
|
|
77
|
+
|
|
78
|
+
def stream_backup
|
|
79
|
+
BoringBackup.config.silence_stdout!
|
|
80
|
+
|
|
81
|
+
@started = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
82
|
+
@frame = 0
|
|
83
|
+
|
|
84
|
+
BoringBackup::Commands::Backup.execute(progress: meter)
|
|
85
|
+
ensure
|
|
86
|
+
clear_line
|
|
87
|
+
end
|
|
88
|
+
|
|
89
|
+
def meter
|
|
90
|
+
return unless $stdout.tty?
|
|
91
|
+
|
|
92
|
+
->(bytes) { draw_meter(bytes) }
|
|
93
|
+
end
|
|
94
|
+
|
|
95
|
+
def draw_meter(bytes)
|
|
96
|
+
now = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
97
|
+
|
|
98
|
+
return if @drawn_at && now - @drawn_at < REDRAW_INTERVAL
|
|
99
|
+
|
|
100
|
+
@drawn_at = now
|
|
101
|
+
elapsed = now - @started
|
|
102
|
+
rate = elapsed.zero? ? 0 : bytes / elapsed
|
|
103
|
+
tick = SPINNER[@frame = (@frame + 1) % SPINNER.size]
|
|
104
|
+
|
|
105
|
+
$stdout.print "\r #{pastel.cyan(tick)} Dumping #{pastel.bold(human(bytes))} " \
|
|
106
|
+
"#{pastel.dim("· #{human(rate)}/s · #{elapsed.round}s")}\e[K"
|
|
107
|
+
end
|
|
108
|
+
|
|
109
|
+
def clear_line
|
|
110
|
+
$stdout.print "\r\e[K" if $stdout.tty?
|
|
111
|
+
end
|
|
112
|
+
|
|
113
|
+
def store_line(store_result)
|
|
114
|
+
if store_result.success
|
|
115
|
+
"#{pastel.green("✓")} #{pastel.bold(store_result.store.to_s.ljust(6))} " \
|
|
116
|
+
"#{human(store_result.bytes)} in #{store_result.duration.round(1)}s #{pastel.dim("→ /#{store_result.key}")}"
|
|
117
|
+
else
|
|
118
|
+
"#{pastel.red("✗")} #{pastel.bold(store_result.store.to_s.ljust(6))} #{pastel.red(store_result.error.message)}"
|
|
119
|
+
end
|
|
120
|
+
end
|
|
121
|
+
|
|
122
|
+
def verdict(result)
|
|
123
|
+
case result.status
|
|
124
|
+
when :success then pastel.green.bold("Backed up to #{result.store_results.size} store(s).")
|
|
125
|
+
when :partial_success then pastel.yellow.bold("Partial: #{result.store_results.count(&:success)}/#{result.store_results.size} stores succeeded.")
|
|
126
|
+
else pastel.red.bold("Backup failed. #{result.error&.message}")
|
|
127
|
+
end
|
|
128
|
+
end
|
|
129
|
+
|
|
130
|
+
def human(bytes)
|
|
131
|
+
BoringBackup.utils.human_size(bytes)
|
|
132
|
+
end
|
|
133
|
+
|
|
134
|
+
CHECK_MARKS = {
|
|
135
|
+
ok: ["✓", :green],
|
|
136
|
+
fail: ["✗", :red],
|
|
137
|
+
skip: ["–", :yellow]
|
|
138
|
+
}.freeze
|
|
139
|
+
|
|
140
|
+
def check_line(check, width)
|
|
141
|
+
mark, colour = CHECK_MARKS.fetch(check.status)
|
|
142
|
+
|
|
143
|
+
"#{pastel.decorate(mark, colour)} #{pastel.bold(check.name.to_s.ljust(width))} #{pastel.dim(check.detail.to_s)}"
|
|
144
|
+
end
|
|
145
|
+
|
|
146
|
+
def doctor_verdict(result)
|
|
147
|
+
return pastel.red.bold("#{result.failures.size} check(s) failed.") unless result.success?
|
|
148
|
+
|
|
149
|
+
skipped = result.checks.count { |check| check.status == :skip }
|
|
150
|
+
|
|
151
|
+
if skipped.zero?
|
|
152
|
+
pastel.green.bold("Everything checks out.")
|
|
153
|
+
else
|
|
154
|
+
pastel.green.bold("Checks passed.") + pastel.dim(" #{skipped} skipped — run this where the credentials live to verify the bucket.")
|
|
155
|
+
end
|
|
156
|
+
end
|
|
157
|
+
|
|
158
|
+
def preflight
|
|
159
|
+
check "Rails app", rails? && "config/environment.rb"
|
|
160
|
+
check "Scheduler", solid_queue? && "Solid Queue (#{RECURRING})", hint: "no supported scheduler found"
|
|
161
|
+
check "Database", database, hint: "could not read ActiveRecord config"
|
|
162
|
+
check "Store", configured_store_summary, hint: "none configured"
|
|
163
|
+
|
|
164
|
+
say ""
|
|
165
|
+
end
|
|
166
|
+
|
|
167
|
+
def check(label, value, hint: "not found")
|
|
168
|
+
mark = value ? pastel.green("✓") : pastel.yellow("✗")
|
|
169
|
+
detail = value ? pastel.dim(value.to_s) : pastel.dim(hint)
|
|
170
|
+
|
|
171
|
+
say " #{mark} #{label.to_s.ljust(10)} #{detail}"
|
|
172
|
+
end
|
|
173
|
+
|
|
174
|
+
def rails?
|
|
175
|
+
File.exist?("config/environment.rb")
|
|
176
|
+
end
|
|
177
|
+
|
|
178
|
+
def solid_queue?
|
|
179
|
+
File.exist?(RECURRING) && gemfile_lock.include?("solid_queue")
|
|
180
|
+
end
|
|
181
|
+
|
|
182
|
+
def gemfile_lock
|
|
183
|
+
@gemfile_lock ||= File.exist?("Gemfile.lock") ? read("Gemfile.lock") : ""
|
|
184
|
+
end
|
|
185
|
+
|
|
186
|
+
def read(path)
|
|
187
|
+
File.read(path, mode: "r:UTF-8")
|
|
188
|
+
end
|
|
189
|
+
|
|
190
|
+
def database
|
|
191
|
+
return @database if defined?(@database)
|
|
192
|
+
|
|
193
|
+
@database = spinner("Reading database config") do
|
|
194
|
+
BoringBackup.prepare!
|
|
195
|
+
BoringBackup.config.pg_env["PGDATABASE"]
|
|
196
|
+
rescue
|
|
197
|
+
nil
|
|
198
|
+
end
|
|
199
|
+
end
|
|
200
|
+
|
|
201
|
+
def spinner(message)
|
|
202
|
+
return yield unless $stdout.tty?
|
|
203
|
+
|
|
204
|
+
spinner = TTY::Spinner.new(" #{pastel.dim(message)} :spinner", format: :dots, clear: true, output: $stdout)
|
|
205
|
+
spinner.auto_spin
|
|
206
|
+
|
|
207
|
+
yield
|
|
208
|
+
ensure
|
|
209
|
+
spinner&.stop
|
|
210
|
+
clear_line
|
|
211
|
+
end
|
|
212
|
+
|
|
213
|
+
def configured_store_summary
|
|
214
|
+
"S3 — #{INITIALIZER}" if File.exist?(INITIALIZER)
|
|
215
|
+
end
|
|
216
|
+
|
|
217
|
+
def configure_store
|
|
218
|
+
return if File.exist?(INITIALIZER)
|
|
219
|
+
|
|
220
|
+
prompt.select("Where should backups go?") do |menu|
|
|
221
|
+
menu.choice "S3 (or S3-compatible: R2, MinIO, Spaces)", :s3
|
|
222
|
+
end
|
|
223
|
+
|
|
224
|
+
{
|
|
225
|
+
bucket: prompt.ask("Bucket name:", required: true),
|
|
226
|
+
region: prompt.ask("Region:", default: ENV.fetch("AWS_REGION", "us-east-1"))
|
|
227
|
+
}
|
|
228
|
+
end
|
|
229
|
+
|
|
230
|
+
def write_initializer(store)
|
|
231
|
+
unless store
|
|
232
|
+
say_status :identical, INITIALIZER, :blue
|
|
233
|
+
return
|
|
234
|
+
end
|
|
235
|
+
|
|
236
|
+
create_file INITIALIZER, initializer_body(store)
|
|
237
|
+
|
|
238
|
+
say_credentials_note
|
|
239
|
+
end
|
|
240
|
+
|
|
241
|
+
def initializer_body(store)
|
|
242
|
+
<<~RUBY
|
|
243
|
+
BoringBackup.configure do |config|
|
|
244
|
+
config.register(:s3) do |store|
|
|
245
|
+
store.bucket = #{store[:bucket].inspect}
|
|
246
|
+
store.region = #{store[:region].inspect}
|
|
247
|
+
end
|
|
248
|
+
end
|
|
249
|
+
RUBY
|
|
250
|
+
end
|
|
251
|
+
|
|
252
|
+
def say_credentials_note
|
|
253
|
+
say_status :note, "credentials come from AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY in the backup environment, or an IAM role", :blue
|
|
254
|
+
say " #{pastel.dim("Keep them out of #{INITIALIZER} — it's committed.")}\n"
|
|
255
|
+
end
|
|
256
|
+
|
|
257
|
+
def install_schedule
|
|
258
|
+
return print_manual_schedule unless solid_queue?
|
|
259
|
+
|
|
260
|
+
recurring = read(RECURRING)
|
|
261
|
+
|
|
262
|
+
if recurring.include?("BoringBackup::BackupJob")
|
|
263
|
+
say_status :identical, "#{RECURRING} already runs BoringBackup::BackupJob", :blue
|
|
264
|
+
return
|
|
265
|
+
end
|
|
266
|
+
|
|
267
|
+
unless recurring.match?(PRODUCTION_ANCHOR)
|
|
268
|
+
say_status :skip, "#{RECURRING} has no `production:` block", :yellow
|
|
269
|
+
return print_manual_schedule
|
|
270
|
+
end
|
|
271
|
+
|
|
272
|
+
schedule = prompt.ask("Schedule:", default: DEFAULT_SCHEDULE)
|
|
273
|
+
|
|
274
|
+
return unless prompt.yes?("Add the backup job to #{RECURRING} under `production:`?")
|
|
275
|
+
|
|
276
|
+
inject_schedule(recurring, schedule)
|
|
277
|
+
|
|
278
|
+
say_status :insert, RECURRING, :green
|
|
279
|
+
say_status :note, "backups run in production only — dev and staging are untouched", :blue
|
|
280
|
+
end
|
|
281
|
+
|
|
282
|
+
def inject_schedule(recurring, schedule)
|
|
283
|
+
entry = <<~YAML.gsub(/^(?=.)/, " ")
|
|
284
|
+
boring_backup:
|
|
285
|
+
class: BoringBackup::BackupJob
|
|
286
|
+
schedule: #{schedule}
|
|
287
|
+
YAML
|
|
288
|
+
|
|
289
|
+
updated = recurring.sub(PRODUCTION_ANCHOR) { |anchor| "#{anchor}#{entry}" }
|
|
290
|
+
|
|
291
|
+
File.write(RECURRING, updated, mode: "w:UTF-8")
|
|
292
|
+
end
|
|
293
|
+
|
|
294
|
+
def print_manual_schedule
|
|
295
|
+
say "\n #{pastel.yellow("No supported scheduler detected.")} Run the backup from cron instead:\n\n"
|
|
296
|
+
say "#{pastel.dim(" 0 3 * * * cd /path/to/app && #{backup_command}")}\n\n"
|
|
297
|
+
end
|
|
298
|
+
|
|
299
|
+
def install_binstub
|
|
300
|
+
return unless File.directory?("bin")
|
|
301
|
+
|
|
302
|
+
if binstub?
|
|
303
|
+
say_status :identical, BINSTUB, :blue
|
|
304
|
+
return
|
|
305
|
+
end
|
|
306
|
+
|
|
307
|
+
return unless prompt.yes?("Add a #{BINSTUB} binstub? (drops the `bundle exec` prefix)")
|
|
308
|
+
|
|
309
|
+
run "bundle binstubs boring-backup"
|
|
310
|
+
end
|
|
311
|
+
|
|
312
|
+
def binstub?
|
|
313
|
+
File.exist?(BINSTUB)
|
|
314
|
+
end
|
|
315
|
+
|
|
316
|
+
def backup_command
|
|
317
|
+
binstub? ? "#{BINSTUB} backup" : "bundle exec bb backup"
|
|
318
|
+
end
|
|
319
|
+
|
|
320
|
+
def say_next_steps
|
|
321
|
+
say "\n #{pastel.bold("Next:")} #{pastel.cyan(doctor_command)} #{pastel.dim("# check the setup before it runs at 3am")}\n\n"
|
|
322
|
+
end
|
|
323
|
+
|
|
324
|
+
def doctor_command
|
|
325
|
+
binstub? ? "#{BINSTUB} doctor" : "bundle exec bb doctor"
|
|
326
|
+
end
|
|
327
|
+
|
|
328
|
+
def prompt
|
|
329
|
+
@prompt ||= TTY::Prompt.new
|
|
330
|
+
end
|
|
331
|
+
|
|
332
|
+
def pastel
|
|
333
|
+
@pastel ||= Pastel.new
|
|
334
|
+
end
|
|
16
335
|
end
|
|
17
336
|
end
|
|
@@ -26,16 +26,17 @@ module BoringBackup::Commands
|
|
|
26
26
|
end
|
|
27
27
|
|
|
28
28
|
class Backup
|
|
29
|
-
def self.execute(report: BoringBackup.config.report
|
|
30
|
-
result = new.execute
|
|
29
|
+
def self.execute(report: BoringBackup.config.report?, progress: nil)
|
|
30
|
+
result = new(progress: progress).execute
|
|
31
31
|
|
|
32
32
|
result.report if report
|
|
33
33
|
|
|
34
34
|
result
|
|
35
35
|
end
|
|
36
36
|
|
|
37
|
-
def initialize
|
|
37
|
+
def initialize(progress: nil)
|
|
38
38
|
@key = generate_key
|
|
39
|
+
@progress = progress
|
|
39
40
|
@store_results = []
|
|
40
41
|
@sinks = []
|
|
41
42
|
end
|
|
@@ -45,8 +46,8 @@ module BoringBackup::Commands
|
|
|
45
46
|
|
|
46
47
|
commands = [config.dump_command]
|
|
47
48
|
|
|
48
|
-
# Pipe pg_dump through pv
|
|
49
|
-
commands << ["pv", "-btra"] if config.report? && system("which", "pv", out: File::NULL, err: File::NULL)
|
|
49
|
+
# Pipe pg_dump through pv only when the caller isn't counting bytes itself
|
|
50
|
+
commands << ["pv", "-btra"] if @progress.nil? && config.report? && system("which", "pv", out: File::NULL, err: File::NULL)
|
|
50
51
|
|
|
51
52
|
Open3.pipeline_r(*commands) do |last_stdout, wait_threads|
|
|
52
53
|
@sinks = config.stores.map do |store|
|
|
@@ -56,7 +57,7 @@ module BoringBackup::Commands
|
|
|
56
57
|
store.backup!(@key, reader)
|
|
57
58
|
rescue => e
|
|
58
59
|
# *always* return a Result, even if unexpected. Add store name for debugging.
|
|
59
|
-
BoringBackup::Stores::Result.new(success: false, error: e, store: store.
|
|
60
|
+
BoringBackup::Stores::Result.new(success: false, error: e, store: store.name, key: @key)
|
|
60
61
|
ensure
|
|
61
62
|
reader.close
|
|
62
63
|
end
|
|
@@ -64,7 +65,7 @@ module BoringBackup::Commands
|
|
|
64
65
|
BoringBackup::Tee::Sink.new(store:, writer:, thread:)
|
|
65
66
|
end
|
|
66
67
|
|
|
67
|
-
sinks_fanout = BoringBackup::Tee.new(@sinks)
|
|
68
|
+
sinks_fanout = BoringBackup::Tee.new(@sinks, progress: @progress)
|
|
68
69
|
|
|
69
70
|
begin
|
|
70
71
|
IO.copy_stream(last_stdout, sinks_fanout)
|
|
@@ -0,0 +1,133 @@
|
|
|
1
|
+
module BoringBackup::Commands
|
|
2
|
+
Check = Struct.new(:name, :status, :detail, keyword_init: true) do
|
|
3
|
+
def failed? = status == :fail
|
|
4
|
+
def ok? = status == :ok
|
|
5
|
+
end
|
|
6
|
+
|
|
7
|
+
DoctorResult = Struct.new(:checks, keyword_init: true) do
|
|
8
|
+
def success? = checks.none?(&:failed?)
|
|
9
|
+
|
|
10
|
+
def failures = checks.select(&:failed?)
|
|
11
|
+
end
|
|
12
|
+
|
|
13
|
+
class Doctor
|
|
14
|
+
SOLID_QUEUE_SCHEDULE_FILE = "config/recurring.yml"
|
|
15
|
+
SIDEKIQ_CRON_SCHEDULE_FILE = "config/schedule.yml"
|
|
16
|
+
def self.execute = new.execute
|
|
17
|
+
|
|
18
|
+
def execute
|
|
19
|
+
checks = [
|
|
20
|
+
pg_dump_present,
|
|
21
|
+
pg_dump_version,
|
|
22
|
+
database_resolved,
|
|
23
|
+
stores_registered,
|
|
24
|
+
*config.stores.flat_map { |store| store_checks(store) },
|
|
25
|
+
*scheduler_checks
|
|
26
|
+
]
|
|
27
|
+
|
|
28
|
+
DoctorResult.new(checks: checks.compact)
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
private
|
|
32
|
+
|
|
33
|
+
def check(name, status, detail = nil)
|
|
34
|
+
Check.new(name: name, status: status, detail: detail)
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
def pg_dump_present
|
|
38
|
+
if pg_dump_path
|
|
39
|
+
check("pg_dump", :ok, pg_dump_path)
|
|
40
|
+
else
|
|
41
|
+
check("pg_dump", :fail, "not found on PATH")
|
|
42
|
+
end
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
def pg_dump_path
|
|
46
|
+
return @pg_dump_path if defined?(@pg_dump_path)
|
|
47
|
+
|
|
48
|
+
@pg_dump_path = ENV.fetch("PATH", "").split(File::PATH_SEPARATOR)
|
|
49
|
+
.map { |dir| File.join(dir, "pg_dump") }
|
|
50
|
+
.find { |path| File.executable?(path) }
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
def pg_dump_version
|
|
54
|
+
return unless pg_dump_path
|
|
55
|
+
|
|
56
|
+
client = major_version(`#{pg_dump_path} --version`)
|
|
57
|
+
server = major_version(server_version)
|
|
58
|
+
|
|
59
|
+
return check("versions", :skip, "could not read server version") unless server
|
|
60
|
+
return check("versions", :skip, "could not read pg_dump version") unless client
|
|
61
|
+
|
|
62
|
+
if client >= server
|
|
63
|
+
check("versions", :ok, "pg_dump #{client}, server #{server}")
|
|
64
|
+
else
|
|
65
|
+
check("versions", :fail, "pg_dump #{client} is older than server #{server} — the dump may not restore")
|
|
66
|
+
end
|
|
67
|
+
end
|
|
68
|
+
|
|
69
|
+
def major_version(string)
|
|
70
|
+
string.to_s[/(\d+)/, 1]&.to_i
|
|
71
|
+
end
|
|
72
|
+
|
|
73
|
+
def server_version
|
|
74
|
+
return unless defined?(::ActiveRecord)
|
|
75
|
+
|
|
76
|
+
::ActiveRecord::Base.connection.select_value("SHOW server_version")
|
|
77
|
+
rescue
|
|
78
|
+
nil
|
|
79
|
+
end
|
|
80
|
+
|
|
81
|
+
def database_resolved
|
|
82
|
+
database = config.pg_env["PGDATABASE"]
|
|
83
|
+
|
|
84
|
+
if database.to_s.empty?
|
|
85
|
+
check("database", :fail, "could not resolve PGDATABASE")
|
|
86
|
+
else
|
|
87
|
+
check("database", :ok, database)
|
|
88
|
+
end
|
|
89
|
+
end
|
|
90
|
+
|
|
91
|
+
def stores_registered
|
|
92
|
+
if config.stores.empty?
|
|
93
|
+
check("stores", :fail, "no stores registered")
|
|
94
|
+
else
|
|
95
|
+
check("stores", :ok, "#{config.stores.size} registered")
|
|
96
|
+
end
|
|
97
|
+
end
|
|
98
|
+
|
|
99
|
+
def store_checks(store)
|
|
100
|
+
store.validate!
|
|
101
|
+
|
|
102
|
+
[check(store.name, :ok, store.description)]
|
|
103
|
+
rescue => e
|
|
104
|
+
[check(store.name, :fail, e.message)]
|
|
105
|
+
end
|
|
106
|
+
|
|
107
|
+
def scheduler_checks
|
|
108
|
+
checks = [solid_queue_check, sidekiq_cron_check].compact
|
|
109
|
+
|
|
110
|
+
if checks.none?(&:ok?)
|
|
111
|
+
[check("scheduler detected", :skip, "no schedulers detected. Make sure BackupJob is executed in a cron or within a scheduler.")]
|
|
112
|
+
else
|
|
113
|
+
checks.select(&:ok?)
|
|
114
|
+
end
|
|
115
|
+
end
|
|
116
|
+
|
|
117
|
+
def solid_queue_check
|
|
118
|
+
if File.exist?(SOLID_QUEUE_SCHEDULE_FILE) && File.read(SOLID_QUEUE_SCHEDULE_FILE, mode: "r:UTF-8").include?("BoringBackup::BackupJob")
|
|
119
|
+
check("scheduler detected", :ok, "solid-queue: #{SOLID_QUEUE_SCHEDULE_FILE}")
|
|
120
|
+
end
|
|
121
|
+
end
|
|
122
|
+
|
|
123
|
+
def sidekiq_cron_check
|
|
124
|
+
if File.exist?(SIDEKIQ_CRON_SCHEDULE_FILE) && File.read(SIDEKIQ_CRON_SCHEDULE_FILE, mode: "r:UTF-8").include?("BoringBackup::BackupJob")
|
|
125
|
+
check("scheduler detected", :ok, "sidekiq-cron: #{SIDEKIQ_CRON_SCHEDULE_FILE}")
|
|
126
|
+
end
|
|
127
|
+
end
|
|
128
|
+
|
|
129
|
+
def config
|
|
130
|
+
@config ||= BoringBackup.config
|
|
131
|
+
end
|
|
132
|
+
end
|
|
133
|
+
end
|
|
@@ -47,6 +47,10 @@ module BoringBackup
|
|
|
47
47
|
[*@notifiers, sentinel].compact
|
|
48
48
|
end
|
|
49
49
|
|
|
50
|
+
def silence_stdout!
|
|
51
|
+
@notifiers.reject! { |notifier| notifier.is_a?(BoringBackup::Notifiers::Stdout) }
|
|
52
|
+
end
|
|
53
|
+
|
|
50
54
|
def ignore_tables
|
|
51
55
|
Array(@ignore_tables).map { |table| table.to_s.strip }.reject(&:empty?).uniq
|
|
52
56
|
end
|
|
@@ -62,17 +66,18 @@ module BoringBackup
|
|
|
62
66
|
end
|
|
63
67
|
|
|
64
68
|
def register(store_type)
|
|
65
|
-
raise BoringBackup::ConfigurationError, "`config.register` requires a block" unless block_given?
|
|
66
|
-
|
|
67
69
|
store =
|
|
68
70
|
case store_type.to_sym
|
|
69
|
-
when :s3 then
|
|
71
|
+
when :s3 then build_s3_store
|
|
72
|
+
when :r2 then build_r2_store
|
|
70
73
|
else raise BoringBackup::ConfigurationError, "unknown store type: #{store_type}"
|
|
71
74
|
end
|
|
72
75
|
|
|
73
76
|
@stores << store
|
|
74
77
|
|
|
75
|
-
yield store
|
|
78
|
+
yield store if block_given?
|
|
79
|
+
|
|
80
|
+
store
|
|
76
81
|
end
|
|
77
82
|
|
|
78
83
|
def notifier(type)
|
|
@@ -87,13 +92,31 @@ module BoringBackup
|
|
|
87
92
|
end
|
|
88
93
|
end
|
|
89
94
|
|
|
95
|
+
def build_s3_store
|
|
96
|
+
require_relative "stores/s3"
|
|
97
|
+
|
|
98
|
+
BoringBackup::Stores::S3.new
|
|
99
|
+
rescue LoadError
|
|
100
|
+
raise BoringBackup::ConfigurationError,
|
|
101
|
+
"the :s3 store needs the aws-sdk-s3 gem. Add `gem \"aws-sdk-s3\"` to your Gemfile."
|
|
102
|
+
end
|
|
103
|
+
|
|
104
|
+
def build_r2_store
|
|
105
|
+
require_relative "stores/r2"
|
|
106
|
+
|
|
107
|
+
BoringBackup::Stores::R2.new
|
|
108
|
+
rescue LoadError
|
|
109
|
+
raise BoringBackup::ConfigurationError,
|
|
110
|
+
"the :r2 store needs the aws-sdk-s3 gem. Add `gem \"aws-sdk-s3\"` to your Gemfile."
|
|
111
|
+
end
|
|
112
|
+
|
|
90
113
|
def pg_env
|
|
91
114
|
{
|
|
92
|
-
"PGHOST" => (pg_host || db_config[:host])&.to_s,
|
|
93
|
-
"PGPORT" => (pg_port || db_config[:port])&.to_s,
|
|
94
|
-
"PGUSER" => (pg_user || db_config[:username])&.to_s,
|
|
95
|
-
"PGPASSWORD" => (pg_password || db_config[:password])&.to_s,
|
|
96
|
-
"PGDATABASE" => (pg_database || db_config[:database])&.to_s
|
|
115
|
+
"PGHOST" => (pg_host || db_config[:host] || ENV["PGHOST"])&.to_s,
|
|
116
|
+
"PGPORT" => (pg_port || db_config[:port] || ENV["PGPORT"])&.to_s,
|
|
117
|
+
"PGUSER" => (pg_user || db_config[:username] || ENV["PGUSER"])&.to_s,
|
|
118
|
+
"PGPASSWORD" => (pg_password || db_config[:password] || ENV["PGPASSWORD"])&.to_s,
|
|
119
|
+
"PGDATABASE" => (pg_database || db_config[:database] || ENV["PGDATABASE"])&.to_s
|
|
97
120
|
}.compact
|
|
98
121
|
end
|
|
99
122
|
|
|
@@ -52,11 +52,11 @@ module BoringBackup::Notifiers
|
|
|
52
52
|
private
|
|
53
53
|
|
|
54
54
|
def normalized_host
|
|
55
|
-
|
|
55
|
+
return host if host.include?("://")
|
|
56
56
|
|
|
57
|
-
|
|
57
|
+
scheme = LOCAL_HOSTS.include?(host.split(":").first) ? "http" : "https"
|
|
58
58
|
|
|
59
|
-
"#{
|
|
59
|
+
"#{scheme}://#{host}"
|
|
60
60
|
end
|
|
61
61
|
|
|
62
62
|
def post(body)
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
require "aws-sdk-s3"
|
|
2
|
+
require_relative "s3"
|
|
3
|
+
|
|
4
|
+
module BoringBackup::Stores
|
|
5
|
+
class R2 < S3
|
|
6
|
+
def name = :r2
|
|
7
|
+
|
|
8
|
+
def region
|
|
9
|
+
value = super
|
|
10
|
+
|
|
11
|
+
value.to_s.empty? ? "auto" : value
|
|
12
|
+
end
|
|
13
|
+
|
|
14
|
+
def validate!
|
|
15
|
+
super
|
|
16
|
+
|
|
17
|
+
raise BoringBackup::ConfigurationError, "endpoint is not configured" if endpoint.to_s.empty?
|
|
18
|
+
end
|
|
19
|
+
end
|
|
20
|
+
end
|
|
@@ -18,11 +18,10 @@ module BoringBackup::Stores
|
|
|
18
18
|
snow
|
|
19
19
|
].freeze
|
|
20
20
|
|
|
21
|
-
attr_accessor :bucket, :region, :access_key_id, :secret_access_key, :part_size, :thread_count
|
|
21
|
+
attr_accessor :bucket, :region, :access_key_id, :secret_access_key, :part_size, :thread_count, :endpoint
|
|
22
22
|
attr_writer :storage_class
|
|
23
23
|
|
|
24
24
|
def initialize
|
|
25
|
-
@bucket = ENV["AWS_S3_BUCKET"]
|
|
26
25
|
@region = ENV["AWS_REGION"]
|
|
27
26
|
@access_key_id = ENV["AWS_ACCESS_KEY_ID"]
|
|
28
27
|
@secret_access_key = ENV["AWS_SECRET_ACCESS_KEY"]
|
|
@@ -37,6 +36,12 @@ module BoringBackup::Stores
|
|
|
37
36
|
value.empty? ? nil : value.to_sym
|
|
38
37
|
end
|
|
39
38
|
|
|
39
|
+
def name = :s3
|
|
40
|
+
|
|
41
|
+
def description
|
|
42
|
+
[bucket.to_s.empty? ? "no bucket" : "#{name}://#{bucket}", region].compact.join(" ")
|
|
43
|
+
end
|
|
44
|
+
|
|
40
45
|
# Prefer Aws::S3::TransferManager for streaming uploads if available.
|
|
41
46
|
# Aws::S3::Resource.upload_stream is deprecated in newer versions
|
|
42
47
|
def backup!(key, stream)
|
|
@@ -67,13 +72,13 @@ module BoringBackup::Stores
|
|
|
67
72
|
raise BoringBackup::DumpTooSmallError, "backup too small: #{BoringBackup.utils.human_size(bytes)} < min_size (#{BoringBackup.utils.human_size(config.min_size)})"
|
|
68
73
|
end
|
|
69
74
|
|
|
70
|
-
Result.new(success: true, store:
|
|
75
|
+
Result.new(success: true, store: name, bytes:, key:, duration:)
|
|
71
76
|
rescue => e
|
|
72
77
|
duration = Process.clock_gettime(Process::CLOCK_MONOTONIC) - started
|
|
73
78
|
|
|
74
79
|
cleanup!(key) if upload_attempted
|
|
75
80
|
|
|
76
|
-
Result.new(success: false, error: e, store:
|
|
81
|
+
Result.new(success: false, error: e, store: name, bytes:, key:, duration:)
|
|
77
82
|
end
|
|
78
83
|
|
|
79
84
|
def validate!
|
|
@@ -110,9 +115,10 @@ module BoringBackup::Stores
|
|
|
110
115
|
|
|
111
116
|
def s3_config
|
|
112
117
|
{
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
118
|
+
endpoint:,
|
|
119
|
+
region:,
|
|
120
|
+
access_key_id:,
|
|
121
|
+
secret_access_key:
|
|
116
122
|
}.compact
|
|
117
123
|
end
|
|
118
124
|
end
|
data/lib/boring_backup/stores.rb
CHANGED
data/lib/boring_backup/tee.rb
CHANGED
|
@@ -22,11 +22,20 @@ module BoringBackup
|
|
|
22
22
|
end
|
|
23
23
|
end
|
|
24
24
|
|
|
25
|
-
|
|
25
|
+
attr_reader :bytes
|
|
26
|
+
|
|
27
|
+
def initialize(sinks, progress: nil)
|
|
28
|
+
@sinks = sinks
|
|
29
|
+
@progress = progress
|
|
30
|
+
@bytes = 0
|
|
31
|
+
end
|
|
26
32
|
|
|
27
33
|
def write(chunk)
|
|
28
34
|
@sinks.each { |sink| sink.write(chunk) }
|
|
29
35
|
|
|
36
|
+
@bytes += chunk.bytesize
|
|
37
|
+
@progress&.call(@bytes)
|
|
38
|
+
|
|
30
39
|
chunk.bytesize
|
|
31
40
|
end
|
|
32
41
|
end
|
data/lib/boring_backup.rb
CHANGED
|
@@ -4,6 +4,7 @@ require_relative "boring_backup/tee"
|
|
|
4
4
|
require_relative "boring_backup/utils"
|
|
5
5
|
require_relative "boring_backup/stores"
|
|
6
6
|
require_relative "boring_backup/commands/backup"
|
|
7
|
+
require_relative "boring_backup/commands/doctor"
|
|
7
8
|
require_relative "boring_backup/notifiers/slack"
|
|
8
9
|
require_relative "boring_backup/notifiers/stdout"
|
|
9
10
|
require_relative "boring_backup/notifiers/sentinel"
|
|
@@ -53,6 +54,16 @@ module BoringBackup
|
|
|
53
54
|
require env_file if File.exist?(env_file)
|
|
54
55
|
end
|
|
55
56
|
|
|
57
|
+
def environment
|
|
58
|
+
return ::Rails.env.to_s if defined?(::Rails) && ::Rails.respond_to?(:env)
|
|
59
|
+
|
|
60
|
+
ENV["RAILS_ENV"] || ENV["RACK_ENV"] || "development"
|
|
61
|
+
end
|
|
62
|
+
|
|
63
|
+
def production?
|
|
64
|
+
environment == "production"
|
|
65
|
+
end
|
|
66
|
+
|
|
56
67
|
def notify(result)
|
|
57
68
|
config.notifiers.each do |notifier|
|
|
58
69
|
notifier.notify(result)
|
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: boring-backup
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.
|
|
4
|
+
version: 0.6.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Georgi Mitrev
|
|
@@ -24,19 +24,47 @@ dependencies:
|
|
|
24
24
|
- !ruby/object:Gem::Version
|
|
25
25
|
version: '0'
|
|
26
26
|
- !ruby/object:Gem::Dependency
|
|
27
|
-
name:
|
|
27
|
+
name: pastel
|
|
28
28
|
requirement: !ruby/object:Gem::Requirement
|
|
29
29
|
requirements:
|
|
30
|
-
- - "
|
|
30
|
+
- - ">="
|
|
31
|
+
- !ruby/object:Gem::Version
|
|
32
|
+
version: '0'
|
|
33
|
+
type: :runtime
|
|
34
|
+
prerelease: false
|
|
35
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
36
|
+
requirements:
|
|
37
|
+
- - ">="
|
|
38
|
+
- !ruby/object:Gem::Version
|
|
39
|
+
version: '0'
|
|
40
|
+
- !ruby/object:Gem::Dependency
|
|
41
|
+
name: tty-prompt
|
|
42
|
+
requirement: !ruby/object:Gem::Requirement
|
|
43
|
+
requirements:
|
|
44
|
+
- - ">="
|
|
45
|
+
- !ruby/object:Gem::Version
|
|
46
|
+
version: '0'
|
|
47
|
+
type: :runtime
|
|
48
|
+
prerelease: false
|
|
49
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
50
|
+
requirements:
|
|
51
|
+
- - ">="
|
|
31
52
|
- !ruby/object:Gem::Version
|
|
32
|
-
version: '
|
|
53
|
+
version: '0'
|
|
54
|
+
- !ruby/object:Gem::Dependency
|
|
55
|
+
name: tty-spinner
|
|
56
|
+
requirement: !ruby/object:Gem::Requirement
|
|
57
|
+
requirements:
|
|
58
|
+
- - ">="
|
|
59
|
+
- !ruby/object:Gem::Version
|
|
60
|
+
version: '0'
|
|
33
61
|
type: :runtime
|
|
34
62
|
prerelease: false
|
|
35
63
|
version_requirements: !ruby/object:Gem::Requirement
|
|
36
64
|
requirements:
|
|
37
|
-
- - "
|
|
65
|
+
- - ">="
|
|
38
66
|
- !ruby/object:Gem::Version
|
|
39
|
-
version: '
|
|
67
|
+
version: '0'
|
|
40
68
|
description: The simplest way to add recurring database backups to your project with
|
|
41
69
|
minimal setup required.
|
|
42
70
|
email:
|
|
@@ -55,6 +83,7 @@ files:
|
|
|
55
83
|
- lib/boring_backup.rb
|
|
56
84
|
- lib/boring_backup/cli.rb
|
|
57
85
|
- lib/boring_backup/commands/backup.rb
|
|
86
|
+
- lib/boring_backup/commands/doctor.rb
|
|
58
87
|
- lib/boring_backup/configuration.rb
|
|
59
88
|
- lib/boring_backup/jobs/backup_job.rb
|
|
60
89
|
- lib/boring_backup/notifiers/sentinel.rb
|
|
@@ -62,6 +91,7 @@ files:
|
|
|
62
91
|
- lib/boring_backup/notifiers/stdout.rb
|
|
63
92
|
- lib/boring_backup/plugins/rails.rb
|
|
64
93
|
- lib/boring_backup/stores.rb
|
|
94
|
+
- lib/boring_backup/stores/r2.rb
|
|
65
95
|
- lib/boring_backup/stores/result.rb
|
|
66
96
|
- lib/boring_backup/stores/s3.rb
|
|
67
97
|
- lib/boring_backup/stores/store.rb
|