bard-backup 0.1.0 → 0.3.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +1 -1
- data/lib/bard/backup/controller.rb +12 -0
- data/lib/bard/backup/deleter.rb +41 -0
- data/lib/bard/backup/s3_dir.rb +61 -0
- data/lib/bard/backup/version.rb +6 -0
- data/lib/bard/backup.rb +10 -85
- metadata +63 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 30f8ca36634a8ee2f414e6aa238a27227d71b9033904b19ff55c649dd8bfe71b
|
4
|
+
data.tar.gz: 2c858195ad3e2e23d84cbce7be7d99ab0000fec2d39f2cd1d409774987067f2d
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 44346cf310237c1d24c60733328dff6ac819ce050a45a183af50b269e61d9ebef420af721e961e612d5fff5d083d53c70dbf8b214b1443d9eee265df6f5b25ba
|
7
|
+
data.tar.gz: 5bcf351a8697340af6e2388b42cf513a7dab8b76da456c0d4ba659dd846e32455eb447148cf77006dd32dd4992a4100ffd6b0f9b7662efe55ac633cf544fd3cf
|
data/README.md
CHANGED
@@ -2,7 +2,7 @@
|
|
2
2
|
|
3
3
|
Bard::Backup does 3 things in a bard project
|
4
4
|
1. Takes a database dump and uploads it to our s3 bucket
|
5
|
-
2. Deletes old backups using a backoff heuristic:
|
5
|
+
2. Deletes old backups using a backoff heuristic: 72 hours, 60 days, 52 weeks, 48 months, then yearly
|
6
6
|
3. Raises an error if we don't have a backup from the previous hour
|
7
7
|
|
8
8
|
## Installation
|
@@ -0,0 +1,41 @@
|
|
1
|
+
require "active_support"
|
2
|
+
require "active_support/core_ext/date_time/calculations"
|
3
|
+
require "active_support/core_ext/integer/time"
|
4
|
+
|
5
|
+
module Bard
|
6
|
+
module Backup
|
7
|
+
class Deleter < Struct.new(:s3_dir, :now)
|
8
|
+
def call
|
9
|
+
s3_dir.delete keys_to_delete
|
10
|
+
end
|
11
|
+
|
12
|
+
def keys_to_delete
|
13
|
+
s3_dir.keys.select do |key|
|
14
|
+
[
|
15
|
+
Filter.new(now, 72, :hours),
|
16
|
+
Filter.new(now, 60, :days),
|
17
|
+
Filter.new(now, 52, :weeks),
|
18
|
+
Filter.new(now, 48, :months),
|
19
|
+
Filter.new(now, 100, :years),
|
20
|
+
].all? { |filter| !filter.cover?(key) }
|
21
|
+
end
|
22
|
+
end
|
23
|
+
|
24
|
+
class Filter < Struct.new(:now, :limit, :unit)
|
25
|
+
def cover? key
|
26
|
+
remote = DateTime.parse(key).beginning_of_hour
|
27
|
+
limit.times.any? do |count|
|
28
|
+
remote == ago(count)
|
29
|
+
end
|
30
|
+
end
|
31
|
+
|
32
|
+
private
|
33
|
+
|
34
|
+
def ago count
|
35
|
+
now.send(:"beginning_of_#{unit.to_s.singularize}") - count.send(unit)
|
36
|
+
end
|
37
|
+
end
|
38
|
+
end
|
39
|
+
end
|
40
|
+
end
|
41
|
+
|
@@ -0,0 +1,61 @@
|
|
1
|
+
require "aws-sdk-s3"
|
2
|
+
require "rexml"
|
3
|
+
|
4
|
+
module Bard
|
5
|
+
module Backup
|
6
|
+
class S3Dir < Data.define(:path, :access_key, :secret_key)
|
7
|
+
def keys
|
8
|
+
response = client.list_objects_v2({
|
9
|
+
bucket: bucket_name,
|
10
|
+
prefix: folder_prefix,
|
11
|
+
})
|
12
|
+
raise if response.is_truncated
|
13
|
+
response.contents.map(&:key)
|
14
|
+
end
|
15
|
+
|
16
|
+
def put file_path, body: File.read(file_path)
|
17
|
+
client.put_object({
|
18
|
+
bucket: bucket_name,
|
19
|
+
key: "#{folder_prefix}/#{File.basename(file_path)}",
|
20
|
+
body: body,
|
21
|
+
})
|
22
|
+
end
|
23
|
+
|
24
|
+
def delete keys
|
25
|
+
objects_to_delete = Array(keys).map { |key| { key: key } }
|
26
|
+
client.delete_objects({
|
27
|
+
bucket: bucket_name,
|
28
|
+
delete: {
|
29
|
+
objects: objects_to_delete,
|
30
|
+
quiet: true,
|
31
|
+
}
|
32
|
+
})
|
33
|
+
end
|
34
|
+
|
35
|
+
def empty!
|
36
|
+
keys.each_slice(1000) do |key_batch|
|
37
|
+
delete key_batch
|
38
|
+
end
|
39
|
+
end
|
40
|
+
|
41
|
+
def bucket_name
|
42
|
+
path.split("/").first
|
43
|
+
end
|
44
|
+
|
45
|
+
def folder_prefix
|
46
|
+
path.split("/")[1..].join("/")
|
47
|
+
end
|
48
|
+
|
49
|
+
private
|
50
|
+
|
51
|
+
def client
|
52
|
+
Aws::S3::Client.new({
|
53
|
+
region: "us-west-2",
|
54
|
+
access_key_id: access_key,
|
55
|
+
secret_access_key: secret_key,
|
56
|
+
})
|
57
|
+
end
|
58
|
+
end
|
59
|
+
end
|
60
|
+
end
|
61
|
+
|
data/lib/bard/backup.rb
CHANGED
@@ -1,91 +1,16 @@
|
|
1
|
-
|
2
|
-
|
3
|
-
require "
|
4
|
-
require "openssl"
|
5
|
-
require "base64"
|
1
|
+
require "bard/backup/controller"
|
2
|
+
require "bard/backup/deleter"
|
3
|
+
require "bard/backup/s3_dir"
|
6
4
|
require "backhoe"
|
7
5
|
|
8
6
|
module Bard
|
9
|
-
|
10
|
-
def self.call s3_path, access_key:, secret_key:
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
@time = Time.now
|
16
|
-
|
17
|
-
Backhoe.dump path
|
18
|
-
|
19
|
-
self.full_s3_path = "/#{s3_path}/#{filename}"
|
20
|
-
|
21
|
-
if s3_path.include?("/")
|
22
|
-
s3_bucket = s3_path.split("/").first
|
23
|
-
self.s3_path = s3_path.split("/")[1..].join("/")
|
24
|
-
uri = URI("https://#{s3_bucket}.s3.amazonaws.com/#{s3_path}/#{filename}")
|
25
|
-
else
|
26
|
-
uri = URI("https://#{s3_path}.s3.amazonaws.com/#{filename}")
|
27
|
-
end
|
28
|
-
|
29
|
-
request = Net::HTTP::Put.new(uri, {
|
30
|
-
"Content-Length": File.size(path).to_s,
|
31
|
-
"Content-Type": content_type,
|
32
|
-
"Date": date,
|
33
|
-
"Authorization": "AWS #{access_key}:#{signature}",
|
34
|
-
"x-amz-storage-class": "STANDARD",
|
35
|
-
"x-amz-acl": "private",
|
36
|
-
})
|
37
|
-
request.body_stream = File.open(path)
|
38
|
-
|
39
|
-
Net::HTTP.start(uri.hostname) do |http|
|
40
|
-
response = http.request(request)
|
41
|
-
response.value # raises if not success
|
42
|
-
end
|
43
|
-
end
|
44
|
-
|
45
|
-
private
|
46
|
-
|
47
|
-
attr_accessor :full_s3_path
|
48
|
-
|
49
|
-
def signature
|
50
|
-
digester = OpenSSL::Digest::SHA1.new
|
51
|
-
digest = OpenSSL::HMAC.digest(digester, secret_key, key)
|
52
|
-
Base64.strict_encode64(digest)
|
53
|
-
end
|
54
|
-
|
55
|
-
def key
|
56
|
-
[
|
57
|
-
"PUT",
|
58
|
-
"",
|
59
|
-
content_type,
|
60
|
-
date,
|
61
|
-
acl,
|
62
|
-
storage_type,
|
63
|
-
full_s3_path,
|
64
|
-
].join("\n")
|
65
|
-
end
|
66
|
-
|
67
|
-
def content_type
|
68
|
-
"application/gzip"
|
69
|
-
end
|
70
|
-
|
71
|
-
def date
|
72
|
-
@time.rfc2822
|
73
|
-
end
|
74
|
-
|
75
|
-
def acl
|
76
|
-
"x-amz-acl:private"
|
77
|
-
end
|
78
|
-
|
79
|
-
def storage_type
|
80
|
-
"x-amz-storage-class:STANDARD"
|
81
|
-
end
|
82
|
-
|
83
|
-
def path
|
84
|
-
"/tmp/#{filename}"
|
85
|
-
end
|
86
|
-
|
87
|
-
def filename
|
88
|
-
"#{@time.utc.iso8601}.sql.gz"
|
7
|
+
module Backup
|
8
|
+
def self.call s3_path, access_key:, secret_key:, now: Time.now.utc
|
9
|
+
dumper = Backhoe
|
10
|
+
s3_dir = S3Dir.new(path: s3_path, access_key:, secret_key:)
|
11
|
+
Controller.new(dumper, s3_dir, now).call
|
12
|
+
Deleter.new(s3_dir, now).call
|
89
13
|
end
|
90
14
|
end
|
91
15
|
end
|
16
|
+
|
metadata
CHANGED
@@ -1,15 +1,71 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: bard-backup
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.3.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Micah Geisel
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2024-07-
|
12
|
-
dependencies:
|
11
|
+
date: 2024-07-25 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: backhoe
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - ">="
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: '0'
|
20
|
+
type: :runtime
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - ">="
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: '0'
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: aws-sdk-s3
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - "~>"
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: '1.0'
|
34
|
+
type: :runtime
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - "~>"
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: '1.0'
|
41
|
+
- !ruby/object:Gem::Dependency
|
42
|
+
name: rexml
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
44
|
+
requirements:
|
45
|
+
- - ">="
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: '0'
|
48
|
+
type: :runtime
|
49
|
+
prerelease: false
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - ">="
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: '0'
|
55
|
+
- !ruby/object:Gem::Dependency
|
56
|
+
name: activesupport
|
57
|
+
requirement: !ruby/object:Gem::Requirement
|
58
|
+
requirements:
|
59
|
+
- - ">="
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: '0'
|
62
|
+
type: :runtime
|
63
|
+
prerelease: false
|
64
|
+
version_requirements: !ruby/object:Gem::Requirement
|
65
|
+
requirements:
|
66
|
+
- - ">="
|
67
|
+
- !ruby/object:Gem::Version
|
68
|
+
version: '0'
|
13
69
|
description:
|
14
70
|
email:
|
15
71
|
- micah@botandrose.com
|
@@ -25,6 +81,10 @@ files:
|
|
25
81
|
- Rakefile
|
26
82
|
- lib/bard-backup.rb
|
27
83
|
- lib/bard/backup.rb
|
84
|
+
- lib/bard/backup/controller.rb
|
85
|
+
- lib/bard/backup/deleter.rb
|
86
|
+
- lib/bard/backup/s3_dir.rb
|
87
|
+
- lib/bard/backup/version.rb
|
28
88
|
- sig/bard/backup.rbs
|
29
89
|
homepage: https://github.com/botandrose/bard-backup
|
30
90
|
licenses:
|