s3_backup 0.0.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.rspec +2 -0
- data/Gemfile +3 -0
- data/README.md +56 -0
- data/Rakefile +5 -0
- data/lib/s3_backup/config.rb +60 -0
- data/lib/s3_backup/pg/backup.rb +57 -0
- data/lib/s3_backup/pg/import.rb +62 -0
- data/lib/s3_backup/pg/obfuscate.rb +97 -0
- data/lib/s3_backup/railtie.rb +12 -0
- data/lib/s3_backup/redis/backup.rb +48 -0
- data/lib/s3_backup/redis/import.rb +72 -0
- data/lib/s3_backup/s3.rb +79 -0
- data/lib/s3_backup/version.rb +3 -0
- data/lib/s3_backup.rb +61 -0
- data/lib/tasks/pg.rake +22 -0
- data/lib/tasks/redis.rake +19 -0
- data/spec/fixtures/obfuscate_default.yml +20 -0
- data/spec/s3_backup/pg/obfuscate_spec.rb +42 -0
- data/spec/spec_helper.rb +13 -0
- metadata +161 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: 8b77cce74b179a0fe9c7945f85f624d4fb324bc2
|
4
|
+
data.tar.gz: acd5fe987b7c72884114d3a3f59f7fd3c4431ed4
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: e65aefb84a61bdc2f227724913495eef7321d978356028b575356d4f4aabc1ca358d7d3efe8bc434aa6a2940c9e65c12444dc38f6fc7d080937d6c3058595ba6
|
7
|
+
data.tar.gz: 9602297c7b55861cbfdfbb2d1a07dfa9f15b1c842b56f0e19fd6c57067236b5844165521edd3baac9f49f955353c43f76dabdfd34ceb21573c738281129f2cd6
|
data/.rspec
ADDED
data/Gemfile
ADDED
data/README.md
ADDED
@@ -0,0 +1,56 @@
|
|
1
|
+
# S3 Backup
|
2
|
+
|
3
|
+
## Postgres
|
4
|
+
|
5
|
+
#### Backup
|
6
|
+
```
|
7
|
+
rake s3_backup:pg:backup[db_staging]
|
8
|
+
```
|
9
|
+
|
10
|
+
#### Import
|
11
|
+
```
|
12
|
+
rake s3_backup:pg:import[db_staging]
|
13
|
+
```
|
14
|
+
|
15
|
+
## Redis
|
16
|
+
|
17
|
+
#### Backup
|
18
|
+
```
|
19
|
+
rake s3_backup:redis:backup
|
20
|
+
```
|
21
|
+
|
22
|
+
#### Import
|
23
|
+
```
|
24
|
+
rake s3_backup:redis:import[staging]
|
25
|
+
```
|
26
|
+
|
27
|
+
## Configuration file
|
28
|
+
|
29
|
+
```yaml
|
30
|
+
---
|
31
|
+
|
32
|
+
pg_database:
|
33
|
+
host: <%= ENV['DATABASE_HOST'] %>
|
34
|
+
user: <%= ENV['DATABASE_USER'] %>
|
35
|
+
password: <%= ENV['DATABASE_PASSWORD'] %>
|
36
|
+
|
37
|
+
redis:
|
38
|
+
dump_path: /var/lib/redis/6379/dump.rdb
|
39
|
+
|
40
|
+
s3:
|
41
|
+
aws_access_key_id: <%= ENV['AWS_ACCESS_KEY_ID'] %>
|
42
|
+
aws_secret_access_key: <%= ENV['AWS_SECRET_ACCESS_KEY'] %>
|
43
|
+
bucket: <%= ENV['S3_BUCKET'] %>
|
44
|
+
aws_region: <%= ENV['AWS_REGION'] %>
|
45
|
+
pg_path: rds_backup
|
46
|
+
redis_path: redis_backup
|
47
|
+
keep: 5
|
48
|
+
|
49
|
+
tables:
|
50
|
+
users:
|
51
|
+
columns:
|
52
|
+
first_name: first_name
|
53
|
+
last_name: last_name
|
54
|
+
email: email
|
55
|
+
exception: '@mycompany.me'
|
56
|
+
```
|
data/Rakefile
ADDED
@@ -0,0 +1,60 @@
|
|
1
|
+
require 'pry'
|
2
|
+
require 'erb'
|
3
|
+
require 'yaml'
|
4
|
+
|
5
|
+
module S3Backup
|
6
|
+
class Config
|
7
|
+
class << self
|
8
|
+
|
9
|
+
attr_accessor :pg_host
|
10
|
+
attr_accessor :pg_user
|
11
|
+
attr_accessor :pg_password
|
12
|
+
attr_accessor :redis_dump_path
|
13
|
+
attr_accessor :aws_access_key_id
|
14
|
+
attr_accessor :aws_secret_access_key
|
15
|
+
attr_accessor :bucket
|
16
|
+
attr_accessor :aws_region
|
17
|
+
attr_accessor :s3_pg_path
|
18
|
+
attr_accessor :s3_redis_path
|
19
|
+
attr_accessor :s3_keep
|
20
|
+
attr_accessor :tables
|
21
|
+
|
22
|
+
def load!(file_path)
|
23
|
+
template = ERB.new File.new(file_path).read
|
24
|
+
@configuration = YAML.load(template.result(binding))
|
25
|
+
|
26
|
+
self.pg_host = config('pg_database', 'host')
|
27
|
+
self.pg_user = config('pg_database', 'user')
|
28
|
+
self.pg_password = config('pg_database', 'password')
|
29
|
+
|
30
|
+
self.redis_dump_path = config('redis', 'dump_path')
|
31
|
+
|
32
|
+
self.aws_access_key_id = config('s3', 'aws_access_key_id')
|
33
|
+
self.aws_secret_access_key = config('s3', 'aws_secret_access_key')
|
34
|
+
self.bucket = config('s3', 'bucket')
|
35
|
+
self.aws_region = config('s3', 'aws_region')
|
36
|
+
self.s3_keep = config('s3', 'keep')
|
37
|
+
self.s3_pg_path = config('s3', 'pg_path')
|
38
|
+
self.s3_redis_path = config('s3', 'redis_path')
|
39
|
+
|
40
|
+
self.tables = config('tables') || {}
|
41
|
+
|
42
|
+
true
|
43
|
+
end
|
44
|
+
|
45
|
+
def requires!(*args)
|
46
|
+
args.each do |argv|
|
47
|
+
raise "Configuration missing: #{argv}" unless Config.send(argv)
|
48
|
+
end
|
49
|
+
true
|
50
|
+
end
|
51
|
+
|
52
|
+
def config(*args)
|
53
|
+
args.inject(@configuration) do |hash, key|
|
54
|
+
(hash.is_a?(Hash) && hash[key]) || nil
|
55
|
+
end
|
56
|
+
end
|
57
|
+
|
58
|
+
end
|
59
|
+
end
|
60
|
+
end
|
@@ -0,0 +1,57 @@
|
|
1
|
+
module S3Backup
|
2
|
+
module Pg
|
3
|
+
class Backup
|
4
|
+
attr_reader :db_name
|
5
|
+
|
6
|
+
def initialize(db_name)
|
7
|
+
@db_name = db_name
|
8
|
+
end
|
9
|
+
|
10
|
+
def now!
|
11
|
+
puts 'Setup environement'
|
12
|
+
set_pg_password_env
|
13
|
+
puts 'Starting downloading dump ...'
|
14
|
+
dump_database
|
15
|
+
puts 'Dump downloaded.'
|
16
|
+
puts 'Starting obfuscation ...'
|
17
|
+
Obfuscate.new(pg_dump_file.path, obfuscated_file.path).obfuscate_dump!
|
18
|
+
puts 'Obfuscation done.'
|
19
|
+
puts 'Upload to S3 ...'
|
20
|
+
S3Backup::S3.new.upload!(obfucated_file_name, Config.s3_pg_path, obfuscated_file.path)
|
21
|
+
puts 'Uploaded.'
|
22
|
+
puts 'Clean environement.'
|
23
|
+
clean_env
|
24
|
+
S3Backup::S3.new.clean!(db_name, Config.s3_pg_path)
|
25
|
+
end
|
26
|
+
|
27
|
+
private
|
28
|
+
|
29
|
+
def set_pg_password_env
|
30
|
+
ENV['PGPASSWORD'] = Config.pg_password
|
31
|
+
end
|
32
|
+
|
33
|
+
def dump_database
|
34
|
+
`pg_dump -h #{Config.pg_host} -U #{Config.pg_user} -d #{db_name} > #{pg_dump_file.path}`
|
35
|
+
end
|
36
|
+
|
37
|
+
def pg_dump_file
|
38
|
+
@pg_dump_file ||= Tempfile.new(db_name)
|
39
|
+
end
|
40
|
+
|
41
|
+
def obfucated_file_name
|
42
|
+
@obfucated_file_name ||= "#{db_name}-#{Time.now.to_i}.gz"
|
43
|
+
end
|
44
|
+
|
45
|
+
def obfuscated_file
|
46
|
+
@obfuscated_file ||= Tempfile.new(obfucated_file_name)
|
47
|
+
end
|
48
|
+
|
49
|
+
def clean_env
|
50
|
+
pg_dump_file.unlink
|
51
|
+
obfuscated_file.unlink
|
52
|
+
ENV['PGPASSWORD'] = ''
|
53
|
+
end
|
54
|
+
|
55
|
+
end
|
56
|
+
end
|
57
|
+
end
|
@@ -0,0 +1,62 @@
|
|
1
|
+
require 'zlib'
|
2
|
+
|
3
|
+
module S3Backup
|
4
|
+
module Pg
|
5
|
+
class Import
|
6
|
+
attr_reader :pg_database_name, :database
|
7
|
+
|
8
|
+
def initialize(pg_database_name)
|
9
|
+
@pg_database_name = pg_database_name
|
10
|
+
|
11
|
+
config = Rails.configuration.database_configuration
|
12
|
+
@database = config[Rails.env]['database']
|
13
|
+
end
|
14
|
+
|
15
|
+
def now!
|
16
|
+
puts 'Setup local database ...'
|
17
|
+
setup_local_database
|
18
|
+
puts 'Downloading pg database ...'
|
19
|
+
S3Backup::S3.new.download!(pg_database_name, Config.s3_pg_path, pg_dump_s3_file.path)
|
20
|
+
umcompress_file
|
21
|
+
puts "Loading data in #{database} ..."
|
22
|
+
load_file
|
23
|
+
clean_env
|
24
|
+
puts '🍺 Done!'
|
25
|
+
end
|
26
|
+
|
27
|
+
private
|
28
|
+
|
29
|
+
def pg_dump_s3_file
|
30
|
+
@pg_dump_s3_file ||= Tempfile.new(pg_database_name + '_compressed')
|
31
|
+
end
|
32
|
+
|
33
|
+
def pg_dump_file
|
34
|
+
@pg_dump_file ||= Tempfile.new(pg_database_name)
|
35
|
+
end
|
36
|
+
|
37
|
+
def umcompress_file
|
38
|
+
file = File.open(pg_dump_file.path, 'w')
|
39
|
+
|
40
|
+
Zlib::GzipReader.open(pg_dump_s3_file.path) do |gz|
|
41
|
+
file.write gz.read
|
42
|
+
end
|
43
|
+
file.close
|
44
|
+
end
|
45
|
+
|
46
|
+
def load_file
|
47
|
+
`psql -d #{database} -f #{pg_dump_file.path} 2> /dev/null`
|
48
|
+
end
|
49
|
+
|
50
|
+
def setup_local_database
|
51
|
+
Rake::Task['db:drop'].invoke
|
52
|
+
Rake::Task['db:create'].invoke
|
53
|
+
end
|
54
|
+
|
55
|
+
def clean_env
|
56
|
+
pg_dump_file.unlink
|
57
|
+
pg_dump_s3_file.unlink
|
58
|
+
end
|
59
|
+
|
60
|
+
end
|
61
|
+
end
|
62
|
+
end
|
@@ -0,0 +1,97 @@
|
|
1
|
+
require 'zlib'
|
2
|
+
require 'faker'
|
3
|
+
|
4
|
+
module S3Backup
|
5
|
+
module Pg
|
6
|
+
class Obfuscate
|
7
|
+
|
8
|
+
LINE_SEPARATOR = "\t".freeze
|
9
|
+
END_OF_STATEMENT = '\.'.freeze
|
10
|
+
|
11
|
+
attr_reader :dump_file, :file_path
|
12
|
+
|
13
|
+
def initialize(dump_file, file_path)
|
14
|
+
@dump_file = dump_file
|
15
|
+
@file_path = file_path
|
16
|
+
end
|
17
|
+
|
18
|
+
def obfuscate_dump!
|
19
|
+
file = Zlib::GzipWriter.open(file_path)
|
20
|
+
|
21
|
+
File.open(dump_file).each do |line|
|
22
|
+
file.write(obfuscate_line(line.chomp) + "\n")
|
23
|
+
end
|
24
|
+
file.close
|
25
|
+
end
|
26
|
+
|
27
|
+
private
|
28
|
+
|
29
|
+
def obfuscate_line(line)
|
30
|
+
if line == END_OF_STATEMENT
|
31
|
+
@inside_copy_statement = false
|
32
|
+
elsif table_to_obfuscate(line)
|
33
|
+
pepare_statement_line_to_replace(line)
|
34
|
+
elsif @inside_copy_statement && !line.include?(@exception)
|
35
|
+
line_split = line.split(LINE_SEPARATOR)
|
36
|
+
|
37
|
+
@replacements.each do |position, type|
|
38
|
+
line_split[position] = replace_value(type)
|
39
|
+
end
|
40
|
+
|
41
|
+
return line_split.join(LINE_SEPARATOR)
|
42
|
+
end
|
43
|
+
|
44
|
+
line
|
45
|
+
end
|
46
|
+
|
47
|
+
# INFO:
|
48
|
+
# Test if the line start with COPY
|
49
|
+
# return false if the table is not in the configration file
|
50
|
+
# return the name of the table
|
51
|
+
#
|
52
|
+
def table_to_obfuscate(line)
|
53
|
+
return false if line[0..3] != 'COPY'
|
54
|
+
|
55
|
+
Config.tables.keys.each do |table_name|
|
56
|
+
return table_name if line.include?("COPY #{table_name} (")
|
57
|
+
end
|
58
|
+
|
59
|
+
false
|
60
|
+
end
|
61
|
+
|
62
|
+
# INFO:
|
63
|
+
# Pepare variables for the next insert lines
|
64
|
+
#
|
65
|
+
def pepare_statement_line_to_replace(line)
|
66
|
+
@table_name = table_to_obfuscate(line)
|
67
|
+
column_names = line[/\(([^)]+)\)/].tr('(),', '').split
|
68
|
+
@replacements = {}
|
69
|
+
|
70
|
+
column_names.each_with_index do |column_name, index|
|
71
|
+
if Config.tables[@table_name]['columns'].key?(column_name)
|
72
|
+
@replacements[index] = Config.tables[@table_name]['columns'][column_name]
|
73
|
+
@exception = Config.tables[@table_name]['exception']
|
74
|
+
end
|
75
|
+
end
|
76
|
+
|
77
|
+
@inside_copy_statement = true
|
78
|
+
end
|
79
|
+
|
80
|
+
def replace_value(type)
|
81
|
+
case type
|
82
|
+
when 'email'
|
83
|
+
Faker::Internet.email("#{Faker::Name.name}_#{rand(100_000)}")
|
84
|
+
when 'first_name'
|
85
|
+
Faker::Name.first_name
|
86
|
+
when 'last_name'
|
87
|
+
Faker::Name.last_name
|
88
|
+
when 'name'
|
89
|
+
Faker::Name.name
|
90
|
+
else
|
91
|
+
'not defined'
|
92
|
+
end
|
93
|
+
end
|
94
|
+
|
95
|
+
end
|
96
|
+
end
|
97
|
+
end
|
@@ -0,0 +1,48 @@
|
|
1
|
+
require 'zlib'
|
2
|
+
|
3
|
+
module S3Backup
|
4
|
+
module Redis
|
5
|
+
class Backup
|
6
|
+
|
7
|
+
def now!
|
8
|
+
puts 'Compressing dump ...'
|
9
|
+
compress_file
|
10
|
+
puts 'Compressed.'
|
11
|
+
puts 'Upload to S3 ...'
|
12
|
+
S3Backup::S3.new.upload!(compressed_file_name, Config.s3_redis_path, compressed_file.path)
|
13
|
+
puts 'Uploaded.'
|
14
|
+
puts 'Clean environement.'
|
15
|
+
clean_env
|
16
|
+
S3Backup::S3.new.clean!(base_s3_name, Config.s3_redis_path)
|
17
|
+
end
|
18
|
+
|
19
|
+
private
|
20
|
+
|
21
|
+
def compress_file
|
22
|
+
file = Zlib::GzipWriter.open(compressed_file.path)
|
23
|
+
|
24
|
+
File.open(Config.redis_dump_path).each do |line|
|
25
|
+
file.write(line)
|
26
|
+
end
|
27
|
+
file.close
|
28
|
+
end
|
29
|
+
|
30
|
+
def base_s3_name
|
31
|
+
"redis-#{Rails.env}"
|
32
|
+
end
|
33
|
+
|
34
|
+
def compressed_file_name
|
35
|
+
@compressed_file_name ||= "#{base_s3_name}-#{Time.now.to_i}.gz"
|
36
|
+
end
|
37
|
+
|
38
|
+
def compressed_file
|
39
|
+
@compressed_file ||= Tempfile.new(compressed_file_name)
|
40
|
+
end
|
41
|
+
|
42
|
+
def clean_env
|
43
|
+
compressed_file.unlink
|
44
|
+
end
|
45
|
+
|
46
|
+
end
|
47
|
+
end
|
48
|
+
end
|
@@ -0,0 +1,72 @@
|
|
1
|
+
require 'zlib'
|
2
|
+
|
3
|
+
module S3Backup
|
4
|
+
module Redis
|
5
|
+
class Import
|
6
|
+
STOP_REDIS_COMMAND = 'brew services stop redis'.freeze
|
7
|
+
START_REDIS_COMMAND = 'brew services start redis'.freeze
|
8
|
+
|
9
|
+
attr_reader :redis_evironement, :redis_s3_file_name, :redis_dump_file_path
|
10
|
+
|
11
|
+
def initialize(redis_evironement)
|
12
|
+
@redis_evironement = redis_evironement
|
13
|
+
@redis_s3_file_name = "redis-#{redis_evironement}"
|
14
|
+
@redis_dump_file_path = '/usr/local/var/db/redis/dump.rdb'
|
15
|
+
end
|
16
|
+
|
17
|
+
def now!
|
18
|
+
puts 'Stop redis database ...'
|
19
|
+
stop_redis_database
|
20
|
+
puts 'Downloading redis database ...'
|
21
|
+
S3Backup::S3.new.download!(redis_s3_file_name, Config.s3_redis_path, redis_dump_s3_file.path)
|
22
|
+
umcompress_file
|
23
|
+
copy_file
|
24
|
+
puts 'Start redis database ...'
|
25
|
+
start_redis_database
|
26
|
+
clean_env
|
27
|
+
puts '🍺 Done!'
|
28
|
+
end
|
29
|
+
|
30
|
+
private
|
31
|
+
|
32
|
+
def redis_dump_s3_file
|
33
|
+
@redis_dump_s3_file ||= Tempfile.new("#{redis_s3_file_name}_compressed")
|
34
|
+
end
|
35
|
+
|
36
|
+
def redis_dump_file
|
37
|
+
@redis_dump_file ||= Tempfile.new(redis_s3_file_name)
|
38
|
+
end
|
39
|
+
|
40
|
+
def umcompress_file
|
41
|
+
file = File.open(redis_dump_file.path, 'w')
|
42
|
+
|
43
|
+
Zlib::GzipReader.open(redis_dump_s3_file.path) do |gz|
|
44
|
+
file.write gz.read
|
45
|
+
end
|
46
|
+
file.close
|
47
|
+
end
|
48
|
+
|
49
|
+
def copy_file
|
50
|
+
`mv #{redis_dump_file.path} #{redis_dump_file_path}`
|
51
|
+
end
|
52
|
+
|
53
|
+
def stop_redis_database
|
54
|
+
Bundler.with_clean_env do
|
55
|
+
`#{STOP_REDIS_COMMAND} 2> /dev/null`
|
56
|
+
end
|
57
|
+
end
|
58
|
+
|
59
|
+
def start_redis_database
|
60
|
+
Bundler.with_clean_env do
|
61
|
+
`#{START_REDIS_COMMAND} 2> /dev/null`
|
62
|
+
end
|
63
|
+
end
|
64
|
+
|
65
|
+
def clean_env
|
66
|
+
redis_dump_file.unlink
|
67
|
+
redis_dump_s3_file.unlink
|
68
|
+
end
|
69
|
+
|
70
|
+
end
|
71
|
+
end
|
72
|
+
end
|
data/lib/s3_backup/s3.rb
ADDED
@@ -0,0 +1,79 @@
|
|
1
|
+
require 'fog'
|
2
|
+
require 'zlib'
|
3
|
+
require 'ruby-progressbar'
|
4
|
+
|
5
|
+
module S3Backup
|
6
|
+
class S3
|
7
|
+
|
8
|
+
attr_reader :connection
|
9
|
+
|
10
|
+
def initialize
|
11
|
+
@connection = Fog::Storage.new(
|
12
|
+
provider: 'AWS',
|
13
|
+
aws_access_key_id: Config.aws_access_key_id,
|
14
|
+
aws_secret_access_key: Config.aws_secret_access_key,
|
15
|
+
region: Config.aws_region
|
16
|
+
)
|
17
|
+
end
|
18
|
+
|
19
|
+
def upload!(file_name, bucket_path, file_path)
|
20
|
+
directory = @connection.directories.get(Config.bucket)
|
21
|
+
|
22
|
+
directory.files.create(
|
23
|
+
key: File.join(bucket_path, file_name),
|
24
|
+
body: File.open(file_path),
|
25
|
+
public: false
|
26
|
+
)
|
27
|
+
|
28
|
+
true
|
29
|
+
end
|
30
|
+
|
31
|
+
def download!(database_name, bucket_path, file_path)
|
32
|
+
progress_bar
|
33
|
+
prefix = File.join(bucket_path, database_name)
|
34
|
+
directory = connection.directories.get(Config.bucket, prefix: prefix)
|
35
|
+
|
36
|
+
s3_backup_file = directory.files.sort_by(&:last_modified).reverse.first
|
37
|
+
|
38
|
+
raise "#{database_name} file not found on s3" unless s3_backup_file
|
39
|
+
|
40
|
+
file = File.open(file_path, 'wb')
|
41
|
+
directory.files.get(s3_backup_file.key) do |chunk, remaining_bytes, total_bytes|
|
42
|
+
update_progress_bar(total_bytes, remaining_bytes)
|
43
|
+
file.write chunk
|
44
|
+
end
|
45
|
+
file.close
|
46
|
+
|
47
|
+
true
|
48
|
+
end
|
49
|
+
|
50
|
+
def clean!(base_name, bucket_path)
|
51
|
+
prefix = File.join(bucket_path, base_name)
|
52
|
+
directory = connection.directories.get(Config.bucket, prefix: prefix)
|
53
|
+
|
54
|
+
s3_files = directory.files.sort_by(&:last_modified).reverse
|
55
|
+
files_to_remove = s3_files[Config.s3_keep..-1]
|
56
|
+
|
57
|
+
return true if files_to_remove.blank?
|
58
|
+
|
59
|
+
files_to_remove.each(&:destroy)
|
60
|
+
|
61
|
+
true
|
62
|
+
end
|
63
|
+
|
64
|
+
private
|
65
|
+
|
66
|
+
def progress_bar
|
67
|
+
@progress_bar ||= ProgressBar.create(
|
68
|
+
format: "%a %b\u{15E7}%i %p%% %t",
|
69
|
+
progress_mark: ' ',
|
70
|
+
remainder_mark: "\u{FF65}"
|
71
|
+
)
|
72
|
+
end
|
73
|
+
|
74
|
+
def update_progress_bar(total, remaining)
|
75
|
+
progress_bar.progress = (((total - remaining) * 100) / total).to_i
|
76
|
+
end
|
77
|
+
|
78
|
+
end
|
79
|
+
end
|
data/lib/s3_backup.rb
ADDED
@@ -0,0 +1,61 @@
|
|
1
|
+
require 's3_backup/config'
|
2
|
+
require 's3_backup/s3'
|
3
|
+
|
4
|
+
require 's3_backup/pg/obfuscate'
|
5
|
+
require 's3_backup/pg/backup'
|
6
|
+
require 's3_backup/pg/import'
|
7
|
+
|
8
|
+
require 's3_backup/redis/backup'
|
9
|
+
require 's3_backup/redis/import'
|
10
|
+
|
11
|
+
require 's3_backup/railtie' if defined?(Rails)
|
12
|
+
|
13
|
+
module S3Backup
|
14
|
+
class << self
|
15
|
+
|
16
|
+
def pg_backup!(database_name)
|
17
|
+
require_s3_params
|
18
|
+
Config.requires!(:pg_host, :pg_user, :s3_pg_path, :tables)
|
19
|
+
|
20
|
+
backup = Pg::Backup.new(database_name)
|
21
|
+
backup.now!
|
22
|
+
end
|
23
|
+
|
24
|
+
def pg_import!(pg_database_name)
|
25
|
+
raise 'Need to be run in a rails project' unless defined?(Rails)
|
26
|
+
|
27
|
+
require_s3_params
|
28
|
+
Config.requires!(:s3_pg_path)
|
29
|
+
|
30
|
+
import = Pg::Import.new(pg_database_name)
|
31
|
+
import.now!
|
32
|
+
end
|
33
|
+
|
34
|
+
def redis_backup!
|
35
|
+
require_s3_params
|
36
|
+
Config.requires!(:redis_dump_path, :s3_redis_path)
|
37
|
+
|
38
|
+
backup = Redis::Backup.new
|
39
|
+
backup.now!
|
40
|
+
end
|
41
|
+
|
42
|
+
def redis_import!(redis_evironement)
|
43
|
+
raise 'Import only work with redis installed by brew' if Dir['/usr/local/Cellar/redis/*'].blank?
|
44
|
+
|
45
|
+
require_s3_params
|
46
|
+
Config.requires!(:s3_redis_path)
|
47
|
+
|
48
|
+
import = Redis::Import.new(redis_evironement)
|
49
|
+
import.now!
|
50
|
+
end
|
51
|
+
|
52
|
+
private
|
53
|
+
|
54
|
+
def require_s3_params
|
55
|
+
Config.load!('config/s3_backup_obfuscate.yml')
|
56
|
+
|
57
|
+
Config.requires!(:aws_access_key_id, :aws_secret_access_key, :bucket, :aws_region, :s3_keep)
|
58
|
+
end
|
59
|
+
|
60
|
+
end
|
61
|
+
end
|
data/lib/tasks/pg.rake
ADDED
@@ -0,0 +1,22 @@
|
|
1
|
+
require 'pry'
|
2
|
+
require 's3_backup'
|
3
|
+
|
4
|
+
namespace :s3_backup do
|
5
|
+
namespace :pg do
|
6
|
+
|
7
|
+
desc 'Backup, obfuscate & compress Postgres database to s3'
|
8
|
+
task :backup, [:database] do |_task, args|
|
9
|
+
raise 'You need to specify a database' unless args[:database]
|
10
|
+
|
11
|
+
S3Backup.pg_backup! args[:database]
|
12
|
+
end
|
13
|
+
|
14
|
+
desc 'Import Postgres database from s3'
|
15
|
+
task :import, [:pg_database] do |_task, args|
|
16
|
+
raise 'You need to specify a pg database' unless args[:pg_database]
|
17
|
+
|
18
|
+
S3Backup.pg_import! args[:pg_database]
|
19
|
+
end
|
20
|
+
|
21
|
+
end
|
22
|
+
end
|
@@ -0,0 +1,19 @@
|
|
1
|
+
require 's3_backup'
|
2
|
+
|
3
|
+
namespace :s3_backup do
|
4
|
+
namespace :redis do
|
5
|
+
|
6
|
+
desc 'Backup & compress Redis database to s3'
|
7
|
+
task :backup do |_task, _args|
|
8
|
+
S3Backup.redis_backup!
|
9
|
+
end
|
10
|
+
|
11
|
+
desc 'Import Redis database from s3'
|
12
|
+
task :import, [:redis_environement] do |_task, args|
|
13
|
+
raise 'You need to specify a redis environement' unless args[:redis_environement]
|
14
|
+
|
15
|
+
S3Backup.redis_import! args[:redis_environement]
|
16
|
+
end
|
17
|
+
|
18
|
+
end
|
19
|
+
end
|
@@ -0,0 +1,20 @@
|
|
1
|
+
---
|
2
|
+
|
3
|
+
pg_database:
|
4
|
+
host: my_host
|
5
|
+
user: my_user
|
6
|
+
password: pss
|
7
|
+
|
8
|
+
s3:
|
9
|
+
aws_access_key_id: XXX
|
10
|
+
aws_secret_access_key: DSLK
|
11
|
+
bucket: my_bucket
|
12
|
+
aws_region: p-southeast-2
|
13
|
+
pg_path: pg_backup
|
14
|
+
|
15
|
+
tables:
|
16
|
+
users:
|
17
|
+
columns:
|
18
|
+
name: name
|
19
|
+
email: email
|
20
|
+
exception: '@mycompany.me'
|
@@ -0,0 +1,42 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
describe S3Backup::Pg::Obfuscate do
|
4
|
+
let(:configuration_file) { 'spec/fixtures/obfuscate_default.yml' }
|
5
|
+
|
6
|
+
before { S3Backup::Config.load!(configuration_file) }
|
7
|
+
|
8
|
+
subject(:obfuscate) { S3Backup::Pg::Obfuscate.new(nil, nil) }
|
9
|
+
|
10
|
+
describe 'private#obfuscate_line' do
|
11
|
+
context 'parse line' do
|
12
|
+
let(:table_header) { "COPY users (id, name, email, created_at, updated_at) FROM stdin;" }
|
13
|
+
let(:line_table) { "42\tTom Test\ttom@test.me\t2016-06-15 04:23:00.121\t2016-06-23 07:08:02.08" }
|
14
|
+
|
15
|
+
before { obfuscate.send(:obfuscate_line, table_header) }
|
16
|
+
|
17
|
+
it { expect(obfuscate.send(:obfuscate_line, line_table)).to_not include('tom@test.me') }
|
18
|
+
it { expect(obfuscate.send(:obfuscate_line, line_table)).to_not include('Tom Test') }
|
19
|
+
end
|
20
|
+
|
21
|
+
context 'parse with exception line' do
|
22
|
+
let(:table_header) { "COPY users (id, name, email, created_at, updated_at) FROM stdin;" }
|
23
|
+
let(:line_table) { "42\tTom Test\ttom@mycompany.me\t2016-06-15 04:23:00.121\t2016-06-23 07:08:02.08" }
|
24
|
+
|
25
|
+
before { obfuscate.send(:obfuscate_line, table_header) }
|
26
|
+
|
27
|
+
it { expect(obfuscate.send(:obfuscate_line, line_table)).to include('tom@mycompany.me') }
|
28
|
+
it { expect(obfuscate.send(:obfuscate_line, line_table)).to include('Tom Test') }
|
29
|
+
end
|
30
|
+
|
31
|
+
context 'if table is not difined' do
|
32
|
+
let(:table_header) { "COPY accounts (id, name, email, created_at, updated_at) FROM stdin;" }
|
33
|
+
let(:line_table) { "42\tTom Test\ttom@test.me\t2016-06-15 04:23:00.121\t2016-06-23 07:08:02.08" }
|
34
|
+
|
35
|
+
before { obfuscate.send(:obfuscate_line, table_header) }
|
36
|
+
|
37
|
+
it { expect(obfuscate.send(:obfuscate_line, line_table)).to include('tom@test.me') }
|
38
|
+
it { expect(obfuscate.send(:obfuscate_line, line_table)).to include('Tom Test') }
|
39
|
+
end
|
40
|
+
end
|
41
|
+
|
42
|
+
end
|
data/spec/spec_helper.rb
ADDED
@@ -0,0 +1,13 @@
|
|
1
|
+
require 'rubygems'
|
2
|
+
require 'bundler/setup'
|
3
|
+
require 's3_backup'
|
4
|
+
require 'rspec'
|
5
|
+
|
6
|
+
Dir["#{File.dirname(__FILE__)}/support/**/*.rb"].each { |f| require f }
|
7
|
+
|
8
|
+
RSpec.configure do |config|
|
9
|
+
config.expect_with :rspec do |c|
|
10
|
+
c.syntax = :expect
|
11
|
+
end
|
12
|
+
config.mock_with :rspec
|
13
|
+
end
|
metadata
ADDED
@@ -0,0 +1,161 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: s3_backup
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.0.2
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Tom Floc'h
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
date: 2017-10-03 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: fog
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - "~>"
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: '1.41'
|
20
|
+
type: :runtime
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - "~>"
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: '1.41'
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: faker
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - "~>"
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: '1.8'
|
34
|
+
type: :runtime
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - "~>"
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: '1.8'
|
41
|
+
- !ruby/object:Gem::Dependency
|
42
|
+
name: ruby-progressbar
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
44
|
+
requirements:
|
45
|
+
- - "~>"
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: '1.8'
|
48
|
+
type: :runtime
|
49
|
+
prerelease: false
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - "~>"
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: '1.8'
|
55
|
+
- !ruby/object:Gem::Dependency
|
56
|
+
name: mime-types
|
57
|
+
requirement: !ruby/object:Gem::Requirement
|
58
|
+
requirements:
|
59
|
+
- - "~>"
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: '3.1'
|
62
|
+
type: :runtime
|
63
|
+
prerelease: false
|
64
|
+
version_requirements: !ruby/object:Gem::Requirement
|
65
|
+
requirements:
|
66
|
+
- - "~>"
|
67
|
+
- !ruby/object:Gem::Version
|
68
|
+
version: '3.1'
|
69
|
+
- !ruby/object:Gem::Dependency
|
70
|
+
name: pry
|
71
|
+
requirement: !ruby/object:Gem::Requirement
|
72
|
+
requirements:
|
73
|
+
- - "~>"
|
74
|
+
- !ruby/object:Gem::Version
|
75
|
+
version: '0.10'
|
76
|
+
type: :development
|
77
|
+
prerelease: false
|
78
|
+
version_requirements: !ruby/object:Gem::Requirement
|
79
|
+
requirements:
|
80
|
+
- - "~>"
|
81
|
+
- !ruby/object:Gem::Version
|
82
|
+
version: '0.10'
|
83
|
+
- !ruby/object:Gem::Dependency
|
84
|
+
name: rspec
|
85
|
+
requirement: !ruby/object:Gem::Requirement
|
86
|
+
requirements:
|
87
|
+
- - "~>"
|
88
|
+
- !ruby/object:Gem::Version
|
89
|
+
version: '3.5'
|
90
|
+
type: :development
|
91
|
+
prerelease: false
|
92
|
+
version_requirements: !ruby/object:Gem::Requirement
|
93
|
+
requirements:
|
94
|
+
- - "~>"
|
95
|
+
- !ruby/object:Gem::Version
|
96
|
+
version: '3.5'
|
97
|
+
- !ruby/object:Gem::Dependency
|
98
|
+
name: rubocop
|
99
|
+
requirement: !ruby/object:Gem::Requirement
|
100
|
+
requirements:
|
101
|
+
- - "~>"
|
102
|
+
- !ruby/object:Gem::Version
|
103
|
+
version: '0.49'
|
104
|
+
type: :development
|
105
|
+
prerelease: false
|
106
|
+
version_requirements: !ruby/object:Gem::Requirement
|
107
|
+
requirements:
|
108
|
+
- - "~>"
|
109
|
+
- !ruby/object:Gem::Version
|
110
|
+
version: '0.49'
|
111
|
+
description: Postgres, redis backup and restore
|
112
|
+
email:
|
113
|
+
- thomas.floch@gmail.com
|
114
|
+
executables: []
|
115
|
+
extensions: []
|
116
|
+
extra_rdoc_files: []
|
117
|
+
files:
|
118
|
+
- ".rspec"
|
119
|
+
- Gemfile
|
120
|
+
- README.md
|
121
|
+
- Rakefile
|
122
|
+
- lib/s3_backup.rb
|
123
|
+
- lib/s3_backup/config.rb
|
124
|
+
- lib/s3_backup/pg/backup.rb
|
125
|
+
- lib/s3_backup/pg/import.rb
|
126
|
+
- lib/s3_backup/pg/obfuscate.rb
|
127
|
+
- lib/s3_backup/railtie.rb
|
128
|
+
- lib/s3_backup/redis/backup.rb
|
129
|
+
- lib/s3_backup/redis/import.rb
|
130
|
+
- lib/s3_backup/s3.rb
|
131
|
+
- lib/s3_backup/version.rb
|
132
|
+
- lib/tasks/pg.rake
|
133
|
+
- lib/tasks/redis.rake
|
134
|
+
- spec/fixtures/obfuscate_default.yml
|
135
|
+
- spec/s3_backup/pg/obfuscate_spec.rb
|
136
|
+
- spec/spec_helper.rb
|
137
|
+
homepage: https://github.com/arkes/s3_backup
|
138
|
+
licenses:
|
139
|
+
- MIT
|
140
|
+
metadata: {}
|
141
|
+
post_install_message:
|
142
|
+
rdoc_options: []
|
143
|
+
require_paths:
|
144
|
+
- lib
|
145
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
146
|
+
requirements:
|
147
|
+
- - ">="
|
148
|
+
- !ruby/object:Gem::Version
|
149
|
+
version: '0'
|
150
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
151
|
+
requirements:
|
152
|
+
- - ">="
|
153
|
+
- !ruby/object:Gem::Version
|
154
|
+
version: '0'
|
155
|
+
requirements: []
|
156
|
+
rubyforge_project:
|
157
|
+
rubygems_version: 2.4.5
|
158
|
+
signing_key:
|
159
|
+
specification_version: 4
|
160
|
+
summary: Postgres, redis backup and restore
|
161
|
+
test_files: []
|