webtranslateit-safe 0.4.6 → 0.4.8
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG +9 -0
- data/README.markdown +9 -52
- data/bin/webtranslateit-safe +1 -1
- data/lib/web_translate_it/safe/config/builder.rb +3 -3
- data/lib/web_translate_it/safe/sftp.rb +21 -14
- data/lib/web_translate_it/safe.rb +2 -6
- metadata +4 -7
- data/lib/web_translate_it/safe/ftp.rb +0 -87
- data/lib/web_translate_it/safe/scp.rb +0 -109
- data/lib/web_translate_it/safe/svndump.rb +0 -17
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 622f6f352864fc8dc4b3938346a25a46a47874e3b439938ed3d9b177218f2564
|
4
|
+
data.tar.gz: fa08df6cdc398f48a2bb5f8c7fd2289bf50b9cf6c3f6aaa39b542f8143363357
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 458c754bea2197d46e29e9405705af3961a03b1fec4f2f72e994c022c5549412ff4f1baa00cbd868d36df07a0d94a2cda7007b6fb447b739ed36df0efefd63e5
|
7
|
+
data.tar.gz: e70216d1c71e82b4d980a264e50f411b1c392a9d0f26f60605e71a8dc633199a4e195bed14786e858ff84aa415e90a69204f45869c8ea1a7850583b781591349
|
data/CHANGELOG
CHANGED
data/README.markdown
CHANGED
@@ -1,39 +1,16 @@
|
|
1
1
|
# webtranslateit-safe
|
2
2
|
|
3
|
-
Simple database and filesystem backups with S3 and Rackspace Cloud Files support (with optional encryption)
|
3
|
+
Simple database and filesystem backups with S3 and Rackspace Cloud Files support (with optional encryption).
|
4
4
|
|
5
|
-
|
5
|
+
This is a fork of now abandonned [astrails-safe](https://github.com/astrails/safe) that we’ve been using since 2010. It is now unmaintained and not compatible with ruby 3.2, so we forked it.
|
6
6
|
|
7
|
-
|
7
|
+
We’ve added:
|
8
8
|
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
* support for simple mysqldump of mysql databases
|
15
|
-
* support for symmetric or public key encryption
|
16
|
-
* support for local filesystem, Amazon S3, and Rackspace Cloud Files for storage
|
17
|
-
* support for backup rotation. we don’t want backups filling all the diskspace or cost a fortune on S3 or Cloud Files
|
18
|
-
|
19
|
-
And since we didn't find any, we wrote our own :)
|
20
|
-
|
21
|
-
## Contributions
|
22
|
-
|
23
|
-
The following functionality was contributed by webtranslateit-safe users:
|
24
|
-
|
25
|
-
* PostgreSQL dump using `pg_dump` (by Mark Mansour <mark@stateofflux.com>)
|
26
|
-
* Subversion dump using svndump (by Richard Luther <richard.luther@gmail.com>)
|
27
|
-
* SFTP remote storage (by Adam <adam@mediadrive.ca>)
|
28
|
-
* benchmarking output (By Neer)
|
29
|
-
* README fixes (by Bobby Wilson)
|
30
|
-
* improved config file parsing (by Fedor Kocherga <fkocherga@gmail.com>)
|
31
|
-
* mysql password file quoting (by Jonathan Sutherland <jonathan.sutherland@gmail.com>)
|
32
|
-
* Rackspace Cloud Files support (by H. Wade Minter <minter@lunenburg.org>)
|
33
|
-
* Plan FTP support (by seroy <seroy@bk.ru>)
|
34
|
-
* mongodump support (by Matt Berther <matt@mattberther.com>)
|
35
|
-
|
36
|
-
Thanks to all :)
|
9
|
+
- Support for ruby 3.2
|
10
|
+
- Standardized code with rubocop
|
11
|
+
- Added support for SCP transfers — On some conditions with servers with higher latency, transfering large files (> 18GB) with SFTP can take a very long time.
|
12
|
+
- Removed svndump feature
|
13
|
+
- Removed FTP transfer feature
|
37
14
|
|
38
15
|
## Installation
|
39
16
|
|
@@ -157,13 +134,7 @@ safe do
|
|
157
134
|
user "astrails"
|
158
135
|
# port 8023
|
159
136
|
password "ssh password for sftp"
|
160
|
-
|
161
|
-
|
162
|
-
sftp do
|
163
|
-
host "sftp.astrails.com"
|
164
|
-
user "astrails"
|
165
|
-
# port 8023
|
166
|
-
password "ssh password for scp"
|
137
|
+
use_scp: true # use SCP if possible
|
167
138
|
end
|
168
139
|
|
169
140
|
gpg do
|
@@ -201,12 +172,6 @@ safe do
|
|
201
172
|
|
202
173
|
end
|
203
174
|
|
204
|
-
svndump do
|
205
|
-
repo :my_repo do
|
206
|
-
repo_path "/home/svn/my_repo"
|
207
|
-
end
|
208
|
-
end
|
209
|
-
|
210
175
|
pgdump do
|
211
176
|
options "-i -x -O" # -i => ignore version, -x => do not dump privileges (grant/revoke), -O => skip restoration of object ownership in plain text format
|
212
177
|
|
@@ -237,14 +202,6 @@ safe do
|
|
237
202
|
end
|
238
203
|
```
|
239
204
|
|
240
|
-
## Contributing
|
241
|
-
|
242
|
-
1. Fork it
|
243
|
-
2. Create your feature branch (`git checkout -b my-new-feature`)
|
244
|
-
3. Commit your changes (`git commit -am 'Add some feature'`)
|
245
|
-
4. Push to the branch (`git push origin my-new-feature`)
|
246
|
-
5. Create new Pull Request
|
247
|
-
|
248
205
|
## Copyright
|
249
206
|
|
250
207
|
Copyright (c) 2010-2023 WebTranslateIt Software SL. See LICENSE.txt for details.
|
data/bin/webtranslateit-safe
CHANGED
@@ -74,10 +74,10 @@ module WebTranslateIt
|
|
74
74
|
|
75
75
|
simple_value :verbose, :dry_run, :local_only, :path, :command,
|
76
76
|
:options, :user, :host, :port, :password, :key, :secret, :bucket,
|
77
|
-
:api_key, :container, :socket, :service_net, :repo_path
|
77
|
+
:api_key, :container, :socket, :service_net, :repo_path, :use_scp
|
78
78
|
multi_value :skip_tables, :exclude, :files
|
79
|
-
mixed_value :s3, :local, :cloudfiles, :sftp, :mysqldump, :tar, :gpg, :keep, :pgdump, :tar,
|
80
|
-
:
|
79
|
+
mixed_value :s3, :local, :cloudfiles, :sftp, :mysqldump, :tar, :gpg, :keep, :pgdump, :tar,
|
80
|
+
:mongodump
|
81
81
|
collection :database, :archive, :repo
|
82
82
|
|
83
83
|
private
|
@@ -19,7 +19,7 @@ module WebTranslateIt
|
|
19
19
|
def save # rubocop:todo Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity, Metrics/MethodLength
|
20
20
|
raise 'pipe-streaming not supported for SFTP.' unless @backup.path
|
21
21
|
|
22
|
-
puts "Uploading #{host}:#{full_path} via SFTP" if verbose? || dry_run?
|
22
|
+
puts "Uploading #{host}:#{full_path} via #{use_scp? ? 'SCP' : 'SFTP'}" if verbose? || dry_run?
|
23
23
|
|
24
24
|
return if dry_run? || local_only?
|
25
25
|
|
@@ -27,10 +27,11 @@ module WebTranslateIt
|
|
27
27
|
opts = {}
|
28
28
|
opts[:password] = password if password
|
29
29
|
opts[:port] = port if port
|
30
|
-
Net::
|
30
|
+
protocol = use_scp? ? Net::SCP : Net::SFTP
|
31
|
+
protocol.start(host, user, opts) do |connexion| # rubocop:todo Metrics/BlockLength
|
31
32
|
puts "Sending #{@backup.path} to #{full_path}" if verbose?
|
32
33
|
begin
|
33
|
-
|
34
|
+
connexion.upload! @backup.path, full_path
|
34
35
|
rescue IO::TimeoutError
|
35
36
|
puts 'Upload timed out, retrying'
|
36
37
|
retries += 1
|
@@ -39,17 +40,19 @@ module WebTranslateIt
|
|
39
40
|
else
|
40
41
|
retry unless retries >= MAX_RETRIES
|
41
42
|
end
|
42
|
-
rescue Net::SFTP::StatusException
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
|
43
|
+
rescue Net::SFTP::StatusException, Net::SCP::Error
|
44
|
+
Net::SFTP.start(host, user, opts) do |sftp|
|
45
|
+
puts "Ensuring remote path (#{path}) exists" if verbose?
|
46
|
+
# mkdir -p
|
47
|
+
folders = path.split('/')
|
48
|
+
folders.each_index do |i|
|
49
|
+
folder = folders[0..i].join('/')
|
50
|
+
puts "Creating #{folder} on remote" if verbose?
|
51
|
+
begin
|
52
|
+
sftp.mkdir!(folder)
|
53
|
+
rescue StandardError
|
54
|
+
Net::SFTP::StatusException
|
55
|
+
end
|
53
56
|
end
|
54
57
|
end
|
55
58
|
retry
|
@@ -100,6 +103,10 @@ module WebTranslateIt
|
|
100
103
|
config[:sftp, :port]
|
101
104
|
end
|
102
105
|
|
106
|
+
def use_scp?
|
107
|
+
config[:sftp, :use_scp] == true
|
108
|
+
end
|
109
|
+
|
103
110
|
end
|
104
111
|
|
105
112
|
end
|
@@ -21,7 +21,6 @@ require 'web_translate_it/safe/source'
|
|
21
21
|
require 'web_translate_it/safe/mysqldump'
|
22
22
|
require 'web_translate_it/safe/pgdump'
|
23
23
|
require 'web_translate_it/safe/archive'
|
24
|
-
require 'web_translate_it/safe/svndump'
|
25
24
|
require 'web_translate_it/safe/mongodump'
|
26
25
|
|
27
26
|
require 'web_translate_it/safe/pipe'
|
@@ -32,9 +31,7 @@ require 'web_translate_it/safe/sink'
|
|
32
31
|
require 'web_translate_it/safe/local'
|
33
32
|
require 'web_translate_it/safe/s3'
|
34
33
|
require 'web_translate_it/safe/cloudfiles'
|
35
|
-
require 'web_translate_it/safe/scp'
|
36
34
|
require 'web_translate_it/safe/sftp'
|
37
|
-
require 'web_translate_it/safe/ftp'
|
38
35
|
|
39
36
|
module WebTranslateIt
|
40
37
|
|
@@ -50,12 +47,11 @@ module WebTranslateIt
|
|
50
47
|
[[Mysqldump, %i[mysqldump databases]],
|
51
48
|
[Pgdump, %i[pgdump databases]],
|
52
49
|
[Mongodump, %i[mongodump databases]],
|
53
|
-
[Archive, %i[tar archives]],
|
54
|
-
[Svndump, %i[svndump repos]]].each do |klass, path|
|
50
|
+
[Archive, %i[tar archives]]].each do |klass, path|
|
55
51
|
next unless collection = config[*path]
|
56
52
|
|
57
53
|
collection.each do |name, c|
|
58
|
-
klass.new(name, c).backup.run(c, :gpg, :gzip, :local, :s3, :cloudfiles, :
|
54
|
+
klass.new(name, c).backup.run(c, :gpg, :gzip, :local, :s3, :cloudfiles, :sftp)
|
59
55
|
end
|
60
56
|
end
|
61
57
|
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: webtranslateit-safe
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.4.
|
4
|
+
version: 0.4.8
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Edouard Briere
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2023-
|
12
|
+
date: 2023-07-05 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: aws-s3
|
@@ -68,9 +68,9 @@ dependencies:
|
|
68
68
|
- !ruby/object:Gem::Version
|
69
69
|
version: '0'
|
70
70
|
description: |
|
71
|
-
WebTranslateIt-Safe is a simple tool to backup databases (MySQL and PostgreSQL)
|
71
|
+
WebTranslateIt-Safe is a simple tool to backup databases (MySQL and PostgreSQL) and just files.
|
72
72
|
Backups can be stored locally or remotely and can be enctypted.
|
73
|
-
Remote storage is supported on Amazon S3, Rackspace Cloud Files, or just plain
|
73
|
+
Remote storage is supported on Amazon S3, Rackspace Cloud Files, or just plain SFTP.
|
74
74
|
email:
|
75
75
|
- support@webtranslateit.com
|
76
76
|
executables:
|
@@ -87,7 +87,6 @@ files:
|
|
87
87
|
- lib/web_translate_it/safe/cloudfiles.rb
|
88
88
|
- lib/web_translate_it/safe/config/builder.rb
|
89
89
|
- lib/web_translate_it/safe/config/node.rb
|
90
|
-
- lib/web_translate_it/safe/ftp.rb
|
91
90
|
- lib/web_translate_it/safe/gpg.rb
|
92
91
|
- lib/web_translate_it/safe/gzip.rb
|
93
92
|
- lib/web_translate_it/safe/local.rb
|
@@ -96,12 +95,10 @@ files:
|
|
96
95
|
- lib/web_translate_it/safe/pgdump.rb
|
97
96
|
- lib/web_translate_it/safe/pipe.rb
|
98
97
|
- lib/web_translate_it/safe/s3.rb
|
99
|
-
- lib/web_translate_it/safe/scp.rb
|
100
98
|
- lib/web_translate_it/safe/sftp.rb
|
101
99
|
- lib/web_translate_it/safe/sink.rb
|
102
100
|
- lib/web_translate_it/safe/source.rb
|
103
101
|
- lib/web_translate_it/safe/stream.rb
|
104
|
-
- lib/web_translate_it/safe/svndump.rb
|
105
102
|
- lib/web_translate_it/safe/tmp_file.rb
|
106
103
|
homepage: http://github.com/webtranslateit/safe
|
107
104
|
licenses:
|
@@ -1,87 +0,0 @@
|
|
1
|
-
module WebTranslateIt
|
2
|
-
|
3
|
-
module Safe
|
4
|
-
|
5
|
-
class Ftp < Sink
|
6
|
-
|
7
|
-
protected
|
8
|
-
|
9
|
-
def active?
|
10
|
-
host && user
|
11
|
-
end
|
12
|
-
|
13
|
-
def path
|
14
|
-
@path ||= expand(config[:ftp, :path] || config[:local, :path] || ':kind/:id')
|
15
|
-
end
|
16
|
-
|
17
|
-
def save # rubocop:todo Metrics/CyclomaticComplexity, Metrics/AbcSize, Metrics/MethodLength, Metrics/PerceivedComplexity
|
18
|
-
raise 'pipe-streaming not supported for FTP.' unless @backup.path
|
19
|
-
|
20
|
-
puts "Uploading #{host}:#{full_path} via FTP" if verbose? || dry_run?
|
21
|
-
|
22
|
-
return if dry_run? || local_only?
|
23
|
-
|
24
|
-
port ||= 21
|
25
|
-
Net::FTP.open(host) do |ftp|
|
26
|
-
ftp.connect(host, port)
|
27
|
-
ftp.login(user, password)
|
28
|
-
puts "Sending #{@backup.path} to #{full_path}" if verbose?
|
29
|
-
begin
|
30
|
-
ftp.put(@backup.path, full_path)
|
31
|
-
rescue Net::FTPPermError
|
32
|
-
puts "Ensuring remote path (#{path}) exists" if verbose?
|
33
|
-
end
|
34
|
-
end
|
35
|
-
puts '...done' if verbose?
|
36
|
-
end
|
37
|
-
|
38
|
-
# rubocop:todo Metrics/PerceivedComplexity
|
39
|
-
def cleanup # rubocop:todo Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity, Metrics/AbcSize, Metrics/MethodLength
|
40
|
-
return if local_only? || dry_run?
|
41
|
-
|
42
|
-
return unless keep = config[:keep, :ftp]
|
43
|
-
|
44
|
-
puts "listing files: #{host}:#{base}*" if verbose?
|
45
|
-
port ||= 21
|
46
|
-
Net::FTP.open(host) do |ftp|
|
47
|
-
ftp.connect(host, port)
|
48
|
-
ftp.login(user, password)
|
49
|
-
files = ftp.nlst(path)
|
50
|
-
pattern = File.basename(base.to_s)
|
51
|
-
files = files.select { |x| x.start_with?(pattern) }
|
52
|
-
puts(files.collect { |x| x }) if verbose?
|
53
|
-
|
54
|
-
files = files
|
55
|
-
.collect { |x| x }
|
56
|
-
.sort
|
57
|
-
|
58
|
-
cleanup_with_limit(files, keep) do |f|
|
59
|
-
file = File.join(path, f)
|
60
|
-
puts "removing ftp file #{host}:#{file}" if dry_run? || verbose?
|
61
|
-
ftp.delete(file) unless dry_run? || local_only?
|
62
|
-
end
|
63
|
-
end
|
64
|
-
end
|
65
|
-
# rubocop:enable Metrics/PerceivedComplexity
|
66
|
-
|
67
|
-
def host
|
68
|
-
config[:ftp, :host]
|
69
|
-
end
|
70
|
-
|
71
|
-
def user
|
72
|
-
config[:ftp, :user]
|
73
|
-
end
|
74
|
-
|
75
|
-
def password
|
76
|
-
config[:ftp, :password]
|
77
|
-
end
|
78
|
-
|
79
|
-
def port
|
80
|
-
config[:ftp, :port]
|
81
|
-
end
|
82
|
-
|
83
|
-
end
|
84
|
-
|
85
|
-
end
|
86
|
-
|
87
|
-
end
|
@@ -1,109 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
module WebTranslateIt
|
4
|
-
|
5
|
-
module Safe
|
6
|
-
|
7
|
-
class Scp < Sink
|
8
|
-
|
9
|
-
MAX_RETRIES = 5
|
10
|
-
|
11
|
-
protected
|
12
|
-
|
13
|
-
def active?
|
14
|
-
host && user
|
15
|
-
end
|
16
|
-
|
17
|
-
def path
|
18
|
-
@path ||= expand(config[:scp, :path] || config[:local, :path] || ':kind/:id')
|
19
|
-
end
|
20
|
-
|
21
|
-
def save # rubocop:todo Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity, Metrics/MethodLength
|
22
|
-
raise 'pipe-streaming not supported for SCP.' unless @backup.path
|
23
|
-
|
24
|
-
puts "Uploading #{host}:#{full_path} via SCP" if verbose? || dry_run?
|
25
|
-
|
26
|
-
return if dry_run? || local_only?
|
27
|
-
|
28
|
-
retries = 0
|
29
|
-
opts = {}
|
30
|
-
opts[:password] = password if password
|
31
|
-
opts[:port] = port if port
|
32
|
-
Net::SCP.start(host, user, opts) do |scp|
|
33
|
-
puts "Sending #{@backup.path} to #{full_path}" if verbose?
|
34
|
-
begin
|
35
|
-
scp.upload! @backup.path, full_path
|
36
|
-
rescue IO::TimeoutError
|
37
|
-
puts 'Upload timed out, retrying'
|
38
|
-
retries += 1
|
39
|
-
if retries >= MAX_RETRIES
|
40
|
-
puts "Tried #{retries} times. Giving up."
|
41
|
-
else
|
42
|
-
retry unless retries >= MAX_RETRIES
|
43
|
-
end
|
44
|
-
rescue Net::SCP::Error
|
45
|
-
puts "Ensuring remote path (#{path}) exists" if verbose?
|
46
|
-
# mkdir -p
|
47
|
-
folders = path.split('/')
|
48
|
-
folders.each_index do |i|
|
49
|
-
folder = folders[0..i].join('/')
|
50
|
-
puts "Creating #{folder} on remote" if verbose?
|
51
|
-
begin
|
52
|
-
scp.mkdir!(folder)
|
53
|
-
rescue StandardError
|
54
|
-
Net::SCP::Error
|
55
|
-
end
|
56
|
-
end
|
57
|
-
retry
|
58
|
-
end
|
59
|
-
end
|
60
|
-
puts '...done' if verbose?
|
61
|
-
end
|
62
|
-
|
63
|
-
# rubocop:todo Metrics/PerceivedComplexity
|
64
|
-
def cleanup # rubocop:todo Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity, Metrics/AbcSize, Metrics/MethodLength
|
65
|
-
return if local_only? || dry_run?
|
66
|
-
|
67
|
-
return unless (keep = config[:keep, :scp])
|
68
|
-
|
69
|
-
puts "listing files: #{host}:#{base}*" if verbose?
|
70
|
-
opts = {}
|
71
|
-
opts[:password] = password if password
|
72
|
-
opts[:port] = port if port
|
73
|
-
Net::SFTP.start(host, user, opts) do |sftp|
|
74
|
-
files = sftp.dir.glob(path, File.basename("#{base}*"))
|
75
|
-
|
76
|
-
puts files.collect(&:name) if verbose?
|
77
|
-
|
78
|
-
files = files.collect(&:name).sort
|
79
|
-
|
80
|
-
cleanup_with_limit(files, keep) do |f|
|
81
|
-
file = File.join(path, f)
|
82
|
-
puts "removing sftp file #{host}:#{file}" if dry_run? || verbose?
|
83
|
-
sftp.remove!(file) unless dry_run? || local_only?
|
84
|
-
end
|
85
|
-
end
|
86
|
-
end
|
87
|
-
# rubocop:enable Metrics/PerceivedComplexity
|
88
|
-
|
89
|
-
def host
|
90
|
-
config[:scp, :host]
|
91
|
-
end
|
92
|
-
|
93
|
-
def user
|
94
|
-
config[:scp, :user]
|
95
|
-
end
|
96
|
-
|
97
|
-
def password
|
98
|
-
config[:scp, :password]
|
99
|
-
end
|
100
|
-
|
101
|
-
def port
|
102
|
-
config[:scp, :port]
|
103
|
-
end
|
104
|
-
|
105
|
-
end
|
106
|
-
|
107
|
-
end
|
108
|
-
|
109
|
-
end
|