webtranslateit-safe 0.4.3 → 0.4.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,12 +1,14 @@
1
1
  module WebTranslateIt
2
+
2
3
  module Safe
4
+
3
5
  class Mysqldump < Source
4
6
 
5
7
  def command
6
8
  "mysqldump --defaults-extra-file=#{mysql_password_file} #{config[:options]} #{mysql_skip_tables} #{@id}"
7
9
  end
8
10
 
9
- def extension; '.sql'; end
11
+ def extension = '.sql'
10
12
 
11
13
  protected
12
14
 
@@ -22,11 +24,13 @@ module WebTranslateIt
22
24
  end
23
25
 
24
26
  def mysql_skip_tables
25
- if skip_tables = config[:skip_tables]
26
- [*skip_tables].map{ |t| "--ignore-table=#{@id}.#{t}" }.join(' ')
27
- end
27
+ return unless skip_tables = config[:skip_tables]
28
+
29
+ [*skip_tables].map { |t| "--ignore-table=#{@id}.#{t}" }.join(' ')
28
30
  end
29
31
 
30
32
  end
33
+
31
34
  end
32
- end
35
+
36
+ end
@@ -1,17 +1,15 @@
1
1
  module WebTranslateIt
2
+
2
3
  module Safe
4
+
3
5
  class Pgdump < Source
4
6
 
5
7
  def command
6
- if config['password']
7
- ENV['PGPASSWORD'] = config['password']
8
- else
9
- ENV['PGPASSWORD'] = nil
10
- end
8
+ ENV['PGPASSWORD'] = (config['password'] || nil)
11
9
  "pg_dump #{postgres_options} #{postgres_username} #{postgres_host} #{postgres_port} #{@id}"
12
10
  end
13
11
 
14
- def extension; '.sql'; end
12
+ def extension = '.sql'
15
13
 
16
14
  protected
17
15
 
@@ -20,17 +18,19 @@ module WebTranslateIt
20
18
  end
21
19
 
22
20
  def postgres_host
23
- config['host'] && "--host='#{config["host"]}'"
21
+ config['host'] && "--host='#{config['host']}'"
24
22
  end
25
23
 
26
24
  def postgres_port
27
- config['port'] && "--port='#{config["port"]}'"
25
+ config['port'] && "--port='#{config['port']}'"
28
26
  end
29
27
 
30
28
  def postgres_username
31
- config['user'] && "--username='#{config["user"]}'"
29
+ config['user'] && "--username='#{config['user']}'"
32
30
  end
33
31
 
34
32
  end
33
+
35
34
  end
35
+
36
36
  end
@@ -1,6 +1,9 @@
1
1
  module WebTranslateIt
2
+
2
3
  module Safe
4
+
3
5
  class Pipe < Stream
6
+
4
7
  # process adds required commands to the current
5
8
  # shell command string
6
9
  # :active?, :pipe, :extension and :post_process are
@@ -12,6 +15,9 @@ module WebTranslateIt
12
15
  @backup.extension << extension
13
16
  post_process
14
17
  end
18
+
15
19
  end
20
+
16
21
  end
22
+
17
23
  end
@@ -1,7 +1,10 @@
1
1
  module WebTranslateIt
2
+
2
3
  module Safe
4
+
3
5
  class S3 < Sink
4
- MAX_S3_FILE_SIZE = 5368709120
6
+
7
+ MAX_S3_FILE_SIZE = 5_368_709_120
5
8
 
6
9
  def active?
7
10
  bucket && key && secret
@@ -15,26 +18,26 @@ module WebTranslateIt
15
18
 
16
19
  def save
17
20
  # FIXME: user friendly error here :)
18
- raise RuntimeError, 'pipe-streaming not supported for S3.' unless @backup.path
21
+ raise 'pipe-streaming not supported for S3.' unless @backup.path
19
22
 
20
23
  # needed in cleanup even on dry run
21
- AWS::S3::Base.establish_connection!(:access_key_id => key, :secret_access_key => secret, :use_ssl => true) unless local_only?
24
+ AWS::S3::Base.establish_connection!(access_key_id: key, secret_access_key: secret, use_ssl: true) unless local_only?
22
25
 
23
26
  puts "Uploading #{bucket}:#{full_path}" if verbose? || dry_run?
24
- unless dry_run? || local_only?
25
- if File.stat(@backup.path).size > MAX_S3_FILE_SIZE
26
- STDERR.puts "ERROR: File size exceeds maximum allowed for upload to S3 (#{MAX_S3_FILE_SIZE}): #{@backup.path}"
27
- return
28
- end
29
- benchmark = Benchmark.realtime do
30
- AWS::S3::Bucket.create(bucket) unless bucket_exists?(bucket)
31
- File.open(@backup.path) do |file|
32
- AWS::S3::S3Object.store(full_path, file, bucket)
33
- end
27
+ return if dry_run? || local_only?
28
+
29
+ if File.stat(@backup.path).size > MAX_S3_FILE_SIZE
30
+ warn "ERROR: File size exceeds maximum allowed for upload to S3 (#{MAX_S3_FILE_SIZE}): #{@backup.path}"
31
+ return
32
+ end
33
+ benchmark = Benchmark.realtime do
34
+ AWS::S3::Bucket.create(bucket) unless bucket_exists?(bucket)
35
+ File.open(@backup.path) do |file|
36
+ AWS::S3::S3Object.store(full_path, file, bucket)
34
37
  end
35
- puts '...done' if verbose?
36
- puts('Upload took ' + sprintf('%.2f', benchmark) + ' second(s).') if verbose?
37
38
  end
39
+ puts '...done' if verbose?
40
+ puts("Upload took #{format('%.2f', benchmark)} second(s).") if verbose?
38
41
  end
39
42
 
40
43
  def cleanup
@@ -43,16 +46,16 @@ module WebTranslateIt
43
46
  return unless keep = config[:keep, :s3]
44
47
 
45
48
  puts "listing files: #{bucket}:#{base}*" if verbose?
46
- files = AWS::S3::Bucket.objects(bucket, :prefix => base, :max_keys => keep * 2)
47
- puts files.collect {|x| x.key} if verbose?
49
+ files = AWS::S3::Bucket.objects(bucket, prefix: base, max_keys: keep * 2)
50
+ puts files.collect(&:key) if verbose?
48
51
 
49
- files = files.
50
- collect {|x| x.key}.
51
- sort
52
+ files = files
53
+ .collect(&:key)
54
+ .sort
52
55
 
53
56
  cleanup_with_limit(files, keep) do |f|
54
57
  puts "removing s3 file #{bucket}:#{f}" if dry_run? || verbose?
55
- AWS::S3::Bucket.objects(bucket, :prefix => f)[0].delete unless dry_run? || local_only?
58
+ AWS::S3::Bucket.objects(bucket, prefix: f)[0].delete unless dry_run? || local_only?
56
59
  end
57
60
  end
58
61
 
@@ -69,12 +72,15 @@ module WebTranslateIt
69
72
  end
70
73
 
71
74
  private
72
-
75
+
73
76
  def bucket_exists?(bucket)
74
77
  true if AWS::S3::Bucket.find(bucket)
75
78
  rescue AWS::S3::NoSuchBucket
76
79
  false
77
80
  end
81
+
78
82
  end
83
+
79
84
  end
85
+
80
86
  end
@@ -0,0 +1,107 @@
1
+ # frozen_string_literal: true
2
+
3
+ module WebTranslateIt
4
+
5
+ module Safe
6
+
7
+ class Scp < Sink
8
+
9
+ MAX_RETRIES = 5
10
+
11
+ protected
12
+
13
+ def active?
14
+ host && user
15
+ end
16
+
17
+ def path
18
+ @path ||= expand(config[:scp, :path] || config[:local, :path] || ':kind/:id')
19
+ end
20
+
21
+ def save # rubocop:todo Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
22
+ raise 'pipe-streaming not supported for SCP.' unless @backup.path
23
+
24
+ puts "Uploading #{host}:#{full_path} via SCP" if verbose? || dry_run?
25
+
26
+ return if dry_run? || local_only?
27
+
28
+ retries = 0
29
+ opts = {}
30
+ opts[:password] = password if password
31
+ opts[:port] = port if port
32
+ Net::SCP.start(host, user, opts) do |scp|
33
+ puts "Sending #{@backup.path} to #{full_path}" if verbose?
34
+ begin
35
+ scp.upload! @backup.path, full_path
36
+ rescue IO::TimeoutError
37
+ puts 'Upload timed out, retrying'
38
+ retries += 1
39
+ if retries >= MAX_RETRIES
40
+ puts "Tried #{retries} times. Giving up."
41
+ else
42
+ retry unless retries >= MAX_RETRIES
43
+ end
44
+ rescue Net::SCP::Error
45
+ puts "Ensuring remote path (#{path}) exists" if verbose?
46
+ # mkdir -p
47
+ folders = path.split('/')
48
+ folders.each_index do |i|
49
+ folder = folders[0..i].join('/')
50
+ puts "Creating #{folder} on remote" if verbose?
51
+ begin
52
+ scp.mkdir!(folder)
53
+ rescue StandardError
54
+ Net::SCP::Error
55
+ end
56
+ end
57
+ retry
58
+ end
59
+ end
60
+ puts '...done' if verbose?
61
+ end
62
+
63
+ def cleanup
64
+ return if local_only? || dry_run?
65
+
66
+ return unless (keep = config[:keep, :scp])
67
+
68
+ puts "listing files: #{host}:#{base}*" if verbose?
69
+ opts = {}
70
+ opts[:password] = password if password
71
+ opts[:port] = port if port
72
+ Net::SFTP.start(host, user, opts) do |sftp|
73
+ files = sftp.dir.glob(path, File.basename("#{base}*"))
74
+
75
+ puts files.collect(&:name) if verbose?
76
+
77
+ files = files.collect(&:name).sort
78
+
79
+ cleanup_with_limit(files, keep) do |f|
80
+ file = File.join(path, f)
81
+ puts "removing sftp file #{host}:#{file}" if dry_run? || verbose?
82
+ sftp.remove!(file) unless dry_run? || local_only?
83
+ end
84
+ end
85
+ end
86
+
87
+ def host
88
+ config[:scp, :host]
89
+ end
90
+
91
+ def user
92
+ config[:scp, :user]
93
+ end
94
+
95
+ def password
96
+ config[:scp, :password]
97
+ end
98
+
99
+ def port
100
+ config[:scp, :port]
101
+ end
102
+
103
+ end
104
+
105
+ end
106
+
107
+ end
@@ -1,7 +1,11 @@
1
1
  module WebTranslateIt
2
+
2
3
  module Safe
4
+
3
5
  class Sftp < Sink
4
6
 
7
+ MAX_RETRIES = 5
8
+
5
9
  protected
6
10
 
7
11
  def active?
@@ -12,33 +16,46 @@ module WebTranslateIt
12
16
  @path ||= expand(config[:sftp, :path] || config[:local, :path] || ':kind/:id')
13
17
  end
14
18
 
15
- def save
16
- raise RuntimeError, 'pipe-streaming not supported for SFTP.' unless @backup.path
19
+ def save # rubocop:todo Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
20
+ raise 'pipe-streaming not supported for SFTP.' unless @backup.path
17
21
 
18
22
  puts "Uploading #{host}:#{full_path} via SFTP" if verbose? || dry_run?
19
23
 
20
- unless dry_run? || local_only?
21
- opts = {}
22
- opts[:password] = password if password
23
- opts[:port] = port if port
24
- Net::SFTP.start(host, user, opts) do |sftp|
25
- puts "Sending #{@backup.path} to #{full_path}" if verbose?
26
- begin
27
- sftp.upload! @backup.path, full_path
28
- rescue Net::SFTP::StatusException
29
- puts "Ensuring remote path (#{path}) exists" if verbose?
30
- # mkdir -p
31
- folders = path.split('/')
32
- folders.each_index do |i|
33
- folder = folders[0..i].join('/')
34
- puts "Creating #{folder} on remote" if verbose?
35
- sftp.mkdir!(folder) rescue Net::SFTP::StatusException
24
+ return if dry_run? || local_only?
25
+
26
+ retries = 0
27
+ opts = {}
28
+ opts[:password] = password if password
29
+ opts[:port] = port if port
30
+ Net::SFTP.start(host, user, opts) do |sftp|
31
+ puts "Sending #{@backup.path} to #{full_path}" if verbose?
32
+ begin
33
+ sftp.upload! @backup.path, full_path
34
+ rescue IO::TimeoutError
35
+ puts 'Upload timed out, retrying'
36
+ retries += 1
37
+ if retries >= MAX_RETRIES
38
+ puts "Tried #{retries} times. Giving up."
39
+ else
40
+ retry unless retries >= MAX_RETRIES
41
+ end
42
+ rescue Net::SFTP::StatusException
43
+ puts "Ensuring remote path (#{path}) exists" if verbose?
44
+ # mkdir -p
45
+ folders = path.split('/')
46
+ folders.each_index do |i|
47
+ folder = folders[0..i].join('/')
48
+ puts "Creating #{folder} on remote" if verbose?
49
+ begin
50
+ sftp.mkdir!(folder)
51
+ rescue StandardError
52
+ Net::SFTP::StatusException
36
53
  end
37
- retry
38
54
  end
55
+ retry
39
56
  end
40
- puts '...done' if verbose?
41
57
  end
58
+ puts '...done' if verbose?
42
59
  end
43
60
 
44
61
  def cleanup
@@ -53,11 +70,9 @@ module WebTranslateIt
53
70
  Net::SFTP.start(host, user, opts) do |sftp|
54
71
  files = sftp.dir.glob(path, File.basename("#{base}*"))
55
72
 
56
- puts files.collect {|x| x.name } if verbose?
73
+ puts files.collect(&:name) if verbose?
57
74
 
58
- files = files.
59
- collect {|x| x.name }.
60
- sort
75
+ files = files.collect(&:name).sort
61
76
 
62
77
  cleanup_with_limit(files, keep) do |f|
63
78
  file = File.join(path, f)
@@ -84,5 +99,7 @@ module WebTranslateIt
84
99
  end
85
100
 
86
101
  end
102
+
87
103
  end
88
- end
104
+
105
+ end
@@ -1,5 +1,7 @@
1
1
  module WebTranslateIt
2
+
2
3
  module Safe
4
+
3
5
  class Sink < Stream
4
6
 
5
7
  def process
@@ -15,7 +17,7 @@ module WebTranslateIt
15
17
  # base is used in 'cleanup' to find all files that begin with base. the '.'
16
18
  # at the end is essential to distinguish b/w foo.* and foobar.* archives for example
17
19
  def base
18
- @base ||= File.join(path, File.basename(@backup.filename).split('.').first + '.')
20
+ @base ||= File.join(path, "#{File.basename(@backup.filename).split('.').first}.")
19
21
  end
20
22
 
21
23
  def full_path
@@ -30,6 +32,9 @@ module WebTranslateIt
30
32
  # TODO: validate here
31
33
  to_remove.each(&block)
32
34
  end
35
+
33
36
  end
37
+
34
38
  end
35
- end
39
+
40
+ end
@@ -1,10 +1,14 @@
1
1
  module WebTranslateIt
2
+
2
3
  module Safe
4
+
3
5
  class Source < Stream
4
6
 
5
7
  attr_accessor :id
8
+
6
9
  def initialize(id, config)
7
- @id, @config = id.to_s, config
10
+ @id = id.to_s
11
+ @config = config
8
12
  end
9
13
 
10
14
  def timestamp
@@ -21,12 +25,13 @@ module WebTranslateIt
21
25
 
22
26
  def backup
23
27
  return @backup if @backup
28
+
24
29
  @backup = Backup.new(
25
- :id => @id,
26
- :kind => kind,
27
- :extension => extension,
28
- :command => command,
29
- :timestamp => timestamp
30
+ id: @id,
31
+ kind: kind,
32
+ extension: extension,
33
+ command: command,
34
+ timestamp: timestamp
30
35
  )
31
36
  # can't do this in the initializer hash above since
32
37
  # filename() calls expand() which requires @backup
@@ -35,13 +40,12 @@ module WebTranslateIt
35
40
  @backup
36
41
  end
37
42
 
38
- protected
39
-
40
43
  def self.human_name
41
44
  name.split('::').last.downcase
42
45
  end
43
46
 
44
47
  end
48
+
45
49
  end
46
- end
47
50
 
51
+ end
@@ -1,17 +1,22 @@
1
1
  module WebTranslateIt
2
+
2
3
  module Safe
4
+
3
5
  class Stream
4
6
 
5
7
  attr_accessor :config, :backup
8
+
6
9
  def initialize(config, backup)
7
- @config, @backup = config, backup
10
+ @config = config
11
+ @backup = backup
8
12
  end
13
+
9
14
  # FIXME: move to Backup
10
15
  def expand(path)
11
- path .
12
- gsub(/:kind\b/, @backup.kind.to_s) .
13
- gsub(/:id\b/, @backup.id.to_s) .
14
- gsub(/:timestamp\b/, @backup.timestamp)
16
+ path
17
+ .gsub(/:kind\b/, @backup.kind.to_s)
18
+ .gsub(/:id\b/, @backup.id.to_s)
19
+ .gsub(/:timestamp\b/, @backup.timestamp)
15
20
  end
16
21
 
17
22
  private
@@ -27,6 +32,9 @@ module WebTranslateIt
27
32
  def dry_run?
28
33
  config[:dry_run]
29
34
  end
35
+
30
36
  end
37
+
31
38
  end
32
- end
39
+
40
+ end
@@ -1,13 +1,17 @@
1
1
  module WebTranslateIt
2
+
2
3
  module Safe
4
+
3
5
  class Svndump < Source
4
6
 
5
7
  def command
6
8
  "svnadmin dump #{config[:options]} #{config[:repo_path]}"
7
9
  end
8
10
 
9
- def extension; '.svn'; end
11
+ def extension = '.svn'
10
12
 
11
13
  end
14
+
12
15
  end
16
+
13
17
  end
@@ -1,7 +1,10 @@
1
1
  require 'tmpdir'
2
2
  module WebTranslateIt
3
+
3
4
  module Safe
5
+
4
6
  module TmpFile
7
+
5
8
  @keep_files = []
6
9
 
7
10
  def self.tmproot
@@ -12,22 +15,21 @@ module WebTranslateIt
12
15
  begin
13
16
  FileUtils.remove_entry_secure tmproot
14
17
  rescue ArgumentError => e
15
- if e.message =~ /parent directory is world writable/
16
- puts <<-ERR
18
+ raise unless e.message.include?('parent directory is world writable')
19
+
20
+ puts <<~ERR
17
21
 
18
22
 
19
- ********************************************************************************
20
- It looks like you have wrong permissions on your TEMP directory. The usual
21
- case is when you have world writable TEMP directory withOUT the sticky bit.
23
+ ********************************************************************************
24
+ It looks like you have wrong permissions on your TEMP directory. The usual
25
+ case is when you have world writable TEMP directory withOUT the sticky bit.
22
26
 
23
- Try "chmod +t" on it.
27
+ Try "chmod +t" on it.
24
28
 
25
- ********************************************************************************
29
+ ********************************************************************************
26
30
 
27
- ERR
28
- else
29
- raise
30
- end
31
+ ERR
32
+ raise
31
33
  end
32
34
  @tmproot = nil
33
35
  end
@@ -43,6 +45,9 @@ ERR
43
45
  @keep_files << file # so that it will not get gcollected and removed from filesystem until the end
44
46
  file.path
45
47
  end
48
+
46
49
  end
50
+
47
51
  end
52
+
48
53
  end
@@ -1,12 +1,12 @@
1
1
  require 'aws/s3'
2
2
  require 'cloudfiles'
3
3
  require 'net/sftp'
4
+ require 'net/scp'
4
5
  # require 'net/ftp'
5
6
  require 'fileutils'
6
7
  require 'benchmark'
7
8
 
8
9
  require 'tempfile'
9
- require 'extensions/mktmpdir'
10
10
 
11
11
  require 'webtranslateit/safe/tmp_file'
12
12
 
@@ -32,11 +32,14 @@ require 'webtranslateit/safe/sink'
32
32
  require 'webtranslateit/safe/local'
33
33
  require 'webtranslateit/safe/s3'
34
34
  require 'webtranslateit/safe/cloudfiles'
35
+ require 'webtranslateit/safe/scp'
35
36
  require 'webtranslateit/safe/sftp'
36
37
  require 'webtranslateit/safe/ftp'
37
38
 
38
39
  module WebTranslateIt
40
+
39
41
  module Safe
42
+
40
43
  ROOT = File.join(File.dirname(__FILE__), '..', '..')
41
44
 
42
45
  def safe(&block)
@@ -44,17 +47,15 @@ module WebTranslateIt
44
47
  end
45
48
 
46
49
  def process(config)
50
+ [[Mysqldump, %i[mysqldump databases]],
51
+ [Pgdump, %i[pgdump databases]],
52
+ [Mongodump, %i[mongodump databases]],
53
+ [Archive, %i[tar archives]],
54
+ [Svndump, %i[svndump repos]]].each do |klass, path|
55
+ next unless collection = config[*path]
47
56
 
48
- [[Mysqldump, [:mysqldump, :databases]],
49
- [Pgdump, [:pgdump, :databases]],
50
- [Mongodump, [:mongodump, :databases]],
51
- [Archive, [:tar, :archives]],
52
- [Svndump, [:svndump, :repos]]
53
- ].each do |klass, path|
54
- if collection = config[*path]
55
- collection.each do |name, c|
56
- klass.new(name, c).backup.run(c, :gpg, :gzip, :local, :s3, :cloudfiles, :sftp, :ftp)
57
- end
57
+ collection.each do |name, c|
58
+ klass.new(name, c).backup.run(c, :gpg, :gzip, :local, :s3, :cloudfiles, :scp, :sftp, :ftp)
58
59
  end
59
60
  end
60
61
 
@@ -62,5 +63,7 @@ module WebTranslateIt
62
63
  end
63
64
  module_function :safe
64
65
  module_function :process
66
+
65
67
  end
68
+
66
69
  end