webbynode-safe 0.2.5
Sign up to get free protection for your applications and to get access to all the features.
- data/LICENSE +20 -0
- data/README.markdown +211 -0
- data/Rakefile +53 -0
- data/VERSION.yml +4 -0
- data/bin/astrails-safe +53 -0
- data/examples/example_helper.rb +19 -0
- data/examples/integration/archive_integration_example.rb +86 -0
- data/examples/integration/cleanup_example.rb +62 -0
- data/examples/unit/archive_example.rb +67 -0
- data/examples/unit/config_example.rb +184 -0
- data/examples/unit/gpg_example.rb +138 -0
- data/examples/unit/gzip_example.rb +64 -0
- data/examples/unit/local_example.rb +110 -0
- data/examples/unit/mysqldump_example.rb +83 -0
- data/examples/unit/pgdump_example.rb +45 -0
- data/examples/unit/s3_example.rb +112 -0
- data/examples/unit/svndump_example.rb +39 -0
- data/lib/astrails/safe.rb +62 -0
- data/lib/astrails/safe/archive.rb +24 -0
- data/lib/astrails/safe/backup.rb +20 -0
- data/lib/astrails/safe/config/builder.rb +60 -0
- data/lib/astrails/safe/config/node.rb +66 -0
- data/lib/astrails/safe/gpg.rb +45 -0
- data/lib/astrails/safe/gzip.rb +25 -0
- data/lib/astrails/safe/local.rb +46 -0
- data/lib/astrails/safe/mongodbdump.rb +25 -0
- data/lib/astrails/safe/multi.rb +134 -0
- data/lib/astrails/safe/mysqldump.rb +31 -0
- data/lib/astrails/safe/pgdump.rb +36 -0
- data/lib/astrails/safe/pipe.rb +13 -0
- data/lib/astrails/safe/s3.rb +68 -0
- data/lib/astrails/safe/sftp.rb +79 -0
- data/lib/astrails/safe/sink.rb +37 -0
- data/lib/astrails/safe/source.rb +46 -0
- data/lib/astrails/safe/stream.rb +19 -0
- data/lib/astrails/safe/svndump.rb +13 -0
- data/lib/astrails/safe/tmp_file.rb +48 -0
- data/lib/extensions/mktmpdir.rb +45 -0
- data/templates/script.rb +130 -0
- metadata +126 -0
@@ -0,0 +1,31 @@
|
|
1
|
+
module Astrails
|
2
|
+
module Safe
|
3
|
+
class Mysqldump < Source
|
4
|
+
|
5
|
+
def command
|
6
|
+
"mysqldump --defaults-extra-file=#{mysql_password_file} #{@config[:options]} #{mysql_skip_tables} #{@id}"
|
7
|
+
end
|
8
|
+
|
9
|
+
def extension; '.sql'; end
|
10
|
+
|
11
|
+
protected
|
12
|
+
|
13
|
+
def mysql_password_file
|
14
|
+
Astrails::Safe::TmpFile.create("mysqldump") do |file|
|
15
|
+
file.puts "[mysqldump]"
|
16
|
+
%w/user password socket host port/.each do |k|
|
17
|
+
v = @config[k]
|
18
|
+
file.puts "#{k} = #{v}" if v
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
22
|
+
|
23
|
+
def mysql_skip_tables
|
24
|
+
if skip_tables = @config[:skip_tables]
|
25
|
+
[*skip_tables].map { |t| "--ignore-table=#{@id}.#{t}" } * " "
|
26
|
+
end
|
27
|
+
end
|
28
|
+
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end
|
@@ -0,0 +1,36 @@
|
|
1
|
+
module Astrails
|
2
|
+
module Safe
|
3
|
+
class Pgdump < Source
|
4
|
+
|
5
|
+
def command
|
6
|
+
if @config["password"]
|
7
|
+
ENV['PGPASSWORD'] = @config["password"]
|
8
|
+
else
|
9
|
+
ENV['PGPASSWORD'] = nil
|
10
|
+
end
|
11
|
+
"pg_dump #{postgres_options} #{postgres_username} #{postgres_host} #{postgres_port} #{@id}"
|
12
|
+
end
|
13
|
+
|
14
|
+
def extension; '.sql'; end
|
15
|
+
|
16
|
+
protected
|
17
|
+
|
18
|
+
def postgres_options
|
19
|
+
@config[:options]
|
20
|
+
end
|
21
|
+
|
22
|
+
def postgres_host
|
23
|
+
@config["host"] && "--host='#{@config["host"]}'"
|
24
|
+
end
|
25
|
+
|
26
|
+
def postgres_port
|
27
|
+
@config["port"] && "--port='#{@config["port"]}'"
|
28
|
+
end
|
29
|
+
|
30
|
+
def postgres_username
|
31
|
+
@config["user"] && "--username='#{@config["user"]}'"
|
32
|
+
end
|
33
|
+
|
34
|
+
end
|
35
|
+
end
|
36
|
+
end
|
@@ -0,0 +1,68 @@
|
|
1
|
+
module Astrails
|
2
|
+
module Safe
|
3
|
+
class S3 < Sink
|
4
|
+
|
5
|
+
protected
|
6
|
+
|
7
|
+
def active?
|
8
|
+
bucket && key && secret
|
9
|
+
end
|
10
|
+
|
11
|
+
def path
|
12
|
+
@path ||= expand(config[:s3, :path] || config[:local, :path] || ":kind/:id")
|
13
|
+
end
|
14
|
+
|
15
|
+
def save
|
16
|
+
raise RuntimeError, "pipe-streaming not supported for S3." unless @backup.path
|
17
|
+
|
18
|
+
# needed in cleanup even on dry run
|
19
|
+
AWS::S3::Base.establish_connection!(:access_key_id => key, :secret_access_key => secret, :use_ssl => true) unless $LOCAL
|
20
|
+
|
21
|
+
puts "Uploading #{bucket}:#{full_path}" if $_VERBOSE || $DRY_RUN
|
22
|
+
unless $DRY_RUN || $LOCAL
|
23
|
+
benchmark = Benchmark.realtime do
|
24
|
+
AWS::S3::Bucket.create(bucket)
|
25
|
+
File.open(@backup.path) do |file|
|
26
|
+
AWS::S3::S3Object.store(full_path, file, bucket)
|
27
|
+
end
|
28
|
+
end
|
29
|
+
puts "...done" if $_VERBOSE
|
30
|
+
puts("Upload took " + sprintf("%.2f", benchmark) + " second(s).") if $_VERBOSE
|
31
|
+
end
|
32
|
+
end
|
33
|
+
|
34
|
+
def cleanup
|
35
|
+
return if $LOCAL
|
36
|
+
|
37
|
+
return unless keep = @config[:keep, :s3]
|
38
|
+
|
39
|
+
|
40
|
+
puts "listing files: #{bucket}:#{base}*" if $_VERBOSE
|
41
|
+
files = AWS::S3::Bucket.objects(bucket, :prefix => base, :max_keys => keep * 2)
|
42
|
+
puts files.collect {|x| x.key} if $_VERBOSE
|
43
|
+
|
44
|
+
files = files.
|
45
|
+
collect {|x| x.key}.
|
46
|
+
sort
|
47
|
+
|
48
|
+
cleanup_with_limit(files, keep) do |f|
|
49
|
+
puts "removing s3 file #{bucket}:#{f}" if $DRY_RUN || $_VERBOSE
|
50
|
+
AWS::S3::Bucket.find(bucket)[f].delete unless $DRY_RUN || $LOCAL
|
51
|
+
end
|
52
|
+
end
|
53
|
+
|
54
|
+
def bucket
|
55
|
+
@config[:s3, :bucket]
|
56
|
+
end
|
57
|
+
|
58
|
+
def key
|
59
|
+
@config[:s3, :key]
|
60
|
+
end
|
61
|
+
|
62
|
+
def secret
|
63
|
+
@config[:s3, :secret]
|
64
|
+
end
|
65
|
+
|
66
|
+
end
|
67
|
+
end
|
68
|
+
end
|
@@ -0,0 +1,79 @@
|
|
1
|
+
module Astrails
|
2
|
+
module Safe
|
3
|
+
class Sftp < Sink
|
4
|
+
|
5
|
+
protected
|
6
|
+
|
7
|
+
def active?
|
8
|
+
host && user
|
9
|
+
end
|
10
|
+
|
11
|
+
def path
|
12
|
+
@path ||= expand(config[:sftp, :path] || config[:local, :path] || ":kind/:id")
|
13
|
+
end
|
14
|
+
|
15
|
+
def save
|
16
|
+
raise RuntimeError, "pipe-streaming not supported for SFTP." unless @backup.path
|
17
|
+
|
18
|
+
puts "Uploading #{host}:#{full_path} via SFTP" if $_VERBOSE || $DRY_RUN
|
19
|
+
|
20
|
+
unless $DRY_RUN || $LOCAL
|
21
|
+
opts = {}
|
22
|
+
opts[:password] = password if password
|
23
|
+
Net::SFTP.start(host, user, opts) do |sftp|
|
24
|
+
puts "Sending #{@backup.path} to #{full_path}" if $_VERBOSE
|
25
|
+
begin
|
26
|
+
sftp.upload! @backup.path, full_path
|
27
|
+
rescue Net::SFTP::StatusException
|
28
|
+
puts "Ensuring remote path (#{path}) exists" if $_VERBOSE
|
29
|
+
# mkdir -p
|
30
|
+
folders = path.split('/')
|
31
|
+
folders.each_index do |i|
|
32
|
+
folder = folders[0..i].join('/')
|
33
|
+
puts "Creating #{folder} on remote" if $_VERBOSE
|
34
|
+
sftp.mkdir!(folder) rescue Net::SFTP::StatusException
|
35
|
+
end
|
36
|
+
retry
|
37
|
+
end
|
38
|
+
end
|
39
|
+
puts "...done" if $_VERBOSE
|
40
|
+
end
|
41
|
+
end
|
42
|
+
|
43
|
+
def cleanup
|
44
|
+
return if $LOCAL || $DRY_RUN
|
45
|
+
|
46
|
+
return unless keep = @config[:keep, :sftp]
|
47
|
+
|
48
|
+
puts "listing files: #{host}:#{base}*" if $_VERBOSE
|
49
|
+
Net::SFTP.start(host, user, :password => password) do |sftp|
|
50
|
+
files = sftp.dir.glob(path, File.basename("#{base}*"))
|
51
|
+
|
52
|
+
puts files.collect {|x| x.name } if $_VERBOSE
|
53
|
+
|
54
|
+
files = files.
|
55
|
+
collect {|x| x.name }.
|
56
|
+
sort
|
57
|
+
|
58
|
+
cleanup_with_limit(files, keep) do |f|
|
59
|
+
file = File.join(path, f)
|
60
|
+
puts "removing sftp file #{host}:#{file}" if $DRY_RUN || $_VERBOSE
|
61
|
+
sftp.remove!(file) unless $DRY_RUN || $LOCAL
|
62
|
+
end
|
63
|
+
end
|
64
|
+
end
|
65
|
+
|
66
|
+
def host
|
67
|
+
@config[:sftp, :host]
|
68
|
+
end
|
69
|
+
|
70
|
+
def user
|
71
|
+
@config[:sftp, :user]
|
72
|
+
end
|
73
|
+
|
74
|
+
def password
|
75
|
+
@config[:sftp, :password]
|
76
|
+
end
|
77
|
+
end
|
78
|
+
end
|
79
|
+
end
|
@@ -0,0 +1,37 @@
|
|
1
|
+
module Astrails
|
2
|
+
module Safe
|
3
|
+
class Sink < Stream
|
4
|
+
|
5
|
+
def process
|
6
|
+
return unless active?
|
7
|
+
|
8
|
+
save
|
9
|
+
cleanup
|
10
|
+
end
|
11
|
+
|
12
|
+
protected
|
13
|
+
|
14
|
+
def v(s)
|
15
|
+
puts s if $_VERBOSE
|
16
|
+
end
|
17
|
+
|
18
|
+
# path is defined in subclass
|
19
|
+
def base
|
20
|
+
@base ||= File.join(path, File.basename(@backup.filename).split(".").first + '.')
|
21
|
+
end
|
22
|
+
|
23
|
+
def full_path
|
24
|
+
@full_path ||= File.join(path, @backup.filename) + @backup.extension
|
25
|
+
end
|
26
|
+
|
27
|
+
# call block on files to be removed (all except for the LAST 'limit' files
|
28
|
+
def cleanup_with_limit(files, limit, &block)
|
29
|
+
return unless files.size > limit
|
30
|
+
|
31
|
+
to_remove = files[0..(files.size - limit - 1)]
|
32
|
+
# TODO: validate here
|
33
|
+
to_remove.each(&block)
|
34
|
+
end
|
35
|
+
end
|
36
|
+
end
|
37
|
+
end
|
@@ -0,0 +1,46 @@
|
|
1
|
+
module Astrails
|
2
|
+
module Safe
|
3
|
+
class Source < Stream
|
4
|
+
|
5
|
+
attr_accessor :id
|
6
|
+
def initialize(id, config)
|
7
|
+
@id, @config = id.to_s, config
|
8
|
+
end
|
9
|
+
|
10
|
+
def timestamp
|
11
|
+
Time.now.strftime("%y%m%d-%H%M")
|
12
|
+
end
|
13
|
+
|
14
|
+
def kind
|
15
|
+
self.class.human_name
|
16
|
+
end
|
17
|
+
|
18
|
+
def filename
|
19
|
+
@filename ||= expand(":kind-:id.:timestamp")
|
20
|
+
end
|
21
|
+
|
22
|
+
def backup
|
23
|
+
return @backup if @backup
|
24
|
+
@backup = Backup.new(
|
25
|
+
:id => @id,
|
26
|
+
:kind => kind,
|
27
|
+
:extension => extension,
|
28
|
+
:command => command,
|
29
|
+
:timestamp => timestamp
|
30
|
+
)
|
31
|
+
# can't do this in the initializer hash above since
|
32
|
+
# filename() calls expand() which requires @backup
|
33
|
+
@backup.filename = filename
|
34
|
+
@backup
|
35
|
+
end
|
36
|
+
|
37
|
+
protected
|
38
|
+
|
39
|
+
def self.human_name
|
40
|
+
name.split('::').last.downcase
|
41
|
+
end
|
42
|
+
|
43
|
+
end
|
44
|
+
end
|
45
|
+
end
|
46
|
+
|
@@ -0,0 +1,19 @@
|
|
1
|
+
module Astrails
|
2
|
+
module Safe
|
3
|
+
class Stream
|
4
|
+
|
5
|
+
attr_accessor :config, :backup
|
6
|
+
def initialize(config, backup)
|
7
|
+
@config, @backup = config, backup
|
8
|
+
end
|
9
|
+
|
10
|
+
def expand(path)
|
11
|
+
path .
|
12
|
+
gsub(/:kind\b/, @backup.kind.to_s) .
|
13
|
+
gsub(/:id\b/, @backup.id.to_s) .
|
14
|
+
gsub(/:timestamp\b/, @backup.timestamp)
|
15
|
+
end
|
16
|
+
|
17
|
+
end
|
18
|
+
end
|
19
|
+
end
|
@@ -0,0 +1,48 @@
|
|
1
|
+
require 'tmpdir'
|
2
|
+
module Astrails
|
3
|
+
module Safe
|
4
|
+
module TmpFile
|
5
|
+
@keep_files = []
|
6
|
+
|
7
|
+
def self.tmproot
|
8
|
+
@tmproot ||= Dir.mktmpdir
|
9
|
+
end
|
10
|
+
|
11
|
+
def self.cleanup
|
12
|
+
begin
|
13
|
+
FileUtils.remove_entry_secure tmproot
|
14
|
+
rescue ArgumentError => e
|
15
|
+
if e.message =~ /parent directory is world writable/
|
16
|
+
puts <<-ERR
|
17
|
+
|
18
|
+
|
19
|
+
********************************************************************************
|
20
|
+
It looks like you have wrong permissions on your TEMP directory. The usual
|
21
|
+
case is when you have world writable TEMP directory withOUT the sticky bit.
|
22
|
+
|
23
|
+
Try "chmod +t" on it.
|
24
|
+
|
25
|
+
********************************************************************************
|
26
|
+
|
27
|
+
ERR
|
28
|
+
else
|
29
|
+
raise
|
30
|
+
end
|
31
|
+
end
|
32
|
+
@tmproot = nil
|
33
|
+
end
|
34
|
+
|
35
|
+
def self.create(name)
|
36
|
+
# create temp directory
|
37
|
+
|
38
|
+
file = Tempfile.new(name, tmproot)
|
39
|
+
|
40
|
+
yield file
|
41
|
+
|
42
|
+
file.close
|
43
|
+
@keep_files << file # so that it will not get gcollected and removed from filesystem until the end
|
44
|
+
file.path
|
45
|
+
end
|
46
|
+
end
|
47
|
+
end
|
48
|
+
end
|
@@ -0,0 +1,45 @@
|
|
1
|
+
require 'tmpdir'
|
2
|
+
|
3
|
+
unless Dir.respond_to?(:mktmpdir)
|
4
|
+
# backward compat for 1.8.6
|
5
|
+
class Dir
|
6
|
+
def Dir.mktmpdir(prefix_suffix=nil, tmpdir=nil)
|
7
|
+
case prefix_suffix
|
8
|
+
when nil
|
9
|
+
prefix = "d"
|
10
|
+
suffix = ""
|
11
|
+
when String
|
12
|
+
prefix = prefix_suffix
|
13
|
+
suffix = ""
|
14
|
+
when Array
|
15
|
+
prefix = prefix_suffix[0]
|
16
|
+
suffix = prefix_suffix[1]
|
17
|
+
else
|
18
|
+
raise ArgumentError, "unexpected prefix_suffix: #{prefix_suffix.inspect}"
|
19
|
+
end
|
20
|
+
tmpdir ||= Dir.tmpdir
|
21
|
+
t = Time.now.strftime("%Y%m%d")
|
22
|
+
n = nil
|
23
|
+
begin
|
24
|
+
path = "#{tmpdir}/#{prefix}#{t}-#{$$}-#{rand(0x100000000).to_s(36)}"
|
25
|
+
path << "-#{n}" if n
|
26
|
+
path << suffix
|
27
|
+
Dir.mkdir(path, 0700)
|
28
|
+
rescue Errno::EEXIST
|
29
|
+
n ||= 0
|
30
|
+
n += 1
|
31
|
+
retry
|
32
|
+
end
|
33
|
+
|
34
|
+
if block_given?
|
35
|
+
begin
|
36
|
+
yield path
|
37
|
+
ensure
|
38
|
+
FileUtils.remove_entry_secure path
|
39
|
+
end
|
40
|
+
else
|
41
|
+
path
|
42
|
+
end
|
43
|
+
end
|
44
|
+
end
|
45
|
+
end
|