akupchanko-astrails-safe 0.3.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/.autotest +3 -0
- data/.document +5 -0
- data/.gitignore +18 -0
- data/.rspec +3 -0
- data/CHANGELOG +35 -0
- data/Gemfile +7 -0
- data/LICENSE.txt +22 -0
- data/README.markdown +250 -0
- data/Rakefile +8 -0
- data/TODO +31 -0
- data/akupchanko-astrails-safe.gemspec +35 -0
- data/bin/astrails-safe +64 -0
- data/lib/astrails/safe.rb +68 -0
- data/lib/astrails/safe/archive.rb +24 -0
- data/lib/astrails/safe/backup.rb +20 -0
- data/lib/astrails/safe/cloudfiles.rb +77 -0
- data/lib/astrails/safe/config/builder.rb +90 -0
- data/lib/astrails/safe/config/node.rb +72 -0
- data/lib/astrails/safe/ftp.rb +104 -0
- data/lib/astrails/safe/gpg.rb +46 -0
- data/lib/astrails/safe/gzip.rb +25 -0
- data/lib/astrails/safe/local.rb +51 -0
- data/lib/astrails/safe/mongodump.rb +23 -0
- data/lib/astrails/safe/mysqldump.rb +32 -0
- data/lib/astrails/safe/pgdump.rb +36 -0
- data/lib/astrails/safe/pipe.rb +17 -0
- data/lib/astrails/safe/s3.rb +80 -0
- data/lib/astrails/safe/sftp.rb +88 -0
- data/lib/astrails/safe/sink.rb +35 -0
- data/lib/astrails/safe/source.rb +47 -0
- data/lib/astrails/safe/stream.rb +32 -0
- data/lib/astrails/safe/svndump.rb +13 -0
- data/lib/astrails/safe/tmp_file.rb +48 -0
- data/lib/astrails/safe/version.rb +5 -0
- data/lib/extensions/mktmpdir.rb +45 -0
- data/spec/astrails/safe/archive_spec.rb +67 -0
- data/spec/astrails/safe/cloudfiles_spec.rb +175 -0
- data/spec/astrails/safe/config_spec.rb +307 -0
- data/spec/astrails/safe/gpg_spec.rb +148 -0
- data/spec/astrails/safe/gzip_spec.rb +64 -0
- data/spec/astrails/safe/local_spec.rb +109 -0
- data/spec/astrails/safe/mongodump_spec.rb +54 -0
- data/spec/astrails/safe/mysqldump_spec.rb +83 -0
- data/spec/astrails/safe/pgdump_spec.rb +45 -0
- data/spec/astrails/safe/s3_spec.rb +168 -0
- data/spec/astrails/safe/svndump_spec.rb +39 -0
- data/spec/integration/archive_integration_spec.rb +89 -0
- data/spec/integration/cleanup_spec.rb +62 -0
- data/spec/spec_helper.rb +8 -0
- data/templates/script.rb +183 -0
- metadata +178 -0
@@ -0,0 +1,68 @@
|
|
1
|
+
require "astrails/safe/version"
|
2
|
+
|
3
|
+
require "aws/s3"
|
4
|
+
require "cloudfiles"
|
5
|
+
require 'net/sftp'
|
6
|
+
require 'net/ftp'
|
7
|
+
require 'fileutils'
|
8
|
+
require 'benchmark'
|
9
|
+
|
10
|
+
require 'tempfile'
|
11
|
+
require 'extensions/mktmpdir'
|
12
|
+
|
13
|
+
require 'astrails/safe/tmp_file'
|
14
|
+
|
15
|
+
require 'astrails/safe/config/node'
|
16
|
+
require 'astrails/safe/config/builder'
|
17
|
+
|
18
|
+
require 'astrails/safe/stream'
|
19
|
+
|
20
|
+
require 'astrails/safe/backup'
|
21
|
+
|
22
|
+
require 'astrails/safe/source'
|
23
|
+
require 'astrails/safe/mysqldump'
|
24
|
+
require 'astrails/safe/pgdump'
|
25
|
+
require 'astrails/safe/archive'
|
26
|
+
require 'astrails/safe/svndump'
|
27
|
+
require 'astrails/safe/mongodump'
|
28
|
+
|
29
|
+
require 'astrails/safe/pipe'
|
30
|
+
require 'astrails/safe/gpg'
|
31
|
+
require 'astrails/safe/gzip'
|
32
|
+
|
33
|
+
require 'astrails/safe/sink'
|
34
|
+
require 'astrails/safe/local'
|
35
|
+
require 'astrails/safe/s3'
|
36
|
+
require 'astrails/safe/cloudfiles'
|
37
|
+
require 'astrails/safe/sftp'
|
38
|
+
require 'astrails/safe/ftp'
|
39
|
+
|
40
|
+
module Astrails
|
41
|
+
module Safe
|
42
|
+
ROOT = File.join(File.dirname(__FILE__), "..", "..")
|
43
|
+
|
44
|
+
def safe(&block)
|
45
|
+
Config::Node.new(&block)
|
46
|
+
end
|
47
|
+
|
48
|
+
def process(config)
|
49
|
+
|
50
|
+
[[Mysqldump, [:mysqldump, :databases]],
|
51
|
+
[Pgdump, [:pgdump, :databases]],
|
52
|
+
[Mongodump, [:mongodump, :databases]],
|
53
|
+
[Archive, [:tar, :archives]],
|
54
|
+
[Svndump, [:svndump, :repos]]
|
55
|
+
].each do |klass, path|
|
56
|
+
if collection = config[*path]
|
57
|
+
collection.each do |name, c|
|
58
|
+
klass.new(name, c).backup.run(c, :gpg, :gzip, :local, :s3, :cloudfiles, :sftp, :ftp)
|
59
|
+
end
|
60
|
+
end
|
61
|
+
end
|
62
|
+
|
63
|
+
Astrails::Safe::TmpFile.cleanup
|
64
|
+
end
|
65
|
+
module_function :safe
|
66
|
+
module_function :process
|
67
|
+
end
|
68
|
+
end
|
@@ -0,0 +1,24 @@
|
|
1
|
+
module Astrails
|
2
|
+
module Safe
|
3
|
+
class Archive < Source
|
4
|
+
|
5
|
+
def command
|
6
|
+
"tar -cf - #{config[:options]} #{tar_exclude_files} #{tar_files}"
|
7
|
+
end
|
8
|
+
|
9
|
+
def extension; '.tar'; end
|
10
|
+
|
11
|
+
protected
|
12
|
+
|
13
|
+
def tar_exclude_files
|
14
|
+
[*config[:exclude]].compact.map{|x| "--exclude=#{x}"}.join(" ")
|
15
|
+
end
|
16
|
+
|
17
|
+
def tar_files
|
18
|
+
raise RuntimeError, "missing files for tar" unless config[:files]
|
19
|
+
[*config[:files]].map{|s| s.strip}.join(" ")
|
20
|
+
end
|
21
|
+
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
@@ -0,0 +1,20 @@
|
|
1
|
+
module Astrails
|
2
|
+
module Safe
|
3
|
+
class Backup
|
4
|
+
attr_accessor :id, :kind, :filename, :extension, :command, :compressed, :timestamp, :path
|
5
|
+
def initialize(opts = {})
|
6
|
+
opts.each do |k, v|
|
7
|
+
self.send("#{k}=", v)
|
8
|
+
end
|
9
|
+
end
|
10
|
+
|
11
|
+
def run(config, *mods)
|
12
|
+
mods.each do |mod|
|
13
|
+
mod = mod.to_s
|
14
|
+
mod[0] = mod[0..0].upcase
|
15
|
+
Astrails::Safe.const_get(mod).new(config, self).process
|
16
|
+
end
|
17
|
+
end
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|
@@ -0,0 +1,77 @@
|
|
1
|
+
module Astrails
|
2
|
+
module Safe
|
3
|
+
class Cloudfiles < Sink
|
4
|
+
MAX_CLOUDFILES_FILE_SIZE = 5368709120
|
5
|
+
|
6
|
+
protected
|
7
|
+
|
8
|
+
def active?
|
9
|
+
container && user && api_key
|
10
|
+
end
|
11
|
+
|
12
|
+
def path
|
13
|
+
@path ||= expand(config[:cloudfiles, :path] || config[:local, :path] || ":kind/:id")
|
14
|
+
end
|
15
|
+
|
16
|
+
# UGLY: we need this function for the reason that
|
17
|
+
# we can't double mock on ruby 1.9.2, duh!
|
18
|
+
# so we created this func to mock it all together
|
19
|
+
def get_file_size(path)
|
20
|
+
File.stat(path).size
|
21
|
+
end
|
22
|
+
|
23
|
+
def save
|
24
|
+
raise RuntimeError, "pipe-streaming not supported for S3." unless @backup.path
|
25
|
+
|
26
|
+
# needed in cleanup even on dry run
|
27
|
+
cf = CloudFiles::Connection.new(user, api_key, true, service_net) unless local_only?
|
28
|
+
puts "Uploading #{container}:#{full_path} from #{@backup.path}" if verbose? || dry_run?
|
29
|
+
unless dry_run? || local_only?
|
30
|
+
if get_file_size(@backup.path) > MAX_CLOUDFILES_FILE_SIZE
|
31
|
+
STDERR.puts "ERROR: File size exceeds maximum allowed for upload to Cloud Files (#{MAX_CLOUDFILES_FILE_SIZE}): #{@backup.path}"
|
32
|
+
return
|
33
|
+
end
|
34
|
+
benchmark = Benchmark.realtime do
|
35
|
+
cf_container = cf.create_container(container)
|
36
|
+
o = cf_container.create_object(full_path,true)
|
37
|
+
o.write(File.open(@backup.path))
|
38
|
+
end
|
39
|
+
puts "...done" if verbose?
|
40
|
+
puts("Upload took " + sprintf("%.2f", benchmark) + " second(s).") if verbose?
|
41
|
+
end
|
42
|
+
end
|
43
|
+
|
44
|
+
def cleanup
|
45
|
+
return if local_only?
|
46
|
+
|
47
|
+
return unless keep = config[:keep, :cloudfiles]
|
48
|
+
|
49
|
+
puts "listing files: #{container}:#{base}*" if verbose?
|
50
|
+
cf = CloudFiles::Connection.new(user, api_key, true, service_net) unless local_only?
|
51
|
+
cf_container = cf.container(container)
|
52
|
+
files = cf_container.objects(:prefix => base).sort
|
53
|
+
|
54
|
+
cleanup_with_limit(files, keep) do |f|
|
55
|
+
puts "removing Cloud File #{container}:#{f}" if dry_run? || verbose?
|
56
|
+
cf_container.delete_object(f) unless dry_run? || local_only?
|
57
|
+
end
|
58
|
+
end
|
59
|
+
|
60
|
+
def container
|
61
|
+
config[:cloudfiles, :container]
|
62
|
+
end
|
63
|
+
|
64
|
+
def user
|
65
|
+
config[:cloudfiles, :user]
|
66
|
+
end
|
67
|
+
|
68
|
+
def api_key
|
69
|
+
config[:cloudfiles, :api_key]
|
70
|
+
end
|
71
|
+
|
72
|
+
def service_net
|
73
|
+
config[:cloudfiles, :service_net] || false
|
74
|
+
end
|
75
|
+
end
|
76
|
+
end
|
77
|
+
end
|
@@ -0,0 +1,90 @@
|
|
1
|
+
module Astrails
|
2
|
+
module Safe
|
3
|
+
module Config
|
4
|
+
class Builder
|
5
|
+
|
6
|
+
def initialize(node, data = {})
|
7
|
+
@node = node
|
8
|
+
data.each { |k, v| self.send k, v }
|
9
|
+
end
|
10
|
+
|
11
|
+
|
12
|
+
class << self
|
13
|
+
def simple_value(*names)
|
14
|
+
names.each do |m|
|
15
|
+
define_method(m) do |value|
|
16
|
+
ensure_uniq(m)
|
17
|
+
@node.set m, value
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|
21
|
+
|
22
|
+
def multi_value(*names)
|
23
|
+
names.each do |m|
|
24
|
+
define_method(m) do |value|
|
25
|
+
value = value.map(&:to_s) if value.is_a?(Array)
|
26
|
+
@node.set_multi m, value
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
30
|
+
|
31
|
+
def hash_value(*names)
|
32
|
+
names.each do |m|
|
33
|
+
define_method(m) do |data = {}, &block|
|
34
|
+
ensure_uniq(m)
|
35
|
+
ensure_hash(m, data)
|
36
|
+
@node.set m, Node.new(@node, data || {}, &block)
|
37
|
+
end
|
38
|
+
end
|
39
|
+
end
|
40
|
+
|
41
|
+
def mixed_value(*names)
|
42
|
+
names.each do |m|
|
43
|
+
define_method(m) do |data={}, &block|
|
44
|
+
ensure_uniq(m)
|
45
|
+
if data.is_a?(Hash) || block
|
46
|
+
ensure_hash(m, data) if block
|
47
|
+
@node.set m, Node.new(@node, data, &block)
|
48
|
+
else
|
49
|
+
@node.set m, data
|
50
|
+
end
|
51
|
+
end
|
52
|
+
end
|
53
|
+
end
|
54
|
+
|
55
|
+
def collection(*names)
|
56
|
+
names.each do |m|
|
57
|
+
define_method(m) do |id, data={}, &block|
|
58
|
+
raise "bad collection id: #{id.inspect}" unless id
|
59
|
+
ensure_hash(m, data)
|
60
|
+
|
61
|
+
name = "#{m}s"
|
62
|
+
collection = @node.get(name) || @node.set(name, Node.new(@node, {}))
|
63
|
+
collection.set id, Node.new(collection, data, &block)
|
64
|
+
end
|
65
|
+
end
|
66
|
+
end
|
67
|
+
end
|
68
|
+
|
69
|
+
simple_value :verbose, :dry_run, :local_only, :path, :command,
|
70
|
+
:options, :user, :host, :port, :password, :key, :secret, :bucket,
|
71
|
+
:api_key, :container, :socket, :service_net, :repo_path
|
72
|
+
multi_value :skip_tables, :exclude, :files
|
73
|
+
hash_value :mysqldump, :tar, :gpg, :keep, :pgdump, :tar, :svndump,
|
74
|
+
:sftp, :ftp, :mongodump
|
75
|
+
mixed_value :s3, :local, :cloudfiles
|
76
|
+
collection :database, :archive, :repo
|
77
|
+
|
78
|
+
private
|
79
|
+
|
80
|
+
def ensure_uniq(m)
|
81
|
+
raise(ArgumentError, "duplicate value for '#{m}'") if @node.get(m)
|
82
|
+
end
|
83
|
+
|
84
|
+
def ensure_hash(k, v)
|
85
|
+
raise "#{k}: hash expected: #{v.inspect}" unless v.is_a?(Hash)
|
86
|
+
end
|
87
|
+
end
|
88
|
+
end
|
89
|
+
end
|
90
|
+
end
|
@@ -0,0 +1,72 @@
|
|
1
|
+
require 'astrails/safe/config/builder'
|
2
|
+
module Astrails
|
3
|
+
module Safe
|
4
|
+
module Config
|
5
|
+
class Node
|
6
|
+
attr_reader :parent, :data
|
7
|
+
|
8
|
+
def initialize(parent = nil, data = {}, &block)
|
9
|
+
@parent = parent
|
10
|
+
@data = {}
|
11
|
+
merge data, &block
|
12
|
+
end
|
13
|
+
|
14
|
+
def merge data = {}, &block
|
15
|
+
builder = Builder.new(self, data)
|
16
|
+
builder.instance_eval(&block) if block
|
17
|
+
self
|
18
|
+
end
|
19
|
+
|
20
|
+
# looks for the path from this node DOWN. will not delegate to parent
|
21
|
+
def get(*path)
|
22
|
+
key = path.shift
|
23
|
+
value = @data[key.to_s]
|
24
|
+
return value if (nil != value) && path.empty?
|
25
|
+
|
26
|
+
value && value.get(*path)
|
27
|
+
end
|
28
|
+
|
29
|
+
# recursive find
|
30
|
+
# starts at the node and continues to the parent
|
31
|
+
def find(*path)
|
32
|
+
get(*path) || @parent && @parent.find(*path)
|
33
|
+
end
|
34
|
+
alias :[] :find
|
35
|
+
|
36
|
+
def set_multi(key, value)
|
37
|
+
@data[key.to_s] ||= []
|
38
|
+
@data[key.to_s].concat [*value]
|
39
|
+
end
|
40
|
+
|
41
|
+
def set(key, value)
|
42
|
+
@data[key.to_s] = value
|
43
|
+
end
|
44
|
+
alias :[]= :set
|
45
|
+
|
46
|
+
def each(&block)
|
47
|
+
@data.each(&block)
|
48
|
+
end
|
49
|
+
include Enumerable
|
50
|
+
|
51
|
+
def to_hash
|
52
|
+
@data.keys.inject({}) do |res, key|
|
53
|
+
value = @data[key]
|
54
|
+
res[key] = value.is_a?(Node) ? value.to_hash : value
|
55
|
+
res
|
56
|
+
end
|
57
|
+
end
|
58
|
+
|
59
|
+
def dump(indent = "")
|
60
|
+
@data.each do |key, value|
|
61
|
+
if value.is_a?(Node)
|
62
|
+
puts "#{indent}#{key}:"
|
63
|
+
value.dump(indent + " ")
|
64
|
+
else
|
65
|
+
puts "#{indent}#{key}: #{value.inspect}"
|
66
|
+
end
|
67
|
+
end
|
68
|
+
end
|
69
|
+
end
|
70
|
+
end
|
71
|
+
end
|
72
|
+
end
|
@@ -0,0 +1,104 @@
|
|
1
|
+
module Astrails
|
2
|
+
module Safe
|
3
|
+
class Ftp < Sink
|
4
|
+
|
5
|
+
protected
|
6
|
+
|
7
|
+
def active?
|
8
|
+
host && user
|
9
|
+
end
|
10
|
+
|
11
|
+
def path
|
12
|
+
@path ||= expand(config[:ftp, :path] || config[:local, :path] || ":kind/:id")
|
13
|
+
end
|
14
|
+
|
15
|
+
def save
|
16
|
+
raise RuntimeError, "pipe-streaming not supported for FTP." unless @backup.path
|
17
|
+
|
18
|
+
puts "Uploading #{host}:#{full_path} via FTP" if verbose? || dry_run?
|
19
|
+
|
20
|
+
unless dry_run? || local_only?
|
21
|
+
if !port
|
22
|
+
port = 21
|
23
|
+
end
|
24
|
+
Net::FTP.open(host) do |ftp|
|
25
|
+
ftp.connect(host, port)
|
26
|
+
ftp.login(user, password)
|
27
|
+
|
28
|
+
dir = File.dirname(full_path)
|
29
|
+
parts = dir.split("/")
|
30
|
+
growing_path = ""
|
31
|
+
for part in parts
|
32
|
+
next if part == ""
|
33
|
+
if growing_path == ""
|
34
|
+
growing_path = part
|
35
|
+
else
|
36
|
+
growing_path = File.join(growing_path, part)
|
37
|
+
end
|
38
|
+
puts "Trying to create remote directory (#{growing_path})" if verbose?
|
39
|
+
begin
|
40
|
+
ftp.mkdir(growing_path)
|
41
|
+
rescue Net::FTPPermError
|
42
|
+
puts "Remote directory (#{growing_path}) exists, or no enough permissions" if verbose?
|
43
|
+
end
|
44
|
+
end
|
45
|
+
|
46
|
+
puts "Sending #{@backup.path} to #{full_path}" if verbose?
|
47
|
+
begin
|
48
|
+
ftp.put(@backup.path, full_path)
|
49
|
+
rescue Net::FTPPermError
|
50
|
+
puts "Ensuring remote path (#{path}) exists" if verbose?
|
51
|
+
end
|
52
|
+
end
|
53
|
+
puts "...done" if verbose?
|
54
|
+
end
|
55
|
+
end
|
56
|
+
|
57
|
+
def cleanup
|
58
|
+
return if local_only? || dry_run?
|
59
|
+
|
60
|
+
return unless keep = config[:keep, :ftp]
|
61
|
+
|
62
|
+
puts "listing files: #{host}:#{base}*" if verbose?
|
63
|
+
if !port
|
64
|
+
port = 21
|
65
|
+
end
|
66
|
+
Net::FTP.open(host) do |ftp|
|
67
|
+
ftp.connect(host, port)
|
68
|
+
ftp.login(user, password)
|
69
|
+
files = ftp.nlst(path)
|
70
|
+
pattern = File.basename("#{base}")
|
71
|
+
files = files.reject{ |x| !x.start_with?(pattern)}
|
72
|
+
puts files.collect {|x| x} if verbose?
|
73
|
+
|
74
|
+
files = files.
|
75
|
+
collect {|x| x }.
|
76
|
+
sort
|
77
|
+
|
78
|
+
cleanup_with_limit(files, keep) do |f|
|
79
|
+
file = File.path(f)
|
80
|
+
puts "removing ftp file #{host}:#{file}" if dry_run? || verbose?
|
81
|
+
ftp.delete(file) unless dry_run? || local_only?
|
82
|
+
end
|
83
|
+
end
|
84
|
+
end
|
85
|
+
|
86
|
+
def host
|
87
|
+
config[:ftp, :host]
|
88
|
+
end
|
89
|
+
|
90
|
+
def user
|
91
|
+
config[:ftp, :user]
|
92
|
+
end
|
93
|
+
|
94
|
+
def password
|
95
|
+
config[:ftp, :password]
|
96
|
+
end
|
97
|
+
|
98
|
+
def port
|
99
|
+
config[:ftp, :port]
|
100
|
+
end
|
101
|
+
|
102
|
+
end
|
103
|
+
end
|
104
|
+
end
|