astrails-safe 0.2.7 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/.document +5 -0
- data/.gitignore +18 -0
- data/.rspec +3 -0
- data/CHANGELOG +25 -0
- data/Gemfile +4 -0
- data/{LICENSE → LICENSE.txt} +3 -1
- data/README.markdown +109 -114
- data/Rakefile +5 -55
- data/TODO +11 -0
- data/astrails-safe.gemspec +35 -0
- data/lib/astrails/safe.rb +4 -4
- data/lib/astrails/safe/archive.rb +2 -2
- data/lib/astrails/safe/cloudfiles.rb +16 -3
- data/lib/astrails/safe/config/node.rb +1 -1
- data/lib/astrails/safe/local.rb +3 -0
- data/lib/astrails/safe/mysqldump.rb +1 -1
- data/lib/astrails/safe/pipe.rb +4 -0
- data/lib/astrails/safe/s3.rb +10 -2
- data/lib/astrails/safe/sink.rb +2 -0
- data/lib/astrails/safe/source.rb +1 -0
- data/lib/astrails/safe/stream.rb +1 -0
- data/lib/astrails/safe/version.rb +5 -0
- data/{examples/integration/archive_integration_example.rb → spec/integration/archive_integration_spec.rb} +1 -2
- data/{examples/integration/cleanup_example.rb → spec/integration/cleanup_spec.rb} +2 -3
- data/spec/spec_helper.rb +7 -0
- data/{examples/unit/archive_example.rb → spec/unit/archive_spec.rb} +1 -1
- data/spec/unit/cloudfiles_spec.rb +177 -0
- data/{examples/unit/config_example.rb → spec/unit/config_spec.rb} +1 -1
- data/{examples/unit/gpg_example.rb → spec/unit/gpg_spec.rb} +2 -2
- data/{examples/unit/gzip_example.rb → spec/unit/gzip_spec.rb} +6 -6
- data/{examples/unit/local_example.rb → spec/unit/local_spec.rb} +2 -2
- data/{examples/unit/mysqldump_example.rb → spec/unit/mysqldump_spec.rb} +2 -2
- data/{examples/unit/pgdump_example.rb → spec/unit/pgdump_spec.rb} +2 -2
- data/{examples/unit/s3_example.rb → spec/unit/s3_spec.rb} +13 -5
- data/{examples/unit/svndump_example.rb → spec/unit/svndump_spec.rb} +2 -2
- data/templates/script.rb +1 -1
- metadata +186 -91
- data/VERSION.yml +0 -4
- data/examples/example_helper.rb +0 -19
@@ -0,0 +1,35 @@
|
|
1
|
+
# coding: utf-8
|
2
|
+
lib = File.expand_path('../lib', __FILE__)
|
3
|
+
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
4
|
+
require 'astrails/safe/version'
|
5
|
+
|
6
|
+
Gem::Specification.new do |spec|
|
7
|
+
spec.name = "astrails-safe"
|
8
|
+
spec.version = Astrails::Safe::VERSION
|
9
|
+
spec.authors = ["Vitaly Kushner"]
|
10
|
+
spec.email = ["we@astrails.com"]
|
11
|
+
spec.description = <<-DESC
|
12
|
+
Astrails-Safe is a simple tool to backup databases (MySQL and PostgreSQL), Subversion repositories (with svndump) and just files.
|
13
|
+
Backups can be stored locally or remotely and can be enctypted.
|
14
|
+
Remote storage is supported on Amazon S3, Rackspace Cloud Files, or just plain SFTP.
|
15
|
+
DESC
|
16
|
+
spec.summary = %Q{Backup filesystem and databases (MySQL and PostgreSQL) locally or to a remote server/service (with encryption)}
|
17
|
+
spec.homepage = "http://astrails.com/astrails-safe"
|
18
|
+
spec.license = "MIT"
|
19
|
+
|
20
|
+
spec.default_executable = %q{astrails-safe}
|
21
|
+
|
22
|
+
spec.files = `git ls-files`.split($/)
|
23
|
+
spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
|
24
|
+
spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
|
25
|
+
spec.require_paths = ["lib"]
|
26
|
+
|
27
|
+
spec.add_dependency "aws-s3"
|
28
|
+
spec.add_dependency "cloudfiles"
|
29
|
+
spec.add_dependency "net-sftp"
|
30
|
+
|
31
|
+
spec.add_development_dependency "bundler", "~> 1.3"
|
32
|
+
spec.add_development_dependency "rake"
|
33
|
+
spec.add_development_dependency "rspec"
|
34
|
+
spec.add_development_dependency "rr", "~> 1.0.4"
|
35
|
+
end
|
data/lib/astrails/safe.rb
CHANGED
@@ -1,3 +1,5 @@
|
|
1
|
+
require "astrails/safe/version"
|
2
|
+
|
1
3
|
require "aws/s3"
|
2
4
|
require "cloudfiles"
|
3
5
|
require 'net/sftp'
|
@@ -16,8 +18,6 @@ require 'astrails/safe/stream'
|
|
16
18
|
|
17
19
|
require 'astrails/safe/backup'
|
18
20
|
|
19
|
-
require 'astrails/safe/backup'
|
20
|
-
|
21
21
|
require 'astrails/safe/source'
|
22
22
|
require 'astrails/safe/mysqldump'
|
23
23
|
require 'astrails/safe/pgdump'
|
@@ -49,8 +49,8 @@ module Astrails
|
|
49
49
|
[Svndump, [:svndump, :repos]]
|
50
50
|
].each do |klass, path|
|
51
51
|
if collection = config[*path]
|
52
|
-
collection.each do |name,
|
53
|
-
klass.new(name,
|
52
|
+
collection.each do |name, c|
|
53
|
+
klass.new(name, c).backup.run(c, :gpg, :gzip, :local, :s3, :cloudfiles, :sftp)
|
54
54
|
end
|
55
55
|
end
|
56
56
|
end
|
@@ -11,12 +11,12 @@ module Astrails
|
|
11
11
|
protected
|
12
12
|
|
13
13
|
def tar_exclude_files
|
14
|
-
[*@config[:exclude]].compact.map{|x| "--exclude=#{x}"}
|
14
|
+
[*@config[:exclude]].compact.map{|x| "--exclude=#{x}"}.join(" ")
|
15
15
|
end
|
16
16
|
|
17
17
|
def tar_files
|
18
18
|
raise RuntimeError, "missing files for tar" unless @config[:files]
|
19
|
-
[*@config[:files]].map
|
19
|
+
[*@config[:files]].map{|s| s.strip}.join(" ")
|
20
20
|
end
|
21
21
|
|
22
22
|
end
|
@@ -1,6 +1,7 @@
|
|
1
1
|
module Astrails
|
2
2
|
module Safe
|
3
3
|
class Cloudfiles < Sink
|
4
|
+
MAX_CLOUDFILES_FILE_SIZE = 5368709120
|
4
5
|
|
5
6
|
protected
|
6
7
|
|
@@ -12,6 +13,13 @@ module Astrails
|
|
12
13
|
@path ||= expand(config[:cloudfiles, :path] || config[:local, :path] || ":kind/:id")
|
13
14
|
end
|
14
15
|
|
16
|
+
# UGLY: we need this function for the reason that
|
17
|
+
# we can't double mock on ruby 1.9.2, duh!
|
18
|
+
# so we created this func to mock it all together
|
19
|
+
def get_file_size(path)
|
20
|
+
File.stat(path).size
|
21
|
+
end
|
22
|
+
|
15
23
|
def save
|
16
24
|
raise RuntimeError, "pipe-streaming not supported for S3." unless @backup.path
|
17
25
|
|
@@ -19,10 +27,14 @@ module Astrails
|
|
19
27
|
cf = CloudFiles::Connection.new(user, api_key, true, service_net) unless $LOCAL
|
20
28
|
puts "Uploading #{container}:#{full_path} from #{@backup.path}" if $_VERBOSE || $DRY_RUN
|
21
29
|
unless $DRY_RUN || $LOCAL
|
30
|
+
if get_file_size(@backup.path) > MAX_CLOUDFILES_FILE_SIZE
|
31
|
+
STDERR.puts "ERROR: File size exceeds maximum allowed for upload to Cloud Files (#{MAX_CLOUDFILES_FILE_SIZE}): #{@backup.path}"
|
32
|
+
return
|
33
|
+
end
|
22
34
|
benchmark = Benchmark.realtime do
|
23
35
|
cf_container = cf.create_container(container)
|
24
36
|
o = cf_container.create_object(full_path,true)
|
25
|
-
o.write(open(@backup.path))
|
37
|
+
o.write(File.open(@backup.path))
|
26
38
|
end
|
27
39
|
puts "...done" if $_VERBOSE
|
28
40
|
puts("Upload took " + sprintf("%.2f", benchmark) + " second(s).") if $_VERBOSE
|
@@ -36,11 +48,12 @@ module Astrails
|
|
36
48
|
|
37
49
|
puts "listing files: #{container}:#{base}*" if $_VERBOSE
|
38
50
|
cf = CloudFiles::Connection.new(user, api_key, true, service_net) unless $LOCAL
|
39
|
-
|
51
|
+
cf_container = cf.container(container)
|
52
|
+
files = cf_container.objects(:prefix => base).sort
|
40
53
|
|
41
54
|
cleanup_with_limit(files, keep) do |f|
|
42
55
|
puts "removing Cloud File #{container}:#{f}" if $DRY_RUN || $_VERBOSE
|
43
|
-
|
56
|
+
cf_container.delete_object(f) unless $DRY_RUN || $LOCAL
|
44
57
|
end
|
45
58
|
end
|
46
59
|
|
@@ -38,7 +38,7 @@ module Astrails
|
|
38
38
|
else
|
39
39
|
raise(ArgumentError, "#{key}: no block supported for simple values") if block
|
40
40
|
if @data[key.to_s]
|
41
|
-
@data[key.to_s] =
|
41
|
+
@data[key.to_s] = [*@data[key.to_s]] + [value]
|
42
42
|
else
|
43
43
|
@data[key.to_s] = value
|
44
44
|
end
|
data/lib/astrails/safe/local.rb
CHANGED
@@ -17,6 +17,7 @@ module Astrails
|
|
17
17
|
def save
|
18
18
|
puts "command: #{@backup.command}" if $_VERBOSE
|
19
19
|
|
20
|
+
# FIXME: probably need to change this to smth like @backup.finalize!
|
20
21
|
@backup.path = full_path # need to do it outside DRY_RUN so that it will be avialable for S3 DRY_RUN
|
21
22
|
|
22
23
|
unless $DRY_RUN
|
@@ -34,6 +35,8 @@ module Astrails
|
|
34
35
|
|
35
36
|
puts "listing files #{base}" if $_VERBOSE
|
36
37
|
|
38
|
+
# TODO: cleanup ALL zero-length files
|
39
|
+
|
37
40
|
files = Dir["#{base}*"] .
|
38
41
|
select{|f| File.file?(f) && File.size(f) > 0} .
|
39
42
|
sort
|
data/lib/astrails/safe/pipe.rb
CHANGED
data/lib/astrails/safe/s3.rb
CHANGED
@@ -14,6 +14,7 @@ module Astrails
|
|
14
14
|
end
|
15
15
|
|
16
16
|
def save
|
17
|
+
# FIXME: user friendly error here :)
|
17
18
|
raise RuntimeError, "pipe-streaming not supported for S3." unless @backup.path
|
18
19
|
|
19
20
|
# needed in cleanup even on dry run
|
@@ -26,7 +27,7 @@ module Astrails
|
|
26
27
|
return
|
27
28
|
end
|
28
29
|
benchmark = Benchmark.realtime do
|
29
|
-
AWS::S3::Bucket.create(bucket)
|
30
|
+
AWS::S3::Bucket.create(bucket) unless bucket_exists?(bucket)
|
30
31
|
File.open(@backup.path) do |file|
|
31
32
|
AWS::S3::S3Object.store(full_path, file, bucket)
|
32
33
|
end
|
@@ -51,7 +52,7 @@ module Astrails
|
|
51
52
|
|
52
53
|
cleanup_with_limit(files, keep) do |f|
|
53
54
|
puts "removing s3 file #{bucket}:#{f}" if $DRY_RUN || $_VERBOSE
|
54
|
-
AWS::S3::Bucket.
|
55
|
+
AWS::S3::Bucket.objects(bucket, :prefix => f)[0].delete unless $DRY_RUN || $LOCAL
|
55
56
|
end
|
56
57
|
end
|
57
58
|
|
@@ -67,6 +68,13 @@ module Astrails
|
|
67
68
|
@config[:s3, :secret]
|
68
69
|
end
|
69
70
|
|
71
|
+
private
|
72
|
+
|
73
|
+
def bucket_exists?(bucket)
|
74
|
+
true if AWS::S3::Bucket.find(bucket)
|
75
|
+
rescue AWS::S3::NoSuchBucket
|
76
|
+
false
|
77
|
+
end
|
70
78
|
end
|
71
79
|
end
|
72
80
|
end
|
data/lib/astrails/safe/sink.rb
CHANGED
@@ -12,6 +12,8 @@ module Astrails
|
|
12
12
|
protected
|
13
13
|
|
14
14
|
# path is defined in subclass
|
15
|
+
# base is used in 'cleanup' to find all files that begin with base. the '.'
|
16
|
+
# at the end is essential to distinguish b/w foo.* and foobar.* archives for example
|
15
17
|
def base
|
16
18
|
@base ||= File.join(path, File.basename(@backup.filename).split(".").first + '.')
|
17
19
|
end
|
data/lib/astrails/safe/source.rb
CHANGED
data/lib/astrails/safe/stream.rb
CHANGED
@@ -1,4 +1,4 @@
|
|
1
|
-
require
|
1
|
+
require 'spec_helper'
|
2
2
|
|
3
3
|
require "fileutils"
|
4
4
|
include FileUtils
|
@@ -51,7 +51,6 @@ describe "tar backup" do
|
|
51
51
|
end
|
52
52
|
|
53
53
|
it "should create backup file" do
|
54
|
-
puts "Expecting: #{@backup}"
|
55
54
|
File.exists?(@backup).should be_true
|
56
55
|
end
|
57
56
|
|
@@ -1,4 +1,4 @@
|
|
1
|
-
require
|
1
|
+
require 'spec_helper'
|
2
2
|
|
3
3
|
require "fileutils"
|
4
4
|
include FileUtils
|
@@ -47,7 +47,6 @@ describe "tar backup" do
|
|
47
47
|
end
|
48
48
|
|
49
49
|
it "should create backup file" do
|
50
|
-
puts "Expecting: #{@backup}"
|
51
50
|
File.exists?(@backup).should be_true
|
52
51
|
end
|
53
52
|
|
@@ -59,4 +58,4 @@ describe "tar backup" do
|
|
59
58
|
Dir["#{@dst}/archive/archive-foobar.*"].should == ["#{@dst}/archive/archive-foobar.000001.tar.gz", "#{@dst}/archive/archive-foobar.000002.tar.gz"]
|
60
59
|
end
|
61
60
|
|
62
|
-
end
|
61
|
+
end
|
data/spec/spec_helper.rb
ADDED
@@ -0,0 +1,177 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
describe Astrails::Safe::Cloudfiles do
|
4
|
+
|
5
|
+
def def_config
|
6
|
+
{
|
7
|
+
:cloudfiles => {
|
8
|
+
:container => "_container",
|
9
|
+
:user => "_user",
|
10
|
+
:api_key => "_api_key",
|
11
|
+
},
|
12
|
+
:keep => {
|
13
|
+
:cloudfiles => 2
|
14
|
+
}
|
15
|
+
}
|
16
|
+
end
|
17
|
+
|
18
|
+
def def_backup(extra = {})
|
19
|
+
{
|
20
|
+
:kind => "_kind",
|
21
|
+
:filename => "/backup/somewhere/_kind-_id.NOW.bar",
|
22
|
+
:extension => ".bar",
|
23
|
+
:id => "_id",
|
24
|
+
:timestamp => "NOW"
|
25
|
+
}.merge(extra)
|
26
|
+
end
|
27
|
+
|
28
|
+
def cloudfiles(config = def_config, backup = def_backup)
|
29
|
+
Astrails::Safe::Cloudfiles.new(
|
30
|
+
Astrails::Safe::Config::Node.new(nil, config),
|
31
|
+
Astrails::Safe::Backup.new(backup)
|
32
|
+
)
|
33
|
+
end
|
34
|
+
|
35
|
+
describe :cleanup do
|
36
|
+
|
37
|
+
before(:each) do
|
38
|
+
@cloudfiles = cloudfiles
|
39
|
+
|
40
|
+
@files = [4,1,3,2].map { |i| "aaaaa#{i}" }
|
41
|
+
|
42
|
+
@container = "container"
|
43
|
+
|
44
|
+
stub(@container).objects(:prefix => "_kind/_id/_kind-_id.") { @files }
|
45
|
+
stub(@container).delete_object(anything)
|
46
|
+
|
47
|
+
stub(CloudFiles::Connection).
|
48
|
+
new('_user', '_api_key', true, false).stub!.
|
49
|
+
container('_container') {@container}
|
50
|
+
end
|
51
|
+
|
52
|
+
it "should check [:keep, :cloudfiles]" do
|
53
|
+
@cloudfiles.config[:keep].data["cloudfiles"] = nil
|
54
|
+
dont_allow(@cloudfiles.backup).filename
|
55
|
+
@cloudfiles.send :cleanup
|
56
|
+
end
|
57
|
+
|
58
|
+
it "should delete extra files" do
|
59
|
+
mock(@container).delete_object('aaaaa1')
|
60
|
+
mock(@container).delete_object('aaaaa2')
|
61
|
+
@cloudfiles.send :cleanup
|
62
|
+
end
|
63
|
+
|
64
|
+
end
|
65
|
+
|
66
|
+
describe :active do
|
67
|
+
before(:each) do
|
68
|
+
@cloudfiles = cloudfiles
|
69
|
+
end
|
70
|
+
|
71
|
+
it "should be true when all params are set" do
|
72
|
+
@cloudfiles.should be_active
|
73
|
+
end
|
74
|
+
|
75
|
+
it "should be false if container is missing" do
|
76
|
+
@cloudfiles.config[:cloudfiles].data["container"] = nil
|
77
|
+
@cloudfiles.should_not be_active
|
78
|
+
end
|
79
|
+
|
80
|
+
it "should be false if user is missing" do
|
81
|
+
@cloudfiles.config[:cloudfiles].data["user"] = nil
|
82
|
+
@cloudfiles.should_not be_active
|
83
|
+
end
|
84
|
+
|
85
|
+
it "should be false if api_key is missing" do
|
86
|
+
@cloudfiles.config[:cloudfiles].data["api_key"] = nil
|
87
|
+
@cloudfiles.should_not be_active
|
88
|
+
end
|
89
|
+
end
|
90
|
+
|
91
|
+
describe :path do
|
92
|
+
before(:each) do
|
93
|
+
@cloudfiles = cloudfiles
|
94
|
+
end
|
95
|
+
it "should use cloudfiles/path 1st" do
|
96
|
+
@cloudfiles.config[:cloudfiles].data["path"] = "cloudfiles_path"
|
97
|
+
@cloudfiles.config[:local] = {:path => "local_path"}
|
98
|
+
@cloudfiles.send(:path).should == "cloudfiles_path"
|
99
|
+
end
|
100
|
+
|
101
|
+
it "should use local/path 2nd" do
|
102
|
+
@cloudfiles.config[:local] = {:path => "local_path"}
|
103
|
+
@cloudfiles.send(:path).should == "local_path"
|
104
|
+
end
|
105
|
+
|
106
|
+
it "should use constant 3rd" do
|
107
|
+
@cloudfiles.send(:path).should == "_kind/_id"
|
108
|
+
end
|
109
|
+
|
110
|
+
end
|
111
|
+
|
112
|
+
describe :save do
|
113
|
+
def add_stubs(*stubs)
|
114
|
+
stubs.each do |s|
|
115
|
+
case s
|
116
|
+
when :connection
|
117
|
+
@connection = "connection"
|
118
|
+
stub(CloudFiles::Authentication).new
|
119
|
+
stub(CloudFiles::Connection).
|
120
|
+
new('_user', '_api_key', true, false) {@connection}
|
121
|
+
when :file_size
|
122
|
+
stub(@cloudfiles).get_file_size("foo") {123}
|
123
|
+
when :create_container
|
124
|
+
@container = "container"
|
125
|
+
stub(@container).create_object("_kind/_id/backup/somewhere/_kind-_id.NOW.bar.bar", true) {@object}
|
126
|
+
stub(@connection).create_container {@container}
|
127
|
+
when :file_open
|
128
|
+
stub(File).open("foo")
|
129
|
+
when :cloudfiles_store
|
130
|
+
@object = "object"
|
131
|
+
stub(@object).write(nil) {true}
|
132
|
+
end
|
133
|
+
end
|
134
|
+
end
|
135
|
+
|
136
|
+
before(:each) do
|
137
|
+
@cloudfiles = cloudfiles(def_config, def_backup(:path => "foo"))
|
138
|
+
@full_path = "_kind/_id/backup/somewhere/_kind-_id.NOW.bar.bar"
|
139
|
+
end
|
140
|
+
|
141
|
+
it "should fail if no backup.file is set" do
|
142
|
+
@cloudfiles.backup.path = nil
|
143
|
+
proc {@cloudfiles.send(:save)}.should raise_error(RuntimeError)
|
144
|
+
end
|
145
|
+
|
146
|
+
it "should establish Cloud Files connection" do
|
147
|
+
add_stubs(:connection, :file_size, :create_container, :file_open, :cloudfiles_store)
|
148
|
+
@cloudfiles.send(:save)
|
149
|
+
end
|
150
|
+
|
151
|
+
it "should open local file" do
|
152
|
+
add_stubs(:connection, :file_size, :create_container, :cloudfiles_store)
|
153
|
+
mock(File).open("foo")
|
154
|
+
@cloudfiles.send(:save)
|
155
|
+
end
|
156
|
+
|
157
|
+
it "should call write on the cloudfile object with files' descriptor" do
|
158
|
+
add_stubs(:connection, :file_size, :create_container, :cloudfiles_store)
|
159
|
+
stub(File).open("foo") {"qqq"}
|
160
|
+
mock(@object).write("qqq") {true}
|
161
|
+
@cloudfiles.send(:save)
|
162
|
+
end
|
163
|
+
|
164
|
+
it "should upload file" do
|
165
|
+
add_stubs(:connection, :file_size, :create_container, :file_open, :cloudfiles_store)
|
166
|
+
@cloudfiles.send(:save)
|
167
|
+
end
|
168
|
+
|
169
|
+
it "should fail on files bigger then 5G" do
|
170
|
+
add_stubs(:connection)
|
171
|
+
mock(File).stat("foo").stub!.size {5*1024*1024*1024+1}
|
172
|
+
mock(STDERR).puts(anything)
|
173
|
+
dont_allow(Benchmark).realtime
|
174
|
+
@cloudfiles.send(:save)
|
175
|
+
end
|
176
|
+
end
|
177
|
+
end
|