astrails-safe 0.2.7 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (39) hide show
  1. data/.document +5 -0
  2. data/.gitignore +18 -0
  3. data/.rspec +3 -0
  4. data/CHANGELOG +25 -0
  5. data/Gemfile +4 -0
  6. data/{LICENSE → LICENSE.txt} +3 -1
  7. data/README.markdown +109 -114
  8. data/Rakefile +5 -55
  9. data/TODO +11 -0
  10. data/astrails-safe.gemspec +35 -0
  11. data/lib/astrails/safe.rb +4 -4
  12. data/lib/astrails/safe/archive.rb +2 -2
  13. data/lib/astrails/safe/cloudfiles.rb +16 -3
  14. data/lib/astrails/safe/config/node.rb +1 -1
  15. data/lib/astrails/safe/local.rb +3 -0
  16. data/lib/astrails/safe/mysqldump.rb +1 -1
  17. data/lib/astrails/safe/pipe.rb +4 -0
  18. data/lib/astrails/safe/s3.rb +10 -2
  19. data/lib/astrails/safe/sink.rb +2 -0
  20. data/lib/astrails/safe/source.rb +1 -0
  21. data/lib/astrails/safe/stream.rb +1 -0
  22. data/lib/astrails/safe/version.rb +5 -0
  23. data/{examples/integration/archive_integration_example.rb → spec/integration/archive_integration_spec.rb} +1 -2
  24. data/{examples/integration/cleanup_example.rb → spec/integration/cleanup_spec.rb} +2 -3
  25. data/spec/spec_helper.rb +7 -0
  26. data/{examples/unit/archive_example.rb → spec/unit/archive_spec.rb} +1 -1
  27. data/spec/unit/cloudfiles_spec.rb +177 -0
  28. data/{examples/unit/config_example.rb → spec/unit/config_spec.rb} +1 -1
  29. data/{examples/unit/gpg_example.rb → spec/unit/gpg_spec.rb} +2 -2
  30. data/{examples/unit/gzip_example.rb → spec/unit/gzip_spec.rb} +6 -6
  31. data/{examples/unit/local_example.rb → spec/unit/local_spec.rb} +2 -2
  32. data/{examples/unit/mysqldump_example.rb → spec/unit/mysqldump_spec.rb} +2 -2
  33. data/{examples/unit/pgdump_example.rb → spec/unit/pgdump_spec.rb} +2 -2
  34. data/{examples/unit/s3_example.rb → spec/unit/s3_spec.rb} +13 -5
  35. data/{examples/unit/svndump_example.rb → spec/unit/svndump_spec.rb} +2 -2
  36. data/templates/script.rb +1 -1
  37. metadata +186 -91
  38. data/VERSION.yml +0 -4
  39. data/examples/example_helper.rb +0 -19
@@ -0,0 +1,35 @@
1
+ # coding: utf-8
2
+ lib = File.expand_path('../lib', __FILE__)
3
+ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
4
+ require 'astrails/safe/version'
5
+
6
+ Gem::Specification.new do |spec|
7
+ spec.name = "astrails-safe"
8
+ spec.version = Astrails::Safe::VERSION
9
+ spec.authors = ["Vitaly Kushner"]
10
+ spec.email = ["we@astrails.com"]
11
+ spec.description = <<-DESC
12
+ Astrails-Safe is a simple tool to backup databases (MySQL and PostgreSQL), Subversion repositories (with svndump) and just files.
13
+ Backups can be stored locally or remotely and can be enctypted.
14
+ Remote storage is supported on Amazon S3, Rackspace Cloud Files, or just plain SFTP.
15
+ DESC
16
+ spec.summary = %Q{Backup filesystem and databases (MySQL and PostgreSQL) locally or to a remote server/service (with encryption)}
17
+ spec.homepage = "http://astrails.com/astrails-safe"
18
+ spec.license = "MIT"
19
+
20
+ spec.default_executable = %q{astrails-safe}
21
+
22
+ spec.files = `git ls-files`.split($/)
23
+ spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
24
+ spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
25
+ spec.require_paths = ["lib"]
26
+
27
+ spec.add_dependency "aws-s3"
28
+ spec.add_dependency "cloudfiles"
29
+ spec.add_dependency "net-sftp"
30
+
31
+ spec.add_development_dependency "bundler", "~> 1.3"
32
+ spec.add_development_dependency "rake"
33
+ spec.add_development_dependency "rspec"
34
+ spec.add_development_dependency "rr", "~> 1.0.4"
35
+ end
@@ -1,3 +1,5 @@
1
+ require "astrails/safe/version"
2
+
1
3
  require "aws/s3"
2
4
  require "cloudfiles"
3
5
  require 'net/sftp'
@@ -16,8 +18,6 @@ require 'astrails/safe/stream'
16
18
 
17
19
  require 'astrails/safe/backup'
18
20
 
19
- require 'astrails/safe/backup'
20
-
21
21
  require 'astrails/safe/source'
22
22
  require 'astrails/safe/mysqldump'
23
23
  require 'astrails/safe/pgdump'
@@ -49,8 +49,8 @@ module Astrails
49
49
  [Svndump, [:svndump, :repos]]
50
50
  ].each do |klass, path|
51
51
  if collection = config[*path]
52
- collection.each do |name, config|
53
- klass.new(name, config).backup.run(config, :gpg, :gzip, :local, :s3, :cloudfiles, :sftp)
52
+ collection.each do |name, c|
53
+ klass.new(name, c).backup.run(c, :gpg, :gzip, :local, :s3, :cloudfiles, :sftp)
54
54
  end
55
55
  end
56
56
  end
@@ -11,12 +11,12 @@ module Astrails
11
11
  protected
12
12
 
13
13
  def tar_exclude_files
14
- [*@config[:exclude]].compact.map{|x| "--exclude=#{x}"} * " "
14
+ [*@config[:exclude]].compact.map{|x| "--exclude=#{x}"}.join(" ")
15
15
  end
16
16
 
17
17
  def tar_files
18
18
  raise RuntimeError, "missing files for tar" unless @config[:files]
19
- [*@config[:files]].map {|s| s.strip} * " "
19
+ [*@config[:files]].map{|s| s.strip}.join(" ")
20
20
  end
21
21
 
22
22
  end
@@ -1,6 +1,7 @@
1
1
  module Astrails
2
2
  module Safe
3
3
  class Cloudfiles < Sink
4
+ MAX_CLOUDFILES_FILE_SIZE = 5368709120
4
5
 
5
6
  protected
6
7
 
@@ -12,6 +13,13 @@ module Astrails
12
13
  @path ||= expand(config[:cloudfiles, :path] || config[:local, :path] || ":kind/:id")
13
14
  end
14
15
 
16
+ # UGLY: we need this function for the reason that
17
+ # we can't double mock on ruby 1.9.2, duh!
18
+ # so we created this func to mock it all together
19
+ def get_file_size(path)
20
+ File.stat(path).size
21
+ end
22
+
15
23
  def save
16
24
  raise RuntimeError, "pipe-streaming not supported for S3." unless @backup.path
17
25
 
@@ -19,10 +27,14 @@ module Astrails
19
27
  cf = CloudFiles::Connection.new(user, api_key, true, service_net) unless $LOCAL
20
28
  puts "Uploading #{container}:#{full_path} from #{@backup.path}" if $_VERBOSE || $DRY_RUN
21
29
  unless $DRY_RUN || $LOCAL
30
+ if get_file_size(@backup.path) > MAX_CLOUDFILES_FILE_SIZE
31
+ STDERR.puts "ERROR: File size exceeds maximum allowed for upload to Cloud Files (#{MAX_CLOUDFILES_FILE_SIZE}): #{@backup.path}"
32
+ return
33
+ end
22
34
  benchmark = Benchmark.realtime do
23
35
  cf_container = cf.create_container(container)
24
36
  o = cf_container.create_object(full_path,true)
25
- o.write(open(@backup.path))
37
+ o.write(File.open(@backup.path))
26
38
  end
27
39
  puts "...done" if $_VERBOSE
28
40
  puts("Upload took " + sprintf("%.2f", benchmark) + " second(s).") if $_VERBOSE
@@ -36,11 +48,12 @@ module Astrails
36
48
 
37
49
  puts "listing files: #{container}:#{base}*" if $_VERBOSE
38
50
  cf = CloudFiles::Connection.new(user, api_key, true, service_net) unless $LOCAL
39
- files = cf.container(container).objects(:prefix => base)
51
+ cf_container = cf.container(container)
52
+ files = cf_container.objects(:prefix => base).sort
40
53
 
41
54
  cleanup_with_limit(files, keep) do |f|
42
55
  puts "removing Cloud File #{container}:#{f}" if $DRY_RUN || $_VERBOSE
43
- cf.container(container).delete_object(f) unless $DRY_RUN || $LOCAL
56
+ cf_container.delete_object(f) unless $DRY_RUN || $LOCAL
44
57
  end
45
58
  end
46
59
 
@@ -38,7 +38,7 @@ module Astrails
38
38
  else
39
39
  raise(ArgumentError, "#{key}: no block supported for simple values") if block
40
40
  if @data[key.to_s]
41
- @data[key.to_s] = @data[key.to_s].to_a + value.to_a
41
+ @data[key.to_s] = [*@data[key.to_s]] + [value]
42
42
  else
43
43
  @data[key.to_s] = value
44
44
  end
@@ -17,6 +17,7 @@ module Astrails
17
17
  def save
18
18
  puts "command: #{@backup.command}" if $_VERBOSE
19
19
 
20
+ # FIXME: probably need to change this to smth like @backup.finalize!
20
21
  @backup.path = full_path # need to do it outside DRY_RUN so that it will be avialable for S3 DRY_RUN
21
22
 
22
23
  unless $DRY_RUN
@@ -34,6 +35,8 @@ module Astrails
34
35
 
35
36
  puts "listing files #{base}" if $_VERBOSE
36
37
 
38
+ # TODO: cleanup ALL zero-length files
39
+
37
40
  files = Dir["#{base}*"] .
38
41
  select{|f| File.file?(f) && File.size(f) > 0} .
39
42
  sort
@@ -23,7 +23,7 @@ module Astrails
23
23
 
24
24
  def mysql_skip_tables
25
25
  if skip_tables = @config[:skip_tables]
26
- [*skip_tables].map { |t| "--ignore-table=#{@id}.#{t}" } * " "
26
+ [*skip_tables].map{ |t| "--ignore-table=#{@id}.#{t}" }.join(" ")
27
27
  end
28
28
  end
29
29
 
@@ -1,6 +1,10 @@
1
1
  module Astrails
2
2
  module Safe
3
3
  class Pipe < Stream
4
+ # process adds required commands to the current
5
+ # shell command string
6
+ # :active?, :pipe, :extension and :post_process are
7
+ # defined in inheriting pipe classes
4
8
  def process
5
9
  return unless active?
6
10
 
@@ -14,6 +14,7 @@ module Astrails
14
14
  end
15
15
 
16
16
  def save
17
+ # FIXME: user friendly error here :)
17
18
  raise RuntimeError, "pipe-streaming not supported for S3." unless @backup.path
18
19
 
19
20
  # needed in cleanup even on dry run
@@ -26,7 +27,7 @@ module Astrails
26
27
  return
27
28
  end
28
29
  benchmark = Benchmark.realtime do
29
- AWS::S3::Bucket.create(bucket)
30
+ AWS::S3::Bucket.create(bucket) unless bucket_exists?(bucket)
30
31
  File.open(@backup.path) do |file|
31
32
  AWS::S3::S3Object.store(full_path, file, bucket)
32
33
  end
@@ -51,7 +52,7 @@ module Astrails
51
52
 
52
53
  cleanup_with_limit(files, keep) do |f|
53
54
  puts "removing s3 file #{bucket}:#{f}" if $DRY_RUN || $_VERBOSE
54
- AWS::S3::Bucket.find(bucket)[f].delete unless $DRY_RUN || $LOCAL
55
+ AWS::S3::Bucket.objects(bucket, :prefix => f)[0].delete unless $DRY_RUN || $LOCAL
55
56
  end
56
57
  end
57
58
 
@@ -67,6 +68,13 @@ module Astrails
67
68
  @config[:s3, :secret]
68
69
  end
69
70
 
71
+ private
72
+
73
+ def bucket_exists?(bucket)
74
+ true if AWS::S3::Bucket.find(bucket)
75
+ rescue AWS::S3::NoSuchBucket
76
+ false
77
+ end
70
78
  end
71
79
  end
72
80
  end
@@ -12,6 +12,8 @@ module Astrails
12
12
  protected
13
13
 
14
14
  # path is defined in subclass
15
+ # base is used in 'cleanup' to find all files that begin with base. the '.'
16
+ # at the end is essential to distinguish b/w foo.* and foobar.* archives for example
15
17
  def base
16
18
  @base ||= File.join(path, File.basename(@backup.filename).split(".").first + '.')
17
19
  end
@@ -30,6 +30,7 @@ module Astrails
30
30
  )
31
31
  # can't do this in the initializer hash above since
32
32
  # filename() calls expand() which requires @backup
33
+ # FIXME: move expansion to the backup (last step in ctor) assign :tags here
33
34
  @backup.filename = filename
34
35
  @backup
35
36
  end
@@ -7,6 +7,7 @@ module Astrails
7
7
  @config, @backup = config, backup
8
8
  end
9
9
 
10
+ # FIXME: move to Backup
10
11
  def expand(path)
11
12
  path .
12
13
  gsub(/:kind\b/, @backup.kind.to_s) .
@@ -0,0 +1,5 @@
1
+ module Astrails
2
+ module Safe
3
+ VERSION = "0.3.0"
4
+ end
5
+ end
@@ -1,4 +1,4 @@
1
- require File.expand_path(File.dirname(__FILE__) + '/../example_helper')
1
+ require 'spec_helper'
2
2
 
3
3
  require "fileutils"
4
4
  include FileUtils
@@ -51,7 +51,6 @@ describe "tar backup" do
51
51
  end
52
52
 
53
53
  it "should create backup file" do
54
- puts "Expecting: #{@backup}"
55
54
  File.exists?(@backup).should be_true
56
55
  end
57
56
 
@@ -1,4 +1,4 @@
1
- require File.expand_path(File.dirname(__FILE__) + '/../example_helper')
1
+ require 'spec_helper'
2
2
 
3
3
  require "fileutils"
4
4
  include FileUtils
@@ -47,7 +47,6 @@ describe "tar backup" do
47
47
  end
48
48
 
49
49
  it "should create backup file" do
50
- puts "Expecting: #{@backup}"
51
50
  File.exists?(@backup).should be_true
52
51
  end
53
52
 
@@ -59,4 +58,4 @@ describe "tar backup" do
59
58
  Dir["#{@dst}/archive/archive-foobar.*"].should == ["#{@dst}/archive/archive-foobar.000001.tar.gz", "#{@dst}/archive/archive-foobar.000002.tar.gz"]
60
59
  end
61
60
 
62
- end
61
+ end
@@ -0,0 +1,7 @@
1
+ require 'rubygems'
2
+ require 'bundler/setup'
3
+ require 'astrails/safe'
4
+
5
+ RSpec.configure do |config|
6
+ config.mock_with :rr
7
+ end
@@ -1,4 +1,4 @@
1
- require File.expand_path(File.dirname(__FILE__) + '/../example_helper')
1
+ require 'spec_helper'
2
2
 
3
3
  describe Astrails::Safe::Archive do
4
4
 
@@ -0,0 +1,177 @@
1
+ require 'spec_helper'
2
+
3
+ describe Astrails::Safe::Cloudfiles do
4
+
5
+ def def_config
6
+ {
7
+ :cloudfiles => {
8
+ :container => "_container",
9
+ :user => "_user",
10
+ :api_key => "_api_key",
11
+ },
12
+ :keep => {
13
+ :cloudfiles => 2
14
+ }
15
+ }
16
+ end
17
+
18
+ def def_backup(extra = {})
19
+ {
20
+ :kind => "_kind",
21
+ :filename => "/backup/somewhere/_kind-_id.NOW.bar",
22
+ :extension => ".bar",
23
+ :id => "_id",
24
+ :timestamp => "NOW"
25
+ }.merge(extra)
26
+ end
27
+
28
+ def cloudfiles(config = def_config, backup = def_backup)
29
+ Astrails::Safe::Cloudfiles.new(
30
+ Astrails::Safe::Config::Node.new(nil, config),
31
+ Astrails::Safe::Backup.new(backup)
32
+ )
33
+ end
34
+
35
+ describe :cleanup do
36
+
37
+ before(:each) do
38
+ @cloudfiles = cloudfiles
39
+
40
+ @files = [4,1,3,2].map { |i| "aaaaa#{i}" }
41
+
42
+ @container = "container"
43
+
44
+ stub(@container).objects(:prefix => "_kind/_id/_kind-_id.") { @files }
45
+ stub(@container).delete_object(anything)
46
+
47
+ stub(CloudFiles::Connection).
48
+ new('_user', '_api_key', true, false).stub!.
49
+ container('_container') {@container}
50
+ end
51
+
52
+ it "should check [:keep, :cloudfiles]" do
53
+ @cloudfiles.config[:keep].data["cloudfiles"] = nil
54
+ dont_allow(@cloudfiles.backup).filename
55
+ @cloudfiles.send :cleanup
56
+ end
57
+
58
+ it "should delete extra files" do
59
+ mock(@container).delete_object('aaaaa1')
60
+ mock(@container).delete_object('aaaaa2')
61
+ @cloudfiles.send :cleanup
62
+ end
63
+
64
+ end
65
+
66
+ describe :active do
67
+ before(:each) do
68
+ @cloudfiles = cloudfiles
69
+ end
70
+
71
+ it "should be true when all params are set" do
72
+ @cloudfiles.should be_active
73
+ end
74
+
75
+ it "should be false if container is missing" do
76
+ @cloudfiles.config[:cloudfiles].data["container"] = nil
77
+ @cloudfiles.should_not be_active
78
+ end
79
+
80
+ it "should be false if user is missing" do
81
+ @cloudfiles.config[:cloudfiles].data["user"] = nil
82
+ @cloudfiles.should_not be_active
83
+ end
84
+
85
+ it "should be false if api_key is missing" do
86
+ @cloudfiles.config[:cloudfiles].data["api_key"] = nil
87
+ @cloudfiles.should_not be_active
88
+ end
89
+ end
90
+
91
+ describe :path do
92
+ before(:each) do
93
+ @cloudfiles = cloudfiles
94
+ end
95
+ it "should use cloudfiles/path 1st" do
96
+ @cloudfiles.config[:cloudfiles].data["path"] = "cloudfiles_path"
97
+ @cloudfiles.config[:local] = {:path => "local_path"}
98
+ @cloudfiles.send(:path).should == "cloudfiles_path"
99
+ end
100
+
101
+ it "should use local/path 2nd" do
102
+ @cloudfiles.config[:local] = {:path => "local_path"}
103
+ @cloudfiles.send(:path).should == "local_path"
104
+ end
105
+
106
+ it "should use constant 3rd" do
107
+ @cloudfiles.send(:path).should == "_kind/_id"
108
+ end
109
+
110
+ end
111
+
112
+ describe :save do
113
+ def add_stubs(*stubs)
114
+ stubs.each do |s|
115
+ case s
116
+ when :connection
117
+ @connection = "connection"
118
+ stub(CloudFiles::Authentication).new
119
+ stub(CloudFiles::Connection).
120
+ new('_user', '_api_key', true, false) {@connection}
121
+ when :file_size
122
+ stub(@cloudfiles).get_file_size("foo") {123}
123
+ when :create_container
124
+ @container = "container"
125
+ stub(@container).create_object("_kind/_id/backup/somewhere/_kind-_id.NOW.bar.bar", true) {@object}
126
+ stub(@connection).create_container {@container}
127
+ when :file_open
128
+ stub(File).open("foo")
129
+ when :cloudfiles_store
130
+ @object = "object"
131
+ stub(@object).write(nil) {true}
132
+ end
133
+ end
134
+ end
135
+
136
+ before(:each) do
137
+ @cloudfiles = cloudfiles(def_config, def_backup(:path => "foo"))
138
+ @full_path = "_kind/_id/backup/somewhere/_kind-_id.NOW.bar.bar"
139
+ end
140
+
141
+ it "should fail if no backup.file is set" do
142
+ @cloudfiles.backup.path = nil
143
+ proc {@cloudfiles.send(:save)}.should raise_error(RuntimeError)
144
+ end
145
+
146
+ it "should establish Cloud Files connection" do
147
+ add_stubs(:connection, :file_size, :create_container, :file_open, :cloudfiles_store)
148
+ @cloudfiles.send(:save)
149
+ end
150
+
151
+ it "should open local file" do
152
+ add_stubs(:connection, :file_size, :create_container, :cloudfiles_store)
153
+ mock(File).open("foo")
154
+ @cloudfiles.send(:save)
155
+ end
156
+
157
+ it "should call write on the cloudfile object with files' descriptor" do
158
+ add_stubs(:connection, :file_size, :create_container, :cloudfiles_store)
159
+ stub(File).open("foo") {"qqq"}
160
+ mock(@object).write("qqq") {true}
161
+ @cloudfiles.send(:save)
162
+ end
163
+
164
+ it "should upload file" do
165
+ add_stubs(:connection, :file_size, :create_container, :file_open, :cloudfiles_store)
166
+ @cloudfiles.send(:save)
167
+ end
168
+
169
+ it "should fail on files bigger then 5G" do
170
+ add_stubs(:connection)
171
+ mock(File).stat("foo").stub!.size {5*1024*1024*1024+1}
172
+ mock(STDERR).puts(anything)
173
+ dont_allow(Benchmark).realtime
174
+ @cloudfiles.send(:save)
175
+ end
176
+ end
177
+ end