akupchanko-astrails-safe 0.3.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (52) hide show
  1. checksums.yaml +7 -0
  2. data/.autotest +3 -0
  3. data/.document +5 -0
  4. data/.gitignore +18 -0
  5. data/.rspec +3 -0
  6. data/CHANGELOG +35 -0
  7. data/Gemfile +7 -0
  8. data/LICENSE.txt +22 -0
  9. data/README.markdown +250 -0
  10. data/Rakefile +8 -0
  11. data/TODO +31 -0
  12. data/akupchanko-astrails-safe.gemspec +35 -0
  13. data/bin/astrails-safe +64 -0
  14. data/lib/astrails/safe.rb +68 -0
  15. data/lib/astrails/safe/archive.rb +24 -0
  16. data/lib/astrails/safe/backup.rb +20 -0
  17. data/lib/astrails/safe/cloudfiles.rb +77 -0
  18. data/lib/astrails/safe/config/builder.rb +90 -0
  19. data/lib/astrails/safe/config/node.rb +72 -0
  20. data/lib/astrails/safe/ftp.rb +104 -0
  21. data/lib/astrails/safe/gpg.rb +46 -0
  22. data/lib/astrails/safe/gzip.rb +25 -0
  23. data/lib/astrails/safe/local.rb +51 -0
  24. data/lib/astrails/safe/mongodump.rb +23 -0
  25. data/lib/astrails/safe/mysqldump.rb +32 -0
  26. data/lib/astrails/safe/pgdump.rb +36 -0
  27. data/lib/astrails/safe/pipe.rb +17 -0
  28. data/lib/astrails/safe/s3.rb +80 -0
  29. data/lib/astrails/safe/sftp.rb +88 -0
  30. data/lib/astrails/safe/sink.rb +35 -0
  31. data/lib/astrails/safe/source.rb +47 -0
  32. data/lib/astrails/safe/stream.rb +32 -0
  33. data/lib/astrails/safe/svndump.rb +13 -0
  34. data/lib/astrails/safe/tmp_file.rb +48 -0
  35. data/lib/astrails/safe/version.rb +5 -0
  36. data/lib/extensions/mktmpdir.rb +45 -0
  37. data/spec/astrails/safe/archive_spec.rb +67 -0
  38. data/spec/astrails/safe/cloudfiles_spec.rb +175 -0
  39. data/spec/astrails/safe/config_spec.rb +307 -0
  40. data/spec/astrails/safe/gpg_spec.rb +148 -0
  41. data/spec/astrails/safe/gzip_spec.rb +64 -0
  42. data/spec/astrails/safe/local_spec.rb +109 -0
  43. data/spec/astrails/safe/mongodump_spec.rb +54 -0
  44. data/spec/astrails/safe/mysqldump_spec.rb +83 -0
  45. data/spec/astrails/safe/pgdump_spec.rb +45 -0
  46. data/spec/astrails/safe/s3_spec.rb +168 -0
  47. data/spec/astrails/safe/svndump_spec.rb +39 -0
  48. data/spec/integration/archive_integration_spec.rb +89 -0
  49. data/spec/integration/cleanup_spec.rb +62 -0
  50. data/spec/spec_helper.rb +8 -0
  51. data/templates/script.rb +183 -0
  52. metadata +178 -0
@@ -0,0 +1,68 @@
1
+ require "astrails/safe/version"
2
+
3
+ require "aws/s3"
4
+ require "cloudfiles"
5
+ require 'net/sftp'
6
+ require 'net/ftp'
7
+ require 'fileutils'
8
+ require 'benchmark'
9
+
10
+ require 'tempfile'
11
+ require 'extensions/mktmpdir'
12
+
13
+ require 'astrails/safe/tmp_file'
14
+
15
+ require 'astrails/safe/config/node'
16
+ require 'astrails/safe/config/builder'
17
+
18
+ require 'astrails/safe/stream'
19
+
20
+ require 'astrails/safe/backup'
21
+
22
+ require 'astrails/safe/source'
23
+ require 'astrails/safe/mysqldump'
24
+ require 'astrails/safe/pgdump'
25
+ require 'astrails/safe/archive'
26
+ require 'astrails/safe/svndump'
27
+ require 'astrails/safe/mongodump'
28
+
29
+ require 'astrails/safe/pipe'
30
+ require 'astrails/safe/gpg'
31
+ require 'astrails/safe/gzip'
32
+
33
+ require 'astrails/safe/sink'
34
+ require 'astrails/safe/local'
35
+ require 'astrails/safe/s3'
36
+ require 'astrails/safe/cloudfiles'
37
+ require 'astrails/safe/sftp'
38
+ require 'astrails/safe/ftp'
39
+
40
+ module Astrails
41
+ module Safe
42
+ ROOT = File.join(File.dirname(__FILE__), "..", "..")
43
+
44
+ def safe(&block)
45
+ Config::Node.new(&block)
46
+ end
47
+
48
+ def process(config)
49
+
50
+ [[Mysqldump, [:mysqldump, :databases]],
51
+ [Pgdump, [:pgdump, :databases]],
52
+ [Mongodump, [:mongodump, :databases]],
53
+ [Archive, [:tar, :archives]],
54
+ [Svndump, [:svndump, :repos]]
55
+ ].each do |klass, path|
56
+ if collection = config[*path]
57
+ collection.each do |name, c|
58
+ klass.new(name, c).backup.run(c, :gpg, :gzip, :local, :s3, :cloudfiles, :sftp, :ftp)
59
+ end
60
+ end
61
+ end
62
+
63
+ Astrails::Safe::TmpFile.cleanup
64
+ end
65
+ module_function :safe
66
+ module_function :process
67
+ end
68
+ end
@@ -0,0 +1,24 @@
1
+ module Astrails
2
+ module Safe
3
+ class Archive < Source
4
+
5
+ def command
6
+ "tar -cf - #{config[:options]} #{tar_exclude_files} #{tar_files}"
7
+ end
8
+
9
+ def extension; '.tar'; end
10
+
11
+ protected
12
+
13
+ def tar_exclude_files
14
+ [*config[:exclude]].compact.map{|x| "--exclude=#{x}"}.join(" ")
15
+ end
16
+
17
+ def tar_files
18
+ raise RuntimeError, "missing files for tar" unless config[:files]
19
+ [*config[:files]].map{|s| s.strip}.join(" ")
20
+ end
21
+
22
+ end
23
+ end
24
+ end
@@ -0,0 +1,20 @@
1
+ module Astrails
2
+ module Safe
3
+ class Backup
4
+ attr_accessor :id, :kind, :filename, :extension, :command, :compressed, :timestamp, :path
5
+ def initialize(opts = {})
6
+ opts.each do |k, v|
7
+ self.send("#{k}=", v)
8
+ end
9
+ end
10
+
11
+ def run(config, *mods)
12
+ mods.each do |mod|
13
+ mod = mod.to_s
14
+ mod[0] = mod[0..0].upcase
15
+ Astrails::Safe.const_get(mod).new(config, self).process
16
+ end
17
+ end
18
+ end
19
+ end
20
+ end
@@ -0,0 +1,77 @@
1
+ module Astrails
2
+ module Safe
3
+ class Cloudfiles < Sink
4
+ MAX_CLOUDFILES_FILE_SIZE = 5368709120
5
+
6
+ protected
7
+
8
+ def active?
9
+ container && user && api_key
10
+ end
11
+
12
+ def path
13
+ @path ||= expand(config[:cloudfiles, :path] || config[:local, :path] || ":kind/:id")
14
+ end
15
+
16
+ # UGLY: we need this function for the reason that
17
+ # we can't double mock on ruby 1.9.2, duh!
18
+ # so we created this func to mock it all together
19
+ def get_file_size(path)
20
+ File.stat(path).size
21
+ end
22
+
23
+ def save
24
+ raise RuntimeError, "pipe-streaming not supported for S3." unless @backup.path
25
+
26
+ # needed in cleanup even on dry run
27
+ cf = CloudFiles::Connection.new(user, api_key, true, service_net) unless local_only?
28
+ puts "Uploading #{container}:#{full_path} from #{@backup.path}" if verbose? || dry_run?
29
+ unless dry_run? || local_only?
30
+ if get_file_size(@backup.path) > MAX_CLOUDFILES_FILE_SIZE
31
+ STDERR.puts "ERROR: File size exceeds maximum allowed for upload to Cloud Files (#{MAX_CLOUDFILES_FILE_SIZE}): #{@backup.path}"
32
+ return
33
+ end
34
+ benchmark = Benchmark.realtime do
35
+ cf_container = cf.create_container(container)
36
+ o = cf_container.create_object(full_path,true)
37
+ o.write(File.open(@backup.path))
38
+ end
39
+ puts "...done" if verbose?
40
+ puts("Upload took " + sprintf("%.2f", benchmark) + " second(s).") if verbose?
41
+ end
42
+ end
43
+
44
+ def cleanup
45
+ return if local_only?
46
+
47
+ return unless keep = config[:keep, :cloudfiles]
48
+
49
+ puts "listing files: #{container}:#{base}*" if verbose?
50
+ cf = CloudFiles::Connection.new(user, api_key, true, service_net) unless local_only?
51
+ cf_container = cf.container(container)
52
+ files = cf_container.objects(:prefix => base).sort
53
+
54
+ cleanup_with_limit(files, keep) do |f|
55
+ puts "removing Cloud File #{container}:#{f}" if dry_run? || verbose?
56
+ cf_container.delete_object(f) unless dry_run? || local_only?
57
+ end
58
+ end
59
+
60
+ def container
61
+ config[:cloudfiles, :container]
62
+ end
63
+
64
+ def user
65
+ config[:cloudfiles, :user]
66
+ end
67
+
68
+ def api_key
69
+ config[:cloudfiles, :api_key]
70
+ end
71
+
72
+ def service_net
73
+ config[:cloudfiles, :service_net] || false
74
+ end
75
+ end
76
+ end
77
+ end
@@ -0,0 +1,90 @@
1
+ module Astrails
2
+ module Safe
3
+ module Config
4
+ class Builder
5
+
6
+ def initialize(node, data = {})
7
+ @node = node
8
+ data.each { |k, v| self.send k, v }
9
+ end
10
+
11
+
12
+ class << self
13
+ def simple_value(*names)
14
+ names.each do |m|
15
+ define_method(m) do |value|
16
+ ensure_uniq(m)
17
+ @node.set m, value
18
+ end
19
+ end
20
+ end
21
+
22
+ def multi_value(*names)
23
+ names.each do |m|
24
+ define_method(m) do |value|
25
+ value = value.map(&:to_s) if value.is_a?(Array)
26
+ @node.set_multi m, value
27
+ end
28
+ end
29
+ end
30
+
31
+ def hash_value(*names)
32
+ names.each do |m|
33
+ define_method(m) do |data = {}, &block|
34
+ ensure_uniq(m)
35
+ ensure_hash(m, data)
36
+ @node.set m, Node.new(@node, data || {}, &block)
37
+ end
38
+ end
39
+ end
40
+
41
+ def mixed_value(*names)
42
+ names.each do |m|
43
+ define_method(m) do |data={}, &block|
44
+ ensure_uniq(m)
45
+ if data.is_a?(Hash) || block
46
+ ensure_hash(m, data) if block
47
+ @node.set m, Node.new(@node, data, &block)
48
+ else
49
+ @node.set m, data
50
+ end
51
+ end
52
+ end
53
+ end
54
+
55
+ def collection(*names)
56
+ names.each do |m|
57
+ define_method(m) do |id, data={}, &block|
58
+ raise "bad collection id: #{id.inspect}" unless id
59
+ ensure_hash(m, data)
60
+
61
+ name = "#{m}s"
62
+ collection = @node.get(name) || @node.set(name, Node.new(@node, {}))
63
+ collection.set id, Node.new(collection, data, &block)
64
+ end
65
+ end
66
+ end
67
+ end
68
+
69
+ simple_value :verbose, :dry_run, :local_only, :path, :command,
70
+ :options, :user, :host, :port, :password, :key, :secret, :bucket,
71
+ :api_key, :container, :socket, :service_net, :repo_path
72
+ multi_value :skip_tables, :exclude, :files
73
+ hash_value :mysqldump, :tar, :gpg, :keep, :pgdump, :tar, :svndump,
74
+ :sftp, :ftp, :mongodump
75
+ mixed_value :s3, :local, :cloudfiles
76
+ collection :database, :archive, :repo
77
+
78
+ private
79
+
80
+ def ensure_uniq(m)
81
+ raise(ArgumentError, "duplicate value for '#{m}'") if @node.get(m)
82
+ end
83
+
84
+ def ensure_hash(k, v)
85
+ raise "#{k}: hash expected: #{v.inspect}" unless v.is_a?(Hash)
86
+ end
87
+ end
88
+ end
89
+ end
90
+ end
@@ -0,0 +1,72 @@
1
+ require 'astrails/safe/config/builder'
2
+ module Astrails
3
+ module Safe
4
+ module Config
5
+ class Node
6
+ attr_reader :parent, :data
7
+
8
+ def initialize(parent = nil, data = {}, &block)
9
+ @parent = parent
10
+ @data = {}
11
+ merge data, &block
12
+ end
13
+
14
+ def merge data = {}, &block
15
+ builder = Builder.new(self, data)
16
+ builder.instance_eval(&block) if block
17
+ self
18
+ end
19
+
20
+ # looks for the path from this node DOWN. will not delegate to parent
21
+ def get(*path)
22
+ key = path.shift
23
+ value = @data[key.to_s]
24
+ return value if (nil != value) && path.empty?
25
+
26
+ value && value.get(*path)
27
+ end
28
+
29
+ # recursive find
30
+ # starts at the node and continues to the parent
31
+ def find(*path)
32
+ get(*path) || @parent && @parent.find(*path)
33
+ end
34
+ alias :[] :find
35
+
36
+ def set_multi(key, value)
37
+ @data[key.to_s] ||= []
38
+ @data[key.to_s].concat [*value]
39
+ end
40
+
41
+ def set(key, value)
42
+ @data[key.to_s] = value
43
+ end
44
+ alias :[]= :set
45
+
46
+ def each(&block)
47
+ @data.each(&block)
48
+ end
49
+ include Enumerable
50
+
51
+ def to_hash
52
+ @data.keys.inject({}) do |res, key|
53
+ value = @data[key]
54
+ res[key] = value.is_a?(Node) ? value.to_hash : value
55
+ res
56
+ end
57
+ end
58
+
59
+ def dump(indent = "")
60
+ @data.each do |key, value|
61
+ if value.is_a?(Node)
62
+ puts "#{indent}#{key}:"
63
+ value.dump(indent + " ")
64
+ else
65
+ puts "#{indent}#{key}: #{value.inspect}"
66
+ end
67
+ end
68
+ end
69
+ end
70
+ end
71
+ end
72
+ end
@@ -0,0 +1,104 @@
1
+ module Astrails
2
+ module Safe
3
+ class Ftp < Sink
4
+
5
+ protected
6
+
7
+ def active?
8
+ host && user
9
+ end
10
+
11
+ def path
12
+ @path ||= expand(config[:ftp, :path] || config[:local, :path] || ":kind/:id")
13
+ end
14
+
15
+ def save
16
+ raise RuntimeError, "pipe-streaming not supported for FTP." unless @backup.path
17
+
18
+ puts "Uploading #{host}:#{full_path} via FTP" if verbose? || dry_run?
19
+
20
+ unless dry_run? || local_only?
21
+ if !port
22
+ port = 21
23
+ end
24
+ Net::FTP.open(host) do |ftp|
25
+ ftp.connect(host, port)
26
+ ftp.login(user, password)
27
+
28
+ dir = File.dirname(full_path)
29
+ parts = dir.split("/")
30
+ growing_path = ""
31
+ for part in parts
32
+ next if part == ""
33
+ if growing_path == ""
34
+ growing_path = part
35
+ else
36
+ growing_path = File.join(growing_path, part)
37
+ end
38
+ puts "Trying to create remote directory (#{growing_path})" if verbose?
39
+ begin
40
+ ftp.mkdir(growing_path)
41
+ rescue Net::FTPPermError
42
+ puts "Remote directory (#{growing_path}) exists, or no enough permissions" if verbose?
43
+ end
44
+ end
45
+
46
+ puts "Sending #{@backup.path} to #{full_path}" if verbose?
47
+ begin
48
+ ftp.put(@backup.path, full_path)
49
+ rescue Net::FTPPermError
50
+ puts "Ensuring remote path (#{path}) exists" if verbose?
51
+ end
52
+ end
53
+ puts "...done" if verbose?
54
+ end
55
+ end
56
+
57
+ def cleanup
58
+ return if local_only? || dry_run?
59
+
60
+ return unless keep = config[:keep, :ftp]
61
+
62
+ puts "listing files: #{host}:#{base}*" if verbose?
63
+ if !port
64
+ port = 21
65
+ end
66
+ Net::FTP.open(host) do |ftp|
67
+ ftp.connect(host, port)
68
+ ftp.login(user, password)
69
+ files = ftp.nlst(path)
70
+ pattern = File.basename("#{base}")
71
+ files = files.reject{ |x| !x.start_with?(pattern)}
72
+ puts files.collect {|x| x} if verbose?
73
+
74
+ files = files.
75
+ collect {|x| x }.
76
+ sort
77
+
78
+ cleanup_with_limit(files, keep) do |f|
79
+ file = File.path(f)
80
+ puts "removing ftp file #{host}:#{file}" if dry_run? || verbose?
81
+ ftp.delete(file) unless dry_run? || local_only?
82
+ end
83
+ end
84
+ end
85
+
86
+ def host
87
+ config[:ftp, :host]
88
+ end
89
+
90
+ def user
91
+ config[:ftp, :user]
92
+ end
93
+
94
+ def password
95
+ config[:ftp, :password]
96
+ end
97
+
98
+ def port
99
+ config[:ftp, :port]
100
+ end
101
+
102
+ end
103
+ end
104
+ end