akupchanko-astrails-safe 0.3.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (52) hide show
  1. checksums.yaml +7 -0
  2. data/.autotest +3 -0
  3. data/.document +5 -0
  4. data/.gitignore +18 -0
  5. data/.rspec +3 -0
  6. data/CHANGELOG +35 -0
  7. data/Gemfile +7 -0
  8. data/LICENSE.txt +22 -0
  9. data/README.markdown +250 -0
  10. data/Rakefile +8 -0
  11. data/TODO +31 -0
  12. data/akupchanko-astrails-safe.gemspec +35 -0
  13. data/bin/astrails-safe +64 -0
  14. data/lib/astrails/safe.rb +68 -0
  15. data/lib/astrails/safe/archive.rb +24 -0
  16. data/lib/astrails/safe/backup.rb +20 -0
  17. data/lib/astrails/safe/cloudfiles.rb +77 -0
  18. data/lib/astrails/safe/config/builder.rb +90 -0
  19. data/lib/astrails/safe/config/node.rb +72 -0
  20. data/lib/astrails/safe/ftp.rb +104 -0
  21. data/lib/astrails/safe/gpg.rb +46 -0
  22. data/lib/astrails/safe/gzip.rb +25 -0
  23. data/lib/astrails/safe/local.rb +51 -0
  24. data/lib/astrails/safe/mongodump.rb +23 -0
  25. data/lib/astrails/safe/mysqldump.rb +32 -0
  26. data/lib/astrails/safe/pgdump.rb +36 -0
  27. data/lib/astrails/safe/pipe.rb +17 -0
  28. data/lib/astrails/safe/s3.rb +80 -0
  29. data/lib/astrails/safe/sftp.rb +88 -0
  30. data/lib/astrails/safe/sink.rb +35 -0
  31. data/lib/astrails/safe/source.rb +47 -0
  32. data/lib/astrails/safe/stream.rb +32 -0
  33. data/lib/astrails/safe/svndump.rb +13 -0
  34. data/lib/astrails/safe/tmp_file.rb +48 -0
  35. data/lib/astrails/safe/version.rb +5 -0
  36. data/lib/extensions/mktmpdir.rb +45 -0
  37. data/spec/astrails/safe/archive_spec.rb +67 -0
  38. data/spec/astrails/safe/cloudfiles_spec.rb +175 -0
  39. data/spec/astrails/safe/config_spec.rb +307 -0
  40. data/spec/astrails/safe/gpg_spec.rb +148 -0
  41. data/spec/astrails/safe/gzip_spec.rb +64 -0
  42. data/spec/astrails/safe/local_spec.rb +109 -0
  43. data/spec/astrails/safe/mongodump_spec.rb +54 -0
  44. data/spec/astrails/safe/mysqldump_spec.rb +83 -0
  45. data/spec/astrails/safe/pgdump_spec.rb +45 -0
  46. data/spec/astrails/safe/s3_spec.rb +168 -0
  47. data/spec/astrails/safe/svndump_spec.rb +39 -0
  48. data/spec/integration/archive_integration_spec.rb +89 -0
  49. data/spec/integration/cleanup_spec.rb +62 -0
  50. data/spec/spec_helper.rb +8 -0
  51. data/templates/script.rb +183 -0
  52. metadata +178 -0
@@ -0,0 +1,46 @@
1
+ module Astrails
2
+ module Safe
3
+ class Gpg < Pipe
4
+
5
+ protected
6
+
7
+ def post_process
8
+ @backup.compressed = true
9
+ end
10
+
11
+ def pipe
12
+ command = config[:gpg, :command] || 'gpg'
13
+ if key
14
+ "|#{command} #{config[:gpg, :options]} -e -r #{key}"
15
+ elsif password
16
+ "|#{command} #{config[:gpg, :options]} -c --passphrase-file #{gpg_password_file(password)}"
17
+ end
18
+ end
19
+
20
+ def extension
21
+ ".gpg"
22
+ end
23
+
24
+ def active?
25
+ raise RuntimeError, "can't use both gpg password and pubkey" if key && password
26
+
27
+ !!(password || key)
28
+ end
29
+
30
+ private
31
+
32
+ def password
33
+ @password ||= config[:gpg, :password]
34
+ end
35
+
36
+ def key
37
+ @key ||= config[:gpg, :key]
38
+ end
39
+
40
+ def gpg_password_file(pass)
41
+ return "TEMP_GENERATED_FILENAME" if dry_run?
42
+ Astrails::Safe::TmpFile.create("gpg-pass") { |file| file.write(pass) }
43
+ end
44
+ end
45
+ end
46
+ end
@@ -0,0 +1,25 @@
1
+ module Astrails
2
+ module Safe
3
+ class Gzip < Pipe
4
+
5
+ protected
6
+
7
+ def post_process
8
+ @backup.compressed = true
9
+ end
10
+
11
+ def pipe
12
+ "|gzip"
13
+ end
14
+
15
+ def extension
16
+ ".gz"
17
+ end
18
+
19
+ def active?
20
+ !@backup.compressed
21
+ end
22
+
23
+ end
24
+ end
25
+ end
@@ -0,0 +1,51 @@
1
+ module Astrails
2
+ module Safe
3
+ class Local < Sink
4
+
5
+ protected
6
+
7
+ def active?
8
+ # S3 can't upload from pipe. it needs to know file size, so we must pass through :local
9
+ # will change once we add SSH/FTP sink
10
+ true
11
+ end
12
+
13
+ def path
14
+ @path ||= File.expand_path(expand(config[:local, :path] || raise(RuntimeError, "missing :local/:path")))
15
+ end
16
+
17
+ def save
18
+ puts "command: #{@backup.command}" if verbose?
19
+
20
+ # FIXME: probably need to change this to smth like @backup.finalize!
21
+ @backup.path = full_path # need to do it outside DRY_RUN so that it will be avialable for S3 DRY_RUN
22
+
23
+ unless dry_run?
24
+ FileUtils.mkdir_p(path) unless File.directory?(path)
25
+ benchmark = Benchmark.realtime do
26
+ system "#{@backup.command}>#{@backup.path}"
27
+ end
28
+ puts("command took " + sprintf("%.2f", benchmark) + " second(s).") if verbose?
29
+ end
30
+
31
+ end
32
+
33
+ def cleanup
34
+ return unless keep = config[:keep, :local]
35
+
36
+ puts "listing files #{base}" if verbose?
37
+
38
+ # TODO: cleanup ALL zero-length files
39
+
40
+ files = Dir["#{base}*"] .
41
+ select{|f| File.file?(f) && File.size(f) > 0} .
42
+ sort
43
+
44
+ cleanup_with_limit(files, keep) do |f|
45
+ puts "removing local file #{f}" if dry_run? || verbose?
46
+ File.unlink(f) unless dry_run?
47
+ end
48
+ end
49
+ end
50
+ end
51
+ end
@@ -0,0 +1,23 @@
1
+ module Astrails
2
+ module Safe
3
+ class Mongodump < Source
4
+
5
+ def command
6
+ opts = []
7
+ opts << "--host #{config[:host]}" if config[:host]
8
+ opts << "-u #{config[:user]}" if config[:user]
9
+ opts << "-p #{config[:password]}" if config[:password]
10
+ opts << "--out #{output_directory}"
11
+
12
+ "mongodump -q \"{xxxx : { \\$ne : 0 } }\" --db #{@id} #{opts.join(" ")} && cd #{output_directory} && tar cf - ."
13
+ end
14
+
15
+ def extension; '.tar'; end
16
+
17
+ protected
18
+ def output_directory
19
+ File.join(TmpFile.tmproot, "mongodump")
20
+ end
21
+ end
22
+ end
23
+ end
@@ -0,0 +1,32 @@
1
+ module Astrails
2
+ module Safe
3
+ class Mysqldump < Source
4
+
5
+ def command
6
+ "mysqldump --defaults-extra-file=#{mysql_password_file} #{config[:options]} #{mysql_skip_tables} #{@id}"
7
+ end
8
+
9
+ def extension; '.sql'; end
10
+
11
+ protected
12
+
13
+ def mysql_password_file
14
+ Astrails::Safe::TmpFile.create("mysqldump") do |file|
15
+ file.puts "[mysqldump]"
16
+ %w/user password socket host port/.each do |k|
17
+ v = config[k]
18
+ # values are quoted if needed
19
+ file.puts "#{k} = #{v.inspect}" if v
20
+ end
21
+ end
22
+ end
23
+
24
+ def mysql_skip_tables
25
+ if skip_tables = config[:skip_tables]
26
+ [*skip_tables].map{ |t| "--ignore-table=#{@id}.#{t}" }.join(" ")
27
+ end
28
+ end
29
+
30
+ end
31
+ end
32
+ end
@@ -0,0 +1,36 @@
1
+ module Astrails
2
+ module Safe
3
+ class Pgdump < Source
4
+
5
+ def command
6
+ if config["password"]
7
+ ENV['PGPASSWORD'] = config["password"]
8
+ else
9
+ ENV['PGPASSWORD'] = nil
10
+ end
11
+ "pg_dump #{postgres_options} #{postgres_username} #{postgres_host} #{postgres_port} #{@id}"
12
+ end
13
+
14
+ def extension; '.sql'; end
15
+
16
+ protected
17
+
18
+ def postgres_options
19
+ config[:options]
20
+ end
21
+
22
+ def postgres_host
23
+ config["host"] && "--host='#{config["host"]}'"
24
+ end
25
+
26
+ def postgres_port
27
+ config["port"] && "--port='#{config["port"]}'"
28
+ end
29
+
30
+ def postgres_username
31
+ config["user"] && "--username='#{config["user"]}'"
32
+ end
33
+
34
+ end
35
+ end
36
+ end
@@ -0,0 +1,17 @@
1
+ module Astrails
2
+ module Safe
3
+ class Pipe < Stream
4
+ # process adds required commands to the current
5
+ # shell command string
6
+ # :active?, :pipe, :extension and :post_process are
7
+ # defined in inheriting pipe classes
8
+ def process
9
+ return unless active?
10
+
11
+ @backup.command << pipe
12
+ @backup.extension << extension
13
+ post_process
14
+ end
15
+ end
16
+ end
17
+ end
@@ -0,0 +1,80 @@
1
+ module Astrails
2
+ module Safe
3
+ class S3 < Sink
4
+ MAX_S3_FILE_SIZE = 5368709120
5
+
6
+ protected
7
+
8
+ def active?
9
+ bucket && key && secret
10
+ end
11
+
12
+ def path
13
+ @path ||= expand(config[:s3, :path] || config[:local, :path] || ":kind/:id")
14
+ end
15
+
16
+ def save
17
+ # FIXME: user friendly error here :)
18
+ raise RuntimeError, "pipe-streaming not supported for S3." unless @backup.path
19
+
20
+ # needed in cleanup even on dry run
21
+ AWS::S3::Base.establish_connection!(:access_key_id => key, :secret_access_key => secret, :use_ssl => true) unless local_only?
22
+
23
+ puts "Uploading #{bucket}:#{full_path}" if verbose? || dry_run?
24
+ unless dry_run? || local_only?
25
+ if File.stat(@backup.path).size > MAX_S3_FILE_SIZE
26
+ STDERR.puts "ERROR: File size exceeds maximum allowed for upload to S3 (#{MAX_S3_FILE_SIZE}): #{@backup.path}"
27
+ return
28
+ end
29
+ benchmark = Benchmark.realtime do
30
+ AWS::S3::Bucket.create(bucket) unless bucket_exists?(bucket)
31
+ File.open(@backup.path) do |file|
32
+ AWS::S3::S3Object.store(full_path, file, bucket)
33
+ end
34
+ end
35
+ puts "...done" if verbose?
36
+ puts("Upload took " + sprintf("%.2f", benchmark) + " second(s).") if verbose?
37
+ end
38
+ end
39
+
40
+ def cleanup
41
+ return if local_only?
42
+
43
+ return unless keep = config[:keep, :s3]
44
+
45
+ puts "listing files: #{bucket}:#{base}*" if verbose?
46
+ files = AWS::S3::Bucket.objects(bucket, :prefix => base, :max_keys => keep * 2)
47
+ puts files.collect {|x| x.key} if verbose?
48
+
49
+ files = files.
50
+ collect {|x| x.key}.
51
+ sort
52
+
53
+ cleanup_with_limit(files, keep) do |f|
54
+ puts "removing s3 file #{bucket}:#{f}" if dry_run? || verbose?
55
+ AWS::S3::Bucket.objects(bucket, :prefix => f)[0].delete unless dry_run? || local_only?
56
+ end
57
+ end
58
+
59
+ def bucket
60
+ config[:s3, :bucket]
61
+ end
62
+
63
+ def key
64
+ config[:s3, :key]
65
+ end
66
+
67
+ def secret
68
+ config[:s3, :secret]
69
+ end
70
+
71
+ private
72
+
73
+ def bucket_exists?(bucket)
74
+ true if AWS::S3::Bucket.find(bucket)
75
+ rescue AWS::S3::NoSuchBucket
76
+ false
77
+ end
78
+ end
79
+ end
80
+ end
@@ -0,0 +1,88 @@
1
+ module Astrails
2
+ module Safe
3
+ class Sftp < Sink
4
+
5
+ protected
6
+
7
+ def active?
8
+ host && user
9
+ end
10
+
11
+ def path
12
+ @path ||= expand(config[:sftp, :path] || config[:local, :path] || ":kind/:id")
13
+ end
14
+
15
+ def save
16
+ raise RuntimeError, "pipe-streaming not supported for SFTP." unless @backup.path
17
+
18
+ puts "Uploading #{host}:#{full_path} via SFTP" if verbose? || dry_run?
19
+
20
+ unless dry_run? || local_only?
21
+ opts = {}
22
+ opts[:password] = password if password
23
+ opts[:port] = port if port
24
+ Net::SFTP.start(host, user, opts) do |sftp|
25
+ puts "Sending #{@backup.path} to #{full_path}" if verbose?
26
+ begin
27
+ sftp.upload! @backup.path, full_path
28
+ rescue Net::SFTP::StatusException
29
+ puts "Ensuring remote path (#{path}) exists" if verbose?
30
+ # mkdir -p
31
+ folders = path.split('/')
32
+ folders.each_index do |i|
33
+ folder = folders[0..i].join('/')
34
+ puts "Creating #{folder} on remote" if verbose?
35
+ sftp.mkdir!(folder) rescue Net::SFTP::StatusException
36
+ end
37
+ retry
38
+ end
39
+ end
40
+ puts "...done" if verbose?
41
+ end
42
+ end
43
+
44
+ def cleanup
45
+ return if local_only? || dry_run?
46
+
47
+ return unless keep = config[:keep, :sftp]
48
+
49
+ puts "listing files: #{host}:#{base}*" if verbose?
50
+ opts = {}
51
+ opts[:password] = password if password
52
+ opts[:port] = port if port
53
+ Net::SFTP.start(host, user, opts) do |sftp|
54
+ files = sftp.dir.glob(path, File.basename("#{base}*"))
55
+
56
+ puts files.collect {|x| x.name } if verbose?
57
+
58
+ files = files.
59
+ collect {|x| x.name }.
60
+ sort
61
+
62
+ cleanup_with_limit(files, keep) do |f|
63
+ file = File.join(path, f)
64
+ puts "removing sftp file #{host}:#{file}" if dry_run? || verbose?
65
+ sftp.remove!(file) unless dry_run? || local_only?
66
+ end
67
+ end
68
+ end
69
+
70
+ def host
71
+ config[:sftp, :host]
72
+ end
73
+
74
+ def user
75
+ config[:sftp, :user]
76
+ end
77
+
78
+ def password
79
+ config[:sftp, :password]
80
+ end
81
+
82
+ def port
83
+ config[:sftp, :port]
84
+ end
85
+
86
+ end
87
+ end
88
+ end
@@ -0,0 +1,35 @@
1
+ module Astrails
2
+ module Safe
3
+ class Sink < Stream
4
+
5
+ def process
6
+ return unless active?
7
+
8
+ save
9
+ cleanup
10
+ end
11
+
12
+ protected
13
+
14
+ # path is defined in subclass
15
+ # base is used in 'cleanup' to find all files that begin with base. the '.'
16
+ # at the end is essential to distinguish b/w foo.* and foobar.* archives for example
17
+ def base
18
+ @base ||= File.join(path, File.basename(@backup.filename).split(".").first + '.')
19
+ end
20
+
21
+ def full_path
22
+ @full_path ||= File.join(path, @backup.filename) + @backup.extension
23
+ end
24
+
25
+ # call block on files to be removed (all except for the LAST 'limit' files
26
+ def cleanup_with_limit(files, limit, &block)
27
+ return unless files.size > limit
28
+
29
+ to_remove = files[0..(files.size - limit - 1)]
30
+ # TODO: validate here
31
+ to_remove.each(&block)
32
+ end
33
+ end
34
+ end
35
+ end
@@ -0,0 +1,47 @@
1
+ module Astrails
2
+ module Safe
3
+ class Source < Stream
4
+
5
+ attr_accessor :id
6
+ def initialize(id, config)
7
+ @id, @config = id.to_s, config
8
+ end
9
+
10
+ def timestamp
11
+ Time.now.strftime("%y%m%d-%H%M")
12
+ end
13
+
14
+ def kind
15
+ self.class.human_name
16
+ end
17
+
18
+ def filename
19
+ @filename ||= expand(":kind-:id.:timestamp")
20
+ end
21
+
22
+ def backup
23
+ return @backup if @backup
24
+ @backup = Backup.new(
25
+ :id => @id,
26
+ :kind => kind,
27
+ :extension => extension,
28
+ :command => command,
29
+ :timestamp => timestamp
30
+ )
31
+ # can't do this in the initializer hash above since
32
+ # filename() calls expand() which requires @backup
33
+ # FIXME: move expansion to the backup (last step in ctor) assign :tags here
34
+ @backup.filename = filename
35
+ @backup
36
+ end
37
+
38
+ protected
39
+
40
+ def self.human_name
41
+ name.split('::').last.downcase
42
+ end
43
+
44
+ end
45
+ end
46
+ end
47
+