markmansour-safe 0.1.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,68 @@
1
+ module Astrails
2
+ module Safe
3
+ class S3 < Sink
4
+
5
+ protected
6
+
7
+ def active?
8
+ bucket && key && secret
9
+ end
10
+
11
+ def prefix
12
+ @prefix ||= expand(config[:s3, :path] || expand(config[:local, :path] || ":kind/:id"))
13
+ end
14
+
15
+ def save
16
+ # needed in cleanup even on dry run
17
+ AWS::S3::Base.establish_connection!(:access_key_id => key, :secret_access_key => secret, :use_ssl => true) unless $LOCAL
18
+
19
+ file = @parent.open
20
+ puts "Uploading #{bucket}:#{path}" if $_VERBOSE || $DRY_RUN
21
+ unless $DRY_RUN || $LOCAL
22
+ AWS::S3::Bucket.create(bucket)
23
+ AWS::S3::S3Object.store(path, file, bucket)
24
+ puts "...done" if $_VERBOSE
25
+ end
26
+ file.close if file
27
+
28
+ end
29
+
30
+ def cleanup
31
+
32
+ return if $LOCAL
33
+
34
+ return unless keep = @config[:keep, :s3]
35
+
36
+ bucket = @config[:s3, :bucket]
37
+
38
+ base = File.basename(filename).split(".").first
39
+
40
+ puts "listing files in #{bucket}:#{prefix}/#{base}"
41
+ files = AWS::S3::Bucket.objects(bucket, :prefix => "#{prefix}/#{base}", :max_keys => keep * 2)
42
+ puts files.collect {|x| x.key} if $_VERBOSE
43
+
44
+ files = files.
45
+ collect {|x| x.key}.
46
+ sort
47
+
48
+ cleanup_with_limit(files, keep) do |f|
49
+ puts "removing s3 file #{bucket}:#{f}" if $DRY_RUN || $_VERBOSE
50
+ AWS::S3::Bucket.find(bucket)[f].delete unless $DRY_RUN || $LOCAL
51
+ end
52
+ end
53
+
54
+ def bucket
55
+ config[:s3, :bucket]
56
+ end
57
+
58
+ def key
59
+ config[:s3, :key]
60
+ end
61
+
62
+ def secret
63
+ config[:s3, :secret]
64
+ end
65
+
66
+ end
67
+ end
68
+ end
@@ -0,0 +1,33 @@
1
+ module Astrails
2
+ module Safe
3
+ class Sink < Stream
4
+
5
+ def run
6
+ if active?
7
+ save
8
+ cleanup
9
+ else
10
+ @parent.run
11
+ end
12
+ end
13
+
14
+ protected
15
+
16
+ # prefix is defined in subclass
17
+ def path
18
+ @path ||= File.join(prefix, filename)
19
+ end
20
+
21
+ # call block on files to be removed (all except for the LAST 'limit' files
22
+ def cleanup_with_limit(files, limit, &block)
23
+ return unless files.size > limit
24
+
25
+ to_remove = files[0..(files.size - limit - 1)]
26
+ # TODO: validate here
27
+ to_remove.each(&block)
28
+ end
29
+
30
+ end
31
+ end
32
+ end
33
+
@@ -0,0 +1,31 @@
1
+ module Astrails
2
+ module Safe
3
+ class Source < Stream
4
+
5
+ def initialize(id, config)
6
+ @id, @config = id, config
7
+ end
8
+
9
+ def filename
10
+ @filename ||= expand(":kind-:id.:timestamp#{extension}")
11
+ end
12
+
13
+ # process each config key as source (with full pipe)
14
+ def self.run(config)
15
+ unless config
16
+ puts "No configuration found for #{human_name}"
17
+ return
18
+ end
19
+
20
+ config.each do |key, value|
21
+ stream = [Gpg, Gzip, Local, S3].inject(new(key, value)) do |res, klass|
22
+ klass.new(res)
23
+ end
24
+ stream.run
25
+ end
26
+ end
27
+
28
+ end
29
+ end
30
+ end
31
+
@@ -0,0 +1,45 @@
1
+ module Astrails
2
+ module Safe
3
+ class Stream
4
+
5
+ def initialize(parent)
6
+ @parent = parent
7
+ end
8
+
9
+ def id
10
+ @id ||= @parent.id
11
+ end
12
+
13
+ def config
14
+ @config ||= @parent.config
15
+ end
16
+
17
+ def filename
18
+ @parent.filename
19
+ end
20
+
21
+ def compressed?
22
+ @parent && @parent.compressed?
23
+ end
24
+
25
+ protected
26
+
27
+ def self.human_name
28
+ name.split('::').last.downcase
29
+ end
30
+
31
+ def kind
32
+ @parent ? @parent.kind : self.class.human_name
33
+ end
34
+
35
+ def expand(path)
36
+ path .
37
+ gsub(/:kind\b/, kind) .
38
+ gsub(/:id\b/, id) .
39
+ gsub(/:timestamp\b/, timestamp)
40
+ end
41
+
42
+ end
43
+ end
44
+ end
45
+
@@ -0,0 +1,25 @@
1
+ require 'tmpdir'
2
+ module Astrails
3
+ module Safe
4
+ module TmpFile
5
+ @KEEP_FILES = []
6
+ TMPDIR = Dir.mktmpdir
7
+
8
+ def self.cleanup
9
+ FileUtils.remove_entry_secure TMPDIR
10
+ end
11
+
12
+ def self.create(name)
13
+ # create temp directory
14
+
15
+ file = Tempfile.new(name, TMPDIR)
16
+
17
+ yield file
18
+
19
+ file.close
20
+ @KEEP_FILES << file # so that it will not get gcollected and removed from filesystem until the end
21
+ file.path
22
+ end
23
+ end
24
+ end
25
+ end
@@ -0,0 +1,45 @@
1
+ require 'tmpdir'
2
+
3
+ unless Dir.respond_to?(:mktmpdir)
4
+ # backward compat for 1.8.6
5
+ class Dir
6
+ def Dir.mktmpdir(prefix_suffix=nil, tmpdir=nil)
7
+ case prefix_suffix
8
+ when nil
9
+ prefix = "d"
10
+ suffix = ""
11
+ when String
12
+ prefix = prefix_suffix
13
+ suffix = ""
14
+ when Array
15
+ prefix = prefix_suffix[0]
16
+ suffix = prefix_suffix[1]
17
+ else
18
+ raise ArgumentError, "unexpected prefix_suffix: #{prefix_suffix.inspect}"
19
+ end
20
+ tmpdir ||= Dir.tmpdir
21
+ t = Time.now.strftime("%Y%m%d")
22
+ n = nil
23
+ begin
24
+ path = "#{tmpdir}/#{prefix}#{t}-#{$$}-#{rand(0x100000000).to_s(36)}"
25
+ path << "-#{n}" if n
26
+ path << suffix
27
+ Dir.mkdir(path, 0700)
28
+ rescue Errno::EEXIST
29
+ n ||= 0
30
+ n += 1
31
+ retry
32
+ end
33
+
34
+ if block_given?
35
+ begin
36
+ yield path
37
+ ensure
38
+ FileUtils.remove_entry_secure path
39
+ end
40
+ else
41
+ path
42
+ end
43
+ end
44
+ end
45
+ end
@@ -0,0 +1,122 @@
1
+ safe do
2
+
3
+ # backup file path (not including filename)
4
+ # supported substitutions:
5
+ # :kind -> backup 'engine' kind, e.g. "mysqldump" or "archive"
6
+ # :id -> backup 'id', e.g. "blog", "production", etc.
7
+ # :timestamp -> current run timestamp (same for all the backups in the same 'run')
8
+ # you can set separate :path for all backups (or once globally here)
9
+ local do
10
+ path "/backup/:kind/"
11
+ end
12
+
13
+ ## uncomment to enable uploads to Amazon S3
14
+ ## Amazon S3 auth (optional)
15
+ ## don't forget to add :s3 to the 'store' list
16
+ # s3 do
17
+ # key YOUR_S3_KEY
18
+ # secret YOUR_S3_SECRET
19
+ # bucket S3_BUCKET
20
+ # # path for uploads to S3. supports same substitution like :local/:path
21
+ # path ":kind/" # this is default
22
+ # end
23
+
24
+ ## alternative style:
25
+ # s3 :key => YOUR_S3_KEY, :secret => YOUR_S3_SECRET, :bucket => S3_BUCKET
26
+
27
+ ## uncomment to enable GPG encryption.
28
+ ## Note: you can use public 'key' or symmetric password but not both!
29
+ # gpg do
30
+ # # key "backup@astrails.com"
31
+ # password "astrails"
32
+ # end
33
+
34
+ ## uncomment to enable backup rotation. keep only given number of latest
35
+ ## backups. remove the rest
36
+ # keep do
37
+ # local 4 # keep 4 local backups
38
+ # s3 20 # keep 20 S3 backups
39
+ # end
40
+
41
+ # backup mysql databases with mysqldump
42
+ mysqldump do
43
+ # you can override any setting from parent in a child:
44
+ options "-ceKq --single-transaction --create-options"
45
+
46
+ user "astrails"
47
+ password ""
48
+ # host "localhost"
49
+ # port 3306
50
+ socket "/var/run/mysqld/mysqld.sock"
51
+
52
+ # database is a 'collection' element. it must have a hash or block parameter
53
+ # it will be 'collected' in a 'databases', with database id (1st arg) used as hash key
54
+ # the following code will create mysqldump/databases/blog and mysqldump/databases/mysql ocnfiguration 'nodes'
55
+
56
+ # backup database with default values
57
+ # database :blog
58
+
59
+ # backup overriding some values
60
+ # database :production do
61
+ # # you can override 'partially'
62
+ # keep :local => 3
63
+ # # keep/local is 3, and keep/s3 is 20 (from parent)
64
+
65
+ # # local override for gpg password
66
+ # gpg do
67
+ # password "custom-production-pass"
68
+ # end
69
+
70
+ # skip_tables [:logger_exceptions, :request_logs] # skip those tables during backup
71
+ # end
72
+
73
+ end
74
+
75
+ # # uncomment to enable
76
+ # # backup PostgreSQL databases with pg_dump
77
+ # pgdump do
78
+ # option "-i -x -O"
79
+ #
80
+ # user "markmansour"
81
+ # # password "" - leave this out if you have ident setup
82
+ #
83
+ # # database is a 'collection' element. it must have a hash or block parameter
84
+ # # it will be 'collected' in a 'databases', with database id (1st arg) used as hash key
85
+ # database :blog
86
+ # database :production
87
+ # end
88
+
89
+ tar do
90
+ # 'archive' is a collection item, just like 'database'
91
+ # archive "git-repositories" do
92
+ # # files and directories to backup
93
+ # files "/home/git/repositories"
94
+ # end
95
+
96
+ # archive "etc-files" do
97
+ # files "/etc"
98
+ # # exlude those files/directories
99
+ # exclude "/etc/puppet/other"
100
+ # end
101
+
102
+ # archive "dot-configs" do
103
+ # files "/home/*/.[^.]*"
104
+ # end
105
+
106
+ # archive "blog" do
107
+ # files "/var/www/blog.astrails.com/"
108
+ # # specify multiple files/directories as array
109
+ # exclude ["/var/www/blog.astrails.com/log", "/var/www/blog.astrails.com/tmp"]
110
+ # end
111
+
112
+ # archive "site" do
113
+ # files "/var/www/astrails.com/"
114
+ # exclude ["/var/www/astrails.com/log", "/var/www/astrails.com/tmp"]
115
+ # end
116
+
117
+ # archive :misc do
118
+ # files [ "/backup/*.rb" ]
119
+ # end
120
+ end
121
+
122
+ end
metadata ADDED
@@ -0,0 +1,94 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: markmansour-safe
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.1.7
5
+ platform: ruby
6
+ authors:
7
+ - Astrails Ltd.
8
+ - Mark Mansour
9
+ autorequire:
10
+ bindir: bin
11
+ cert_chain: []
12
+
13
+ date: 2009-04-24 00:00:00 -07:00
14
+ default_executable: astrails-safe
15
+ dependencies:
16
+ - !ruby/object:Gem::Dependency
17
+ name: aws-s3
18
+ type: :runtime
19
+ version_requirement:
20
+ version_requirements: !ruby/object:Gem::Requirement
21
+ requirements:
22
+ - - ">="
23
+ - !ruby/object:Gem::Version
24
+ version: "0"
25
+ version:
26
+ description: Simple tool to backup MySQL and PostgreSQL databases and filesystem locally or to Amazon S3 (with optional encryption)
27
+ email: we@astrails.com
28
+ executables:
29
+ - astrails-safe
30
+ extensions: []
31
+
32
+ extra_rdoc_files:
33
+ - README.markdown
34
+ - LICENSE
35
+ files:
36
+ - README.markdown
37
+ - VERSION.yml
38
+ - bin/astrails-safe
39
+ - examples/example_helper.rb
40
+ - examples/unit
41
+ - examples/unit/config_example.rb
42
+ - examples/unit/stream_example.rb
43
+ - lib/astrails
44
+ - lib/astrails/safe
45
+ - lib/astrails/safe/archive.rb
46
+ - lib/astrails/safe/config
47
+ - lib/astrails/safe/config/builder.rb
48
+ - lib/astrails/safe/config/node.rb
49
+ - lib/astrails/safe/gpg.rb
50
+ - lib/astrails/safe/gzip.rb
51
+ - lib/astrails/safe/local.rb
52
+ - lib/astrails/safe/mysqldump.rb
53
+ - lib/astrails/safe/pgdump.rb
54
+ - lib/astrails/safe/pipe.rb
55
+ - lib/astrails/safe/s3.rb
56
+ - lib/astrails/safe/sink.rb
57
+ - lib/astrails/safe/source.rb
58
+ - lib/astrails/safe/stream.rb
59
+ - lib/astrails/safe/tmp_file.rb
60
+ - lib/astrails/safe.rb
61
+ - lib/extensions
62
+ - lib/extensions/mktmpdir.rb
63
+ - templates/script.rb
64
+ - Rakefile
65
+ - LICENSE
66
+ has_rdoc: true
67
+ homepage: http://github.com/astrails/safe
68
+ post_install_message:
69
+ rdoc_options:
70
+ - --inline-source
71
+ - --charset=UTF-8
72
+ require_paths:
73
+ - lib
74
+ required_ruby_version: !ruby/object:Gem::Requirement
75
+ requirements:
76
+ - - ">="
77
+ - !ruby/object:Gem::Version
78
+ version: "0"
79
+ version:
80
+ required_rubygems_version: !ruby/object:Gem::Requirement
81
+ requirements:
82
+ - - ">="
83
+ - !ruby/object:Gem::Version
84
+ version: "0"
85
+ version:
86
+ requirements: []
87
+
88
+ rubyforge_project:
89
+ rubygems_version: 1.2.0
90
+ signing_key:
91
+ specification_version: 2
92
+ summary: Backup filesystem and database (MySQL and PostgreSQL) to Amazon S3 (with encryption)
93
+ test_files: []
94
+