akupchanko-astrails-safe 0.3.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (52) hide show
  1. checksums.yaml +7 -0
  2. data/.autotest +3 -0
  3. data/.document +5 -0
  4. data/.gitignore +18 -0
  5. data/.rspec +3 -0
  6. data/CHANGELOG +35 -0
  7. data/Gemfile +7 -0
  8. data/LICENSE.txt +22 -0
  9. data/README.markdown +250 -0
  10. data/Rakefile +8 -0
  11. data/TODO +31 -0
  12. data/akupchanko-astrails-safe.gemspec +35 -0
  13. data/bin/astrails-safe +64 -0
  14. data/lib/astrails/safe.rb +68 -0
  15. data/lib/astrails/safe/archive.rb +24 -0
  16. data/lib/astrails/safe/backup.rb +20 -0
  17. data/lib/astrails/safe/cloudfiles.rb +77 -0
  18. data/lib/astrails/safe/config/builder.rb +90 -0
  19. data/lib/astrails/safe/config/node.rb +72 -0
  20. data/lib/astrails/safe/ftp.rb +104 -0
  21. data/lib/astrails/safe/gpg.rb +46 -0
  22. data/lib/astrails/safe/gzip.rb +25 -0
  23. data/lib/astrails/safe/local.rb +51 -0
  24. data/lib/astrails/safe/mongodump.rb +23 -0
  25. data/lib/astrails/safe/mysqldump.rb +32 -0
  26. data/lib/astrails/safe/pgdump.rb +36 -0
  27. data/lib/astrails/safe/pipe.rb +17 -0
  28. data/lib/astrails/safe/s3.rb +80 -0
  29. data/lib/astrails/safe/sftp.rb +88 -0
  30. data/lib/astrails/safe/sink.rb +35 -0
  31. data/lib/astrails/safe/source.rb +47 -0
  32. data/lib/astrails/safe/stream.rb +32 -0
  33. data/lib/astrails/safe/svndump.rb +13 -0
  34. data/lib/astrails/safe/tmp_file.rb +48 -0
  35. data/lib/astrails/safe/version.rb +5 -0
  36. data/lib/extensions/mktmpdir.rb +45 -0
  37. data/spec/astrails/safe/archive_spec.rb +67 -0
  38. data/spec/astrails/safe/cloudfiles_spec.rb +175 -0
  39. data/spec/astrails/safe/config_spec.rb +307 -0
  40. data/spec/astrails/safe/gpg_spec.rb +148 -0
  41. data/spec/astrails/safe/gzip_spec.rb +64 -0
  42. data/spec/astrails/safe/local_spec.rb +109 -0
  43. data/spec/astrails/safe/mongodump_spec.rb +54 -0
  44. data/spec/astrails/safe/mysqldump_spec.rb +83 -0
  45. data/spec/astrails/safe/pgdump_spec.rb +45 -0
  46. data/spec/astrails/safe/s3_spec.rb +168 -0
  47. data/spec/astrails/safe/svndump_spec.rb +39 -0
  48. data/spec/integration/archive_integration_spec.rb +89 -0
  49. data/spec/integration/cleanup_spec.rb +62 -0
  50. data/spec/spec_helper.rb +8 -0
  51. data/templates/script.rb +183 -0
  52. metadata +178 -0
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA512:
3
+ data.tar.gz: 6d883362cf71d5a1efb2a7dcae62842a32f741dcd8c5188346d6d1eac9f3602625943ba3e7b08d2b50a2959f5901faf132927804557f76a073955f849769c8aa
4
+ metadata.gz: fe3a8df8e2a434c865ca80d361bc750cdc20deebe2b64581538b7c1d8f6db2a23e7eb04a13809f3d65bdb0602511bf533a7eeeb0fca8c60fbd6b2517c02f6233
5
+ SHA1:
6
+ data.tar.gz: 50b13bff3068d4d04d716d2e561353735a30f8f1
7
+ metadata.gz: 68c25d04ce2b671b0b035e1c5e4880db3776fdc2
@@ -0,0 +1,3 @@
1
+ Autotest.add_hook(:initialize) do |at|
2
+ at.add_mapping(%r{lib/astrails/safe/config/.*\.rb$}, true) {'spec/astrails/safe/config_spec.rb'}
3
+ end
@@ -0,0 +1,5 @@
1
+ README.rdoc
2
+ lib/**/*.rb
3
+ bin/*
4
+ features/**/*.feature
5
+ LICENSE
@@ -0,0 +1,18 @@
1
+ *.gem
2
+ *.rbc
3
+ .bundle
4
+ .config
5
+ .yardoc
6
+ Gemfile.lock
7
+ InstalledFiles
8
+ _yardoc
9
+ coverage
10
+ doc/
11
+ lib/bundler/man
12
+ pkg
13
+ rdoc
14
+ spec/reports
15
+ test/tmp
16
+ test/version_tmp
17
+ tmp
18
+ tags
data/.rspec ADDED
@@ -0,0 +1,3 @@
1
+ --colour
2
+ --format d
3
+ --profile
@@ -0,0 +1,35 @@
1
+ 0.3.1
2
+
3
+ * plain ftp support from seroy
4
+ * mongodump support from Matt Berther
5
+
6
+ 0.3.0
7
+
8
+ * switch to bundler
9
+ * fixed the rspec
10
+
11
+ 0.2.8
12
+
13
+ * ruby 1.9.2 compatibility (tests mostly)
14
+ * code review, and tons of small fixes
15
+ * check file size before attempting to upload to cloudfiles
16
+ * testing framework changed from micronaut to rspec
17
+
18
+ 0.2.7
19
+
20
+ * default options for gpg now include '--no-use-agent'
21
+ * support for 'command' option for gpg
22
+ * quote values in mysql password file
23
+ * add 'lib' to $:
24
+ * [EXPERIMENTAL] Rackspace Cloud Files support
25
+
26
+ 0.2.6
27
+
28
+ * fix typo in the template config file. (change option to options in pgdump)
29
+ * add example 'options' for tar in the template config file.
30
+ * do not try to upload more then 5G of data to S3. print error instead
31
+
32
+ 0.2.5
33
+
34
+ * Safety mesure: Disable overwrite of existing configuration keys except for multi-value keys
35
+ supported multi-value keys: skip_tables, exclude, files
data/Gemfile ADDED
@@ -0,0 +1,7 @@
1
+ source 'https://rubygems.org'
2
+
3
+ # Specify your gem's dependencies in akupchanko-astrails-safe.gemspec
4
+ gemspec
5
+
6
+ gem 'debugger'
7
+ gem 'awesome_print'
@@ -0,0 +1,22 @@
1
+ Copyright (c) 2010-2013 Astrails Ltd.
2
+
3
+ MIT License
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining
6
+ a copy of this software and associated documentation files (the
7
+ "Software"), to deal in the Software without restriction, including
8
+ without limitation the rights to use, copy, modify, merge, publish,
9
+ distribute, sublicense, and/or sell copies of the Software, and to
10
+ permit persons to whom the Software is furnished to do so, subject to
11
+ the following conditions:
12
+
13
+ The above copyright notice and this permission notice shall be
14
+ included in all copies or substantial portions of the Software.
15
+
16
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
17
+ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
18
+ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
19
+ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
20
+ LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
21
+ OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
22
+ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
@@ -0,0 +1,250 @@
1
+ # astrails-safe
2
+
3
+ Simple database and filesystem backups with S3 and Rackspace Cloud Files support (with optional encryption)
4
+
5
+ * Home: [http://astrails.com/opensource/astrails-safe](http://astrails.com/opensource/astrails-safe)
6
+ * Code: [http://github.com/astrails/safe](http://github.com/astrails/safe)
7
+ * Blog: [http://astrails.com/blog/astrails-safe](http://astrails.com/blog/astrails-safe)
8
+
9
+ [![Build Status](https://travis-ci.org/astrails/safe.png)](https://travis-ci.org/astrails/safe)
10
+ [![Code Climate](https://codeclimate.com/github/astrails/safe.png)](https://codeclimate.com/github/astrails/safe)
11
+
12
+ ## Motivation
13
+
14
+ We needed a backup solution that will satisfy the following requirements:
15
+
16
+ * opensource
17
+ * simple to install and configure
18
+ * support for simple ‘tar’ backups of directories (with includes/excludes)
19
+ * support for simple mysqldump of mysql databases
20
+ * support for symmetric or public key encryption
21
+ * support for local filesystem, Amazon S3, and Rackspace Cloud Files for storage
22
+ * support for backup rotation. we don’t want backups filling all the diskspace or cost a fortune on S3 or Cloud Files
23
+
24
+ And since we didn't find any, we wrote our own :)
25
+
26
+ ## Contributions
27
+
28
+ The following functionality was contributed by astrails-safe users:
29
+
30
+ * PostgreSQL dump using `pg_dump` (by Mark Mansour <mark@stateofflux.com>)
31
+ * Subversion dump using svndump (by Richard Luther <richard.luther@gmail.com>)
32
+ * SFTP remote storage (by Adam <adam@mediadrive.ca>)
33
+ * benchmarking output (By Neer)
34
+ * README fixes (by Bobby Wilson)
35
+ * improved config file parsing (by Fedor Kocherga <fkocherga@gmail.com>)
36
+ * mysql password file quoting (by Jonathan Sutherland <jonathan.sutherland@gmail.com>)
37
+ * Rackspace Cloud Files support (by H. Wade Minter <minter@lunenburg.org>)
38
+ * Plan FTP support (by seroy <seroy@bk.ru>)
39
+ * mongodump support (by Matt Berther <matt@mattberther.com>)
40
+
41
+ Thanks to all :)
42
+
43
+ ## Installation
44
+
45
+ sudo gem install astrails-safe --source http://gemcutter.org
46
+
47
+ ## Reporting problems
48
+
49
+ Please report problems at the [Issues tracker](http://github.com/astrails/safe/issues)
50
+
51
+ ## Usage
52
+
53
+ Usage:
54
+ astrails-safe [OPTIONS] CONFIG_FILE
55
+ Options:
56
+ -h, --help This help screen
57
+ -v, --verbose be verbose, duh!
58
+ -n, --dry-run just pretend, don't do anything.
59
+ -L, --local skip remote storage, only do local backups
60
+
61
+ Note: CONFIG\_FILE will be created from template if missing
62
+
63
+ ## Encryption
64
+
65
+ If you want to encrypt your backups you have 2 options:
66
+ * use simple password encryption
67
+ * use GPG public key encryption
68
+
69
+ > IMPORTANT: some gpg installations automatically set 'use-agent' option in the default
70
+ > configuration file that is created when you run gpg for the first time. This will cause
71
+ > gpg to fail on the 2nd run if you don't have the agent running. The result is that
72
+ > 'astrails-safe' will work ONCE when you manually test it and then fail on any subsequent run.
73
+ > The solution is to remove the 'use-agent' from the config file (usually /root/.gnupg/gpg.conf)
74
+ > To mitigate this problem for the gpg 1.x series '--no-use-agent' option is added by defaults
75
+ > to the autogenerated config file, but for gpg2 is doesn't work. as the manpage says it:
76
+ > "This is dummy option. gpg2 always requires the agent." :(
77
+
78
+ For simple password, just add password entry in gpg section.
79
+ For public key encryption you will need to create a public/secret keypair.
80
+
81
+ We recommend to create your GPG keys only on your local machine and then
82
+ transfer your public key to the server that will do the backups.
83
+
84
+ This way the server will only know how to encrypt the backups but only you
85
+ will be able to decrypt them using the secret key you have locally. Of course
86
+ you MUST backup your backup encryption key :)
87
+ We recommend also pringing the hard paper copy of your GPG key 'just in case'.
88
+
89
+ The procedure to create and transfer the key is as follows:
90
+
91
+ 1. run 'gpg --gen-key' on your local machine and follow onscreen instructions to create the key
92
+ (you can accept all the defaults).
93
+
94
+ 2. extract your public key into a file (assuming you used test@example.com as your key email):
95
+ `gpg -a --export test@example.com > test@example.com.pub`
96
+
97
+ 3. transfer public key to the server
98
+ `scp test@example.com.pub root@example.com:`
99
+
100
+ 4. import public key on the remote system:
101
+
102
+ $ gpg --import test@example.com.pub
103
+ gpg: key 45CA9403: public key "Test Backup <test@example.com>" imported
104
+ gpg: Total number processed: 1
105
+ gpg: imported: 1
106
+
107
+ 5. since we don't keep the secret part of the key on the remote server, gpg has
108
+ no way to know its yours and can be trusted.
109
+ To fix that we can sign it with other trusted key, or just directly modify its
110
+ trust level in gpg (use level 5):
111
+
112
+ $ gpg --edit-key test@example.com
113
+ ...
114
+ Command> trust
115
+ ...
116
+ 1 = I don't know or won't say
117
+ 2 = I do NOT trust
118
+ 3 = I trust marginally
119
+ 4 = I trust fully
120
+ 5 = I trust ultimately
121
+ m = back to the main menu
122
+
123
+ Your decision? 5
124
+ ...
125
+ Command> quit
126
+
127
+ 6. export your secret key for backup
128
+ (we recommend to print it on paper and burn to a CD/DVD and store in a safe place):
129
+
130
+ $ gpg -a --export-secret-key test@example.com > test@example.com.key
131
+
132
+
133
+
134
+ ## Example configuration
135
+
136
+ safe do
137
+ verbose true
138
+
139
+ local :path => "/backup/:kind/:id"
140
+
141
+ s3 do
142
+ key "...................."
143
+ secret "........................................"
144
+ bucket "backup.astrails.com"
145
+ path "servers/alpha/:kind/:id"
146
+ end
147
+
148
+ cloudfiles do
149
+ user "..........."
150
+ api_key "................................."
151
+ container "safe_backup"
152
+ path ":kind/" # this is default
153
+ service_net false
154
+ end
155
+
156
+ sftp do
157
+ host "sftp.astrails.com"
158
+ user "astrails"
159
+ # port 8023
160
+ password "ssh password for sftp"
161
+ end
162
+
163
+ ftp do
164
+ host "YOUR_REMOTE_HOSTNAME"
165
+ user "YOUR_REMOTE_USERNAME"
166
+ # port "NON STANDARD FTP PORT"
167
+ password "YOUR_REMOTE_PASSWORD"
168
+ path ":kind/:id" # this is the default
169
+ end
170
+
171
+ gpg do
172
+ command "/usr/local/bin/gpg"
173
+ options "--no-use-agent"
174
+ # symmetric encryption key
175
+ # password "qwe"
176
+
177
+ # public GPG key (must be known to GPG, i.e. be on the keyring)
178
+ key "backup@astrails.com"
179
+ end
180
+
181
+ keep do
182
+ local 20
183
+ s3 100
184
+ cloudfiles 100
185
+ sftp 100
186
+ end
187
+
188
+ mysqldump do
189
+ options "-ceKq --single-transaction --create-options"
190
+
191
+ user "root"
192
+ password "............"
193
+ socket "/var/run/mysqld/mysqld.sock"
194
+
195
+ database :blog
196
+ database :servershape
197
+ database :astrails_com
198
+ database :secret_project_com do
199
+ skip_tables "foo"
200
+ skip_tables ["bar", "baz"]
201
+ end
202
+
203
+ end
204
+
205
+ svndump do
206
+ repo :my_repo do
207
+ repo_path "/home/svn/my_repo"
208
+ end
209
+ end
210
+
211
+ pgdump do
212
+ options "-i -x -O" # -i => ignore version, -x => do not dump privileges (grant/revoke), -O => skip restoration of object ownership in plain text format
213
+
214
+ user "username"
215
+ password "............" # shouldn't be used, instead setup ident. Current functionality exports a password env to the shell which pg_dump uses - untested!
216
+
217
+ database :blog
218
+ database :stateofflux_com
219
+ end
220
+
221
+ tar do
222
+ options "-h" # dereference symlinks
223
+ archive "git-repositories", :files => "/home/git/repositories"
224
+ archive "dot-configs", :files => "/home/*/.[^.]*"
225
+ archive "etc", :files => "/etc", :exclude => "/etc/puppet/other"
226
+
227
+ archive "blog-astrails-com" do
228
+ files "/var/www/blog.astrails.com/"
229
+ exclude "/var/www/blog.astrails.com/log"
230
+ exclude "/var/www/blog.astrails.com/tmp"
231
+ end
232
+
233
+ archive "astrails-com" do
234
+ files "/var/www/astrails.com/"
235
+ exclude ["/var/www/astrails.com/log", "/var/www/astrails.com/tmp"]
236
+ end
237
+ end
238
+ end
239
+
240
+ ## Contributing
241
+
242
+ 1. Fork it
243
+ 2. Create your feature branch (`git checkout -b my-new-feature`)
244
+ 3. Commit your changes (`git commit -am 'Add some feature'`)
245
+ 4. Push to the branch (`git push origin my-new-feature`)
246
+ 5. Create new Pull Request
247
+
248
+ ## Copyright
249
+
250
+ Copyright (c) 2010-2013 Astrails Ltd. See LICENSE.txt for details.
@@ -0,0 +1,8 @@
1
+ require "bundler/gem_tasks"
2
+
3
+ require 'rspec/core/rake_task'
4
+
5
+ desc "run specs"
6
+ RSpec::Core::RakeTask.new
7
+
8
+ task default: :spec
data/TODO ADDED
@@ -0,0 +1,31 @@
1
+ - refactor
2
+ - refactor out global variables. pass a config object around instead
3
+ - common logging
4
+ - remove 1.8.6 support
5
+ - module registry
6
+ - base => prefix ?
7
+ - move requires into specific modules
8
+ - config.foo instead of config[:foo]
9
+
10
+ - features
11
+ - remote-only s3 support
12
+ - generic notifier support
13
+ - email notifier
14
+ - hipchat
15
+ - generic error notifier support
16
+ - email
17
+ - hipchat
18
+
19
+
20
+
21
+ - add 'silent'
22
+ - handle errors from mysqldump
23
+ - check that gpg is installed
24
+ - support percona XtraBackup as an option instead of mysqldump [patches anyone :) ?]
25
+ - backup validation:
26
+ - support for 'minsize' opition in backup that will check that produced backup is at least the expected size
27
+ this should catch many backup failure scenarious (like broken mysql connection, insufficient disk space etc.
28
+ - support differencial backups
29
+ - it should be fairly easy for filesystem backups using tar's built in incremental functionality.
30
+ - for mysql need to use XtraBackup
31
+ - or we can keep the previous dump locally and store only diff with the latest dump
@@ -0,0 +1,35 @@
1
+ # coding: utf-8
2
+ lib = File.expand_path('../lib', __FILE__)
3
+ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
4
+ require 'astrails/safe/version'
5
+
6
+ Gem::Specification.new do |spec|
7
+ spec.name = "akupchanko-astrails-safe"
8
+ spec.version = Astrails::Safe::VERSION
9
+ spec.authors = ["Vitaly Kushner"]
10
+ spec.email = ["we@astrails.com"]
11
+ spec.description = <<-DESC
12
+ Astrails-Safe is a simple tool to backup databases (MySQL and PostgreSQL), Subversion repositories (with svndump) and just files.
13
+ Backups can be stored locally or remotely and can be enctypted.
14
+ Remote storage is supported on Amazon S3, Rackspace Cloud Files, or just plain FTP/SFTP.
15
+ DESC
16
+ spec.summary = %Q{Backup filesystem and databases (MySQL and PostgreSQL) locally or to a remote server/service (with encryption)}
17
+ spec.homepage = "http://astrails.com/blog/astrails-safe"
18
+ spec.license = "MIT"
19
+
20
+ spec.default_executable = %q{astrails-safe}
21
+
22
+ spec.files = `git ls-files`.split($/)
23
+ spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
24
+ spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
25
+ spec.require_paths = ["lib"]
26
+
27
+ spec.add_dependency "aws-s3"
28
+ spec.add_dependency "cloudfiles"
29
+ spec.add_dependency "net-sftp"
30
+
31
+ spec.add_development_dependency "bundler", "~> 1.3"
32
+ spec.add_development_dependency "rake"
33
+ spec.add_development_dependency "rspec"
34
+ spec.add_development_dependency "rr", "~> 1.0.4"
35
+ end
@@ -0,0 +1,64 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ require 'rubygems'
4
+
5
+ require 'astrails/safe'
6
+
7
+ include Astrails::Safe
8
+
9
+ def die(msg)
10
+ puts "ERROR: #{msg}"
11
+ exit 1
12
+ end
13
+
14
+ def usage
15
+ puts <<-END
16
+ Usage: astrails-safe [OPTIONS] CONFIG_FILE
17
+ Options:
18
+ -h, --help This help screen
19
+ -v, --verbose be verbose, duh!
20
+ -n, --dry-run just pretend, don't do anything.
21
+ -L, --local skip S3 and Cloud Files
22
+
23
+ Note: config file will be created from template if missing
24
+ END
25
+ exit 1
26
+ end
27
+
28
+ OPTS = [
29
+ '-h', '--help',
30
+ '-v', '--verbose', '--not-verbose',
31
+ '-n', '--dry-run', '--not-dry-run',
32
+ '-L', '--local', '--not-local'
33
+ ]
34
+ def main
35
+ opts = ARGV & OPTS
36
+ args = ARGV - OPTS
37
+
38
+ usage unless args.first
39
+ usage if opts.delete("-h") || opts.delete("--help")
40
+
41
+ config_file = File.expand_path(args.first)
42
+
43
+ is_dry = (opts.delete('-n') || opts.delete('--dry-run')) && ! opts.delete('--not-dry-run')
44
+ is_verbose = (opts.delete('-v') || opts.delete('--verbose')) && !opts.delete('--not-verbose')
45
+ is_local_only = (opts.delete('-L') || opts.delete('--local')) && !opts.delete('--not-local')
46
+
47
+ unless File.exists?(config_file)
48
+ die "Missing configuration file. NOT CREATED! Rerun w/o the -n argument to create a template configuration file." if is_dry
49
+
50
+ FileUtils.cp File.join(Astrails::Safe::ROOT, "templates", "script.rb"), config_file
51
+
52
+ die "Created default #{config_file}. Please edit and run again."
53
+ end
54
+
55
+ config = eval(File.read(config_file))
56
+
57
+ config[:verbose] = is_verbose
58
+ config[:dry_run] = is_dry
59
+ config[:local_only] = is_local_only
60
+
61
+ process config
62
+ end
63
+
64
+ main