akupchanko-astrails-safe 0.3.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (52) hide show
  1. checksums.yaml +7 -0
  2. data/.autotest +3 -0
  3. data/.document +5 -0
  4. data/.gitignore +18 -0
  5. data/.rspec +3 -0
  6. data/CHANGELOG +35 -0
  7. data/Gemfile +7 -0
  8. data/LICENSE.txt +22 -0
  9. data/README.markdown +250 -0
  10. data/Rakefile +8 -0
  11. data/TODO +31 -0
  12. data/akupchanko-astrails-safe.gemspec +35 -0
  13. data/bin/astrails-safe +64 -0
  14. data/lib/astrails/safe.rb +68 -0
  15. data/lib/astrails/safe/archive.rb +24 -0
  16. data/lib/astrails/safe/backup.rb +20 -0
  17. data/lib/astrails/safe/cloudfiles.rb +77 -0
  18. data/lib/astrails/safe/config/builder.rb +90 -0
  19. data/lib/astrails/safe/config/node.rb +72 -0
  20. data/lib/astrails/safe/ftp.rb +104 -0
  21. data/lib/astrails/safe/gpg.rb +46 -0
  22. data/lib/astrails/safe/gzip.rb +25 -0
  23. data/lib/astrails/safe/local.rb +51 -0
  24. data/lib/astrails/safe/mongodump.rb +23 -0
  25. data/lib/astrails/safe/mysqldump.rb +32 -0
  26. data/lib/astrails/safe/pgdump.rb +36 -0
  27. data/lib/astrails/safe/pipe.rb +17 -0
  28. data/lib/astrails/safe/s3.rb +80 -0
  29. data/lib/astrails/safe/sftp.rb +88 -0
  30. data/lib/astrails/safe/sink.rb +35 -0
  31. data/lib/astrails/safe/source.rb +47 -0
  32. data/lib/astrails/safe/stream.rb +32 -0
  33. data/lib/astrails/safe/svndump.rb +13 -0
  34. data/lib/astrails/safe/tmp_file.rb +48 -0
  35. data/lib/astrails/safe/version.rb +5 -0
  36. data/lib/extensions/mktmpdir.rb +45 -0
  37. data/spec/astrails/safe/archive_spec.rb +67 -0
  38. data/spec/astrails/safe/cloudfiles_spec.rb +175 -0
  39. data/spec/astrails/safe/config_spec.rb +307 -0
  40. data/spec/astrails/safe/gpg_spec.rb +148 -0
  41. data/spec/astrails/safe/gzip_spec.rb +64 -0
  42. data/spec/astrails/safe/local_spec.rb +109 -0
  43. data/spec/astrails/safe/mongodump_spec.rb +54 -0
  44. data/spec/astrails/safe/mysqldump_spec.rb +83 -0
  45. data/spec/astrails/safe/pgdump_spec.rb +45 -0
  46. data/spec/astrails/safe/s3_spec.rb +168 -0
  47. data/spec/astrails/safe/svndump_spec.rb +39 -0
  48. data/spec/integration/archive_integration_spec.rb +89 -0
  49. data/spec/integration/cleanup_spec.rb +62 -0
  50. data/spec/spec_helper.rb +8 -0
  51. data/templates/script.rb +183 -0
  52. metadata +178 -0
@@ -0,0 +1,32 @@
1
+ module Astrails
2
+ module Safe
3
+ class Stream
4
+
5
+ attr_accessor :config, :backup
6
+ def initialize(config, backup)
7
+ @config, @backup = config, backup
8
+ end
9
+ # FIXME: move to Backup
10
+ def expand(path)
11
+ path .
12
+ gsub(/:kind\b/, @backup.kind.to_s) .
13
+ gsub(/:id\b/, @backup.id.to_s) .
14
+ gsub(/:timestamp\b/, @backup.timestamp)
15
+ end
16
+
17
+ private
18
+
19
+ def verbose?
20
+ config[:verbose]
21
+ end
22
+
23
+ def local_only?
24
+ config[:local_only]
25
+ end
26
+
27
+ def dry_run?
28
+ config[:dry_run]
29
+ end
30
+ end
31
+ end
32
+ end
@@ -0,0 +1,13 @@
1
+ module Astrails
2
+ module Safe
3
+ class Svndump < Source
4
+
5
+ def command
6
+ "svnadmin dump #{config[:options]} #{config[:repo_path]}"
7
+ end
8
+
9
+ def extension; '.svn'; end
10
+
11
+ end
12
+ end
13
+ end
@@ -0,0 +1,48 @@
1
+ require 'tmpdir'
2
+ module Astrails
3
+ module Safe
4
+ module TmpFile
5
+ @keep_files = []
6
+
7
+ def self.tmproot
8
+ @tmproot ||= Dir.mktmpdir
9
+ end
10
+
11
+ def self.cleanup
12
+ begin
13
+ FileUtils.remove_entry_secure tmproot
14
+ rescue ArgumentError => e
15
+ if e.message =~ /parent directory is world writable/
16
+ puts <<-ERR
17
+
18
+
19
+ ********************************************************************************
20
+ It looks like you have wrong permissions on your TEMP directory. The usual
21
+ case is when you have world writable TEMP directory withOUT the sticky bit.
22
+
23
+ Try "chmod +t" on it.
24
+
25
+ ********************************************************************************
26
+
27
+ ERR
28
+ else
29
+ raise
30
+ end
31
+ end
32
+ @tmproot = nil
33
+ end
34
+
35
+ def self.create(name)
36
+ # create temp directory
37
+
38
+ file = Tempfile.new(name, tmproot)
39
+
40
+ yield file
41
+
42
+ file.close
43
+ @keep_files << file # so that it will not get gcollected and removed from filesystem until the end
44
+ file.path
45
+ end
46
+ end
47
+ end
48
+ end
@@ -0,0 +1,5 @@
1
+ module Astrails
2
+ module Safe
3
+ VERSION = "0.3.1"
4
+ end
5
+ end
@@ -0,0 +1,45 @@
1
+ require 'tmpdir'
2
+
3
+ unless Dir.respond_to?(:mktmpdir)
4
+ # backward compat for 1.8.6
5
+ class Dir
6
+ def Dir.mktmpdir(prefix_suffix=nil, tmpdir=nil)
7
+ case prefix_suffix
8
+ when nil
9
+ prefix = "d"
10
+ suffix = ""
11
+ when String
12
+ prefix = prefix_suffix
13
+ suffix = ""
14
+ when Array
15
+ prefix = prefix_suffix[0]
16
+ suffix = prefix_suffix[1]
17
+ else
18
+ raise ArgumentError, "unexpected prefix_suffix: #{prefix_suffix.inspect}"
19
+ end
20
+ tmpdir ||= Dir.tmpdir
21
+ t = Time.now.strftime("%Y%m%d")
22
+ n = nil
23
+ begin
24
+ path = "#{tmpdir}/#{prefix}#{t}-#{$$}-#{rand(0x100000000).to_s(36)}"
25
+ path << "-#{n}" if n
26
+ path << suffix
27
+ Dir.mkdir(path, 0700)
28
+ rescue Errno::EEXIST
29
+ n ||= 0
30
+ n += 1
31
+ retry
32
+ end
33
+
34
+ if block_given?
35
+ begin
36
+ yield path
37
+ ensure
38
+ FileUtils.remove_entry_secure path
39
+ end
40
+ else
41
+ path
42
+ end
43
+ end
44
+ end
45
+ end
@@ -0,0 +1,67 @@
1
+ require 'spec_helper'
2
+
3
+ describe Astrails::Safe::Archive do
4
+
5
+ def def_config
6
+ {
7
+ :options => "OPTS",
8
+ :files => "apples",
9
+ :exclude => "oranges"
10
+ }
11
+ end
12
+
13
+ def archive(id = :foo, config = def_config)
14
+ Astrails::Safe::Archive.new(id, Astrails::Safe::Config::Node.new(nil, config))
15
+ end
16
+
17
+ after(:each) { Astrails::Safe::TmpFile.cleanup }
18
+
19
+ describe :backup do
20
+ before(:each) do
21
+ @archive = archive
22
+ stub(@archive).timestamp {"NOW"}
23
+ end
24
+
25
+ {
26
+ :id => "foo",
27
+ :kind => "archive",
28
+ :extension => ".tar",
29
+ :filename => "archive-foo.NOW",
30
+ :command => "tar -cf - OPTS --exclude=oranges apples",
31
+ }.each do |k, v|
32
+ it "should set #{k} to #{v}" do
33
+ @archive.backup.send(k).should == v
34
+ end
35
+ end
36
+ end
37
+
38
+ describe :tar_exclude_files do
39
+ it "should return '' when no excludes" do
40
+ archive(:foo, {}).send(:tar_exclude_files).should == ''
41
+ end
42
+
43
+ it "should accept single exclude as string" do
44
+ archive(:foo, {:exclude => "bar"}).send(:tar_exclude_files).should == '--exclude=bar'
45
+ end
46
+
47
+ it "should accept multiple exclude as array" do
48
+ archive(:foo, {:exclude => ["foo", "bar"]}).send(:tar_exclude_files).should == '--exclude=foo --exclude=bar'
49
+ end
50
+ end
51
+
52
+ describe :tar_files do
53
+ it "should raise RuntimeError when no files" do
54
+ lambda {
55
+ archive(:foo, {}).send(:tar_files)
56
+ }.should raise_error(RuntimeError, "missing files for tar")
57
+ end
58
+
59
+ it "should accept single file as string" do
60
+ archive(:foo, {:files => "foo"}).send(:tar_files).should == "foo"
61
+ end
62
+
63
+ it "should accept multiple files as array" do
64
+ archive(:foo, {:files => ["foo", "bar"]}).send(:tar_files).should == "foo bar"
65
+ end
66
+ end
67
+ end
@@ -0,0 +1,175 @@
1
+ require 'spec_helper'
2
+
3
+ describe Astrails::Safe::Cloudfiles do
4
+
5
+ def def_config
6
+ {
7
+ cloudfiles: {
8
+ container: '_container',
9
+ user: '_user',
10
+ api_key: '_api_key',
11
+ },
12
+ keep: { cloudfiles: 2 }
13
+ }
14
+ end
15
+
16
+ def def_backup(extra = {})
17
+ {
18
+ kind: '_kind',
19
+ filename: '/backup/somewhere/_kind-_id.NOW.bar',
20
+ extension: '.bar',
21
+ id: '_id',
22
+ timestamp: 'NOW'
23
+ }.merge(extra)
24
+ end
25
+
26
+ def cloudfiles(config = def_config, backup = def_backup)
27
+ Astrails::Safe::Cloudfiles.new(
28
+ Astrails::Safe::Config::Node.new.merge(config),
29
+ Astrails::Safe::Backup.new(backup)
30
+ )
31
+ end
32
+
33
+ describe :cleanup do
34
+
35
+ before(:each) do
36
+ @cloudfiles = cloudfiles
37
+
38
+ @files = [4,1,3,2].map { |i| "aaaaa#{i}" }
39
+
40
+ @container = "container"
41
+
42
+ stub(@container).objects(prefix: "_kind/_id/_kind-_id.") { @files }
43
+ stub(@container).delete_object(anything)
44
+
45
+ stub(CloudFiles::Connection).
46
+ new('_user', '_api_key', true, false).stub!.
47
+ container('_container') {@container}
48
+ end
49
+
50
+ it "should check [:keep, :cloudfiles]" do
51
+ @cloudfiles.config[:keep].data["cloudfiles"] = nil
52
+ dont_allow(@cloudfiles.backup).filename
53
+ @cloudfiles.send :cleanup
54
+ end
55
+
56
+ it "should delete extra files" do
57
+ mock(@container).delete_object('aaaaa1')
58
+ mock(@container).delete_object('aaaaa2')
59
+ @cloudfiles.send :cleanup
60
+ end
61
+
62
+ end
63
+
64
+ describe :active do
65
+ before(:each) do
66
+ @cloudfiles = cloudfiles
67
+ end
68
+
69
+ it "should be true when all params are set" do
70
+ @cloudfiles.should be_active
71
+ end
72
+
73
+ it "should be false if container is missing" do
74
+ @cloudfiles.config[:cloudfiles].data["container"] = nil
75
+ @cloudfiles.should_not be_active
76
+ end
77
+
78
+ it "should be false if user is missing" do
79
+ @cloudfiles.config[:cloudfiles].data["user"] = nil
80
+ @cloudfiles.should_not be_active
81
+ end
82
+
83
+ it "should be false if api_key is missing" do
84
+ @cloudfiles.config[:cloudfiles].data["api_key"] = nil
85
+ @cloudfiles.should_not be_active
86
+ end
87
+ end
88
+
89
+ describe :path do
90
+ before(:each) do
91
+ @cloudfiles = cloudfiles
92
+ end
93
+ it "should use cloudfiles/path 1st" do
94
+ @cloudfiles.config[:cloudfiles].data["path"] = "cloudfiles_path"
95
+ @cloudfiles.config[:local] = {path: "local_path"}
96
+ @cloudfiles.send(:path).should == "cloudfiles_path"
97
+ end
98
+
99
+ it "should use local/path 2nd" do
100
+ @cloudfiles.config.merge local: {path: 'local_path'}
101
+ @cloudfiles.send(:path).should == 'local_path'
102
+ end
103
+
104
+ it "should use constant 3rd" do
105
+ @cloudfiles.send(:path).should == "_kind/_id"
106
+ end
107
+
108
+ end
109
+
110
+ describe :save do
111
+ def add_stubs(*stubs)
112
+ stubs.each do |s|
113
+ case s
114
+ when :connection
115
+ @connection = "connection"
116
+ stub(CloudFiles::Authentication).new
117
+ stub(CloudFiles::Connection).
118
+ new('_user', '_api_key', true, false) {@connection}
119
+ when :file_size
120
+ stub(@cloudfiles).get_file_size("foo") {123}
121
+ when :create_container
122
+ @container = "container"
123
+ stub(@container).create_object("_kind/_id/backup/somewhere/_kind-_id.NOW.bar.bar", true) {@object}
124
+ stub(@connection).create_container {@container}
125
+ when :file_open
126
+ stub(File).open("foo")
127
+ when :cloudfiles_store
128
+ @object = "object"
129
+ stub(@object).write(nil) {true}
130
+ end
131
+ end
132
+ end
133
+
134
+ before(:each) do
135
+ @cloudfiles = cloudfiles(def_config, def_backup(path: 'foo'))
136
+ @full_path = "_kind/_id/backup/somewhere/_kind-_id.NOW.bar.bar"
137
+ end
138
+
139
+ it "should fail if no backup.file is set" do
140
+ @cloudfiles.backup.path = nil
141
+ proc {@cloudfiles.send(:save)}.should raise_error(RuntimeError)
142
+ end
143
+
144
+ it "should establish Cloud Files connection" do
145
+ add_stubs(:connection, :file_size, :create_container, :file_open, :cloudfiles_store)
146
+ @cloudfiles.send(:save)
147
+ end
148
+
149
+ it "should open local file" do
150
+ add_stubs(:connection, :file_size, :create_container, :cloudfiles_store)
151
+ mock(File).open("foo")
152
+ @cloudfiles.send(:save)
153
+ end
154
+
155
+ it "should call write on the cloudfile object with files' descriptor" do
156
+ add_stubs(:connection, :file_size, :create_container, :cloudfiles_store)
157
+ stub(File).open("foo") {"qqq"}
158
+ mock(@object).write("qqq") {true}
159
+ @cloudfiles.send(:save)
160
+ end
161
+
162
+ it "should upload file" do
163
+ add_stubs(:connection, :file_size, :create_container, :file_open, :cloudfiles_store)
164
+ @cloudfiles.send(:save)
165
+ end
166
+
167
+ it "should fail on files bigger then 5G" do
168
+ add_stubs(:connection)
169
+ mock(File).stat("foo").stub!.size {5*1024*1024*1024+1}
170
+ mock(STDERR).puts(anything)
171
+ dont_allow(Benchmark).realtime
172
+ @cloudfiles.send(:save)
173
+ end
174
+ end
175
+ end
@@ -0,0 +1,307 @@
1
+ require 'spec_helper'
2
+
3
+ describe Astrails::Safe::Config do
4
+ it "should parse example config" do
5
+ config = Astrails::Safe::Config::Node.new do
6
+
7
+ dry_run false
8
+ local_only true
9
+ verbose true
10
+
11
+ local do
12
+ path "path"
13
+ end
14
+
15
+ s3 do
16
+ key "s3 key"
17
+ secret "secret"
18
+ bucket "bucket"
19
+ path "path1"
20
+ end
21
+
22
+ sftp do
23
+ user "sftp user"
24
+ password "sftp password"
25
+ host "sftp host"
26
+ end
27
+
28
+ gpg do
29
+ password "astrails"
30
+ key "gpg-key"
31
+ end
32
+
33
+ keep do
34
+ s3 20
35
+ local 4
36
+ end
37
+
38
+ mysqldump do
39
+ options "-ceKq --single-transaction --create-options"
40
+
41
+ user "astrails"
42
+ password ""
43
+ host "localhost"
44
+ port 3306
45
+ socket "/var/run/mysqld/mysqld.sock"
46
+
47
+ database :blog
48
+
49
+ database :production do
50
+ keep :local => 3
51
+
52
+ gpg do
53
+ password "custom-production-pass"
54
+ end
55
+
56
+ skip_tables [:logger_exceptions, :request_logs]
57
+ end
58
+
59
+ end
60
+
61
+ pgdump do
62
+ options "-i -x -O"
63
+
64
+ user "astrails"
65
+ password ""
66
+ host "localhost"
67
+ port 5432
68
+
69
+ database :blog
70
+
71
+ database :production do
72
+ keep :local => 3
73
+
74
+ skip_tables [:logger_exceptions, :request_logs]
75
+ end
76
+
77
+ end
78
+
79
+ svndump do
80
+ repo :my_repo do
81
+ repo_path "/home/svn/my_repo"
82
+ end
83
+ end
84
+
85
+ tar do
86
+ archive "git-repositories" do
87
+ files "/home/git/repositories"
88
+ end
89
+
90
+ archive "etc-files" do
91
+ files "/etc"
92
+ exclude "/etc/puppet/other"
93
+ end
94
+
95
+ archive "dot-configs" do
96
+ files "/home/*/.[^.]*"
97
+ end
98
+
99
+ archive "blog" do
100
+ files "/var/www/blog.astrails.com/"
101
+ exclude ["/var/www/blog.astrails.com/log", "/var/www/blog.astrails.com/tmp"]
102
+ end
103
+
104
+ archive :misc do
105
+ files [ "/backup/*.rb" ]
106
+ end
107
+ end
108
+
109
+ mongodump do
110
+ host "host"
111
+ database "database"
112
+ user "user"
113
+ password "password"
114
+ end
115
+ end
116
+
117
+ expected = {
118
+ "dry_run" => false,
119
+ "local_only" => true,
120
+ "verbose" => true,
121
+
122
+ "local" => {"path" => "path"},
123
+
124
+ "s3" => {
125
+ "key" => "s3 key",
126
+ "secret" => "secret",
127
+ "bucket" => "bucket",
128
+ "path" => "path1",
129
+ },
130
+
131
+ "sftp" => {
132
+ "user" => "sftp user",
133
+ "password" => "sftp password",
134
+ "host" => "sftp host",
135
+ },
136
+
137
+ "gpg" => {"password" => "astrails", "key" => "gpg-key"},
138
+
139
+ "keep" => {"s3" => 20, "local" => 4},
140
+
141
+ "mysqldump" => {
142
+ "options" => "-ceKq --single-transaction --create-options",
143
+ "user" => "astrails",
144
+ "password" => "",
145
+ "host" => "localhost",
146
+ "port" => 3306,
147
+ "socket" => "/var/run/mysqld/mysqld.sock",
148
+
149
+ "databases" => {
150
+ "blog" => {},
151
+ "production" => {
152
+ "keep" => {"local" => 3},
153
+ "gpg" => {"password" => "custom-production-pass"},
154
+ "skip_tables" => ["logger_exceptions", "request_logs"],
155
+ },
156
+ },
157
+ },
158
+
159
+ "pgdump" => {
160
+ "options" => "-i -x -O",
161
+ "user" => "astrails",
162
+ "password" => "",
163
+ "host" => "localhost",
164
+ "port" => 5432,
165
+
166
+ "databases" => {
167
+ "blog" => {},
168
+ "production" => {
169
+ "keep" => {"local" => 3},
170
+ "skip_tables" => ["logger_exceptions", "request_logs"],
171
+ },
172
+ },
173
+ },
174
+
175
+ "svndump" => {
176
+ "repos" => {
177
+ "my_repo"=> {
178
+ "repo_path" => "/home/svn/my_repo"
179
+ }
180
+ }
181
+ },
182
+
183
+ "tar" => {
184
+ "archives" => {
185
+ "git-repositories" => {"files" => ["/home/git/repositories"]},
186
+ "etc-files" => {"files" => ["/etc"], "exclude" => ["/etc/puppet/other"]},
187
+ "dot-configs" => {"files" => ["/home/*/.[^.]*"]},
188
+ "blog" => {
189
+ "files" => ["/var/www/blog.astrails.com/"],
190
+ "exclude" => ["/var/www/blog.astrails.com/log", "/var/www/blog.astrails.com/tmp"],
191
+ },
192
+ "misc" => { "files" => ["/backup/*.rb"] },
193
+ },
194
+ },
195
+
196
+ "mongodump" => {
197
+ "host" => "host",
198
+ "databases" => {
199
+ "database" => {}
200
+ },
201
+ "user" => "user",
202
+ "password" => "password"
203
+ }
204
+ }
205
+
206
+ config.to_hash.should == expected
207
+ end
208
+
209
+ it "should make an array from multivalues" do
210
+ config = Astrails::Safe::Config::Node.new do
211
+ skip_tables "a"
212
+ skip_tables "b"
213
+ files "/foo"
214
+ files "/bar"
215
+ exclude "/foo/bar"
216
+ exclude "/foo/bar/baz"
217
+ end
218
+
219
+ expected = {
220
+ "skip_tables" => ["a", "b"],
221
+ "files" => ["/foo", "/bar"],
222
+ "exclude" => ["/foo/bar", "/foo/bar/baz"],
223
+ }
224
+
225
+ config.to_hash.should == expected
226
+ end
227
+
228
+ it "should raise error on key duplication" do
229
+ proc do
230
+ Astrails::Safe::Config::Node.new do
231
+ path "foo"
232
+ path "bar"
233
+ end
234
+ end.should raise_error(ArgumentError, "duplicate value for 'path'")
235
+ end
236
+
237
+ it "should accept hash as data" do
238
+ Astrails::Safe::Config::Node.new do
239
+ tar do
240
+ archive 'blog', files: 'foo', exclude: ['aaa', 'bbb']
241
+ end
242
+ end.to_hash.should == {
243
+ 'tar' => {
244
+ 'archives' => {
245
+ 'blog' => {
246
+ 'files' => ['foo'],
247
+ 'exclude' => ['aaa', 'bbb']
248
+ }
249
+ }
250
+ }
251
+ }
252
+ end
253
+
254
+ it "should accept hash as data and a block" do
255
+ Astrails::Safe::Config::Node.new do
256
+ tar do
257
+ archive 'blog', files: 'foo' do
258
+ exclude ['aaa', 'bbb']
259
+ end
260
+ end
261
+ end.to_hash.should == {
262
+ 'tar' => {
263
+ 'archives' => {
264
+ 'blog' => {
265
+ 'files' => ['foo'],
266
+ 'exclude' => ['aaa', 'bbb']
267
+ }
268
+ }
269
+ }
270
+ }
271
+ end
272
+
273
+ it 'should accept multiple levels of data hash' do
274
+ config = Astrails::Safe::Config::Node.new nil, tar: {
275
+ s3: { bucket: '_bucket', key: '_key', secret: '_secret', },
276
+ keep: { s3: 2 }
277
+ }
278
+
279
+ config.to_hash.should == {
280
+ 'tar' => {
281
+ 's3' => { 'bucket' => '_bucket', 'key' => '_key', 'secret' => '_secret', },
282
+ 'keep' => { 's3' => 2 }
283
+ }
284
+ }
285
+ end
286
+
287
+ it 'should set multi value as array' do
288
+ config = Astrails::Safe::Config::Node.new do
289
+ tar do
290
+ archive 'foo' do
291
+ files 'bar'
292
+ end
293
+ end
294
+ end
295
+
296
+ config.to_hash.should == {
297
+ 'tar' => {
298
+ 'archives' => {
299
+ 'foo' => {
300
+ 'files' => ['bar']
301
+ }
302
+ }
303
+ }
304
+ }
305
+ end
306
+
307
+ end