darkofabijan-astrails-safe 0.2.8 → 0.2.9
Sign up to get free protection for your applications and to get access to all the features.
@@ -0,0 +1,34 @@
|
|
1
|
+
module Astrails
|
2
|
+
module Safe
|
3
|
+
class Page < Pipe
|
4
|
+
|
5
|
+
protected
|
6
|
+
|
7
|
+
def post_process
|
8
|
+
@backup.compressed = true
|
9
|
+
end
|
10
|
+
|
11
|
+
def pipe
|
12
|
+
#puts "Hi from Cut pipe"
|
13
|
+
#puts "CUT CONFIG #{@config.parent.inspect}"
|
14
|
+
#puts "PIPE: #{@backup.inspect}"
|
15
|
+
"|gzip|split -d -b #{page_size} - #{@backup.filename}#{@backup.extension}."
|
16
|
+
end
|
17
|
+
|
18
|
+
def extension
|
19
|
+
""
|
20
|
+
end
|
21
|
+
|
22
|
+
def active?
|
23
|
+
page_size
|
24
|
+
end
|
25
|
+
|
26
|
+
private
|
27
|
+
|
28
|
+
def page_size
|
29
|
+
@page_size ||= @config[:page, :page_size]
|
30
|
+
end
|
31
|
+
|
32
|
+
end
|
33
|
+
end
|
34
|
+
end
|
@@ -0,0 +1,92 @@
|
|
1
|
+
require File.expand_path(File.dirname(__FILE__) + '/../spec_helper')
|
2
|
+
|
3
|
+
require "fileutils"
|
4
|
+
include FileUtils
|
5
|
+
|
6
|
+
describe "tar backup" do
|
7
|
+
before(:all) do
|
8
|
+
# need both local and instance vars
|
9
|
+
# instance variables are used in tests
|
10
|
+
# local variables are used in the backup definition (instance vars can't be seen)
|
11
|
+
@root = root = "/tmp/safe-test"
|
12
|
+
|
13
|
+
# clean state
|
14
|
+
rm_rf @root
|
15
|
+
mkdir_p @root
|
16
|
+
|
17
|
+
# create source tree
|
18
|
+
@src = src = "#{@root}/src"
|
19
|
+
mkdir_p "#{@src}/q/w/e"
|
20
|
+
mkdir_p "#{@src}/a/s/d"
|
21
|
+
|
22
|
+
File.open("#{@src}/qwe1", "w") {|f| f.write("qwe") }
|
23
|
+
File.open("#{@src}/q/qwe2", "w") {|f| f.write("qwe"*2) }
|
24
|
+
File.open("#{@src}/q/w/qwe3", "w") {|f| f.write("qwe"*3) }
|
25
|
+
File.open("#{@src}/q/w/e/qwe4", "w") {|f| f.write("qwe"*4) }
|
26
|
+
|
27
|
+
File.open("#{@src}/asd1", "w") {|f| f.write("asd") }
|
28
|
+
File.open("#{@src}/a/asd2", "w") {|f| f.write("asd" * 2) }
|
29
|
+
File.open("#{@src}/a/s/asd3", "w") {|f| f.write("asd" * 3) }
|
30
|
+
|
31
|
+
@dst = dst = "#{@root}/backup"
|
32
|
+
mkdir_p @dst
|
33
|
+
|
34
|
+
@now = Time.now
|
35
|
+
@timestamp = @now.strftime("%y%m%d-%H%M")
|
36
|
+
|
37
|
+
stub(Time).now {@now} # Freeze
|
38
|
+
|
39
|
+
Astrails::Safe.safe do
|
40
|
+
local :path => "#{dst}/:kind"
|
41
|
+
page do
|
42
|
+
page_size 20
|
43
|
+
end
|
44
|
+
tar do
|
45
|
+
archive :test1 do
|
46
|
+
files src
|
47
|
+
exclude "#{src}/q/w"
|
48
|
+
exclude "#{src}/q/w/e"
|
49
|
+
end
|
50
|
+
end
|
51
|
+
end
|
52
|
+
|
53
|
+
@backup = "#{dst}/archive/archive-test1.#{@timestamp}.tar"
|
54
|
+
end
|
55
|
+
|
56
|
+
describe "after extracting" do
|
57
|
+
before(:all) do
|
58
|
+
# prepare target dir
|
59
|
+
@target = "#{@root}/test"
|
60
|
+
mkdir_p @target
|
61
|
+
|
62
|
+
backup_file_path = File.dirname(@backup)
|
63
|
+
backup_file_prefix = File.basename(@backup)
|
64
|
+
system "cat #{backup_file_prefix}.* >> #{backup_file_prefix}"
|
65
|
+
|
66
|
+
system "tar -zxvf #{@backup} -C #{@target}"
|
67
|
+
|
68
|
+
@test = "#{@target}/#{@root}/src"
|
69
|
+
puts @test
|
70
|
+
end
|
71
|
+
|
72
|
+
it "should include asd1/2/3" do
|
73
|
+
File.exists?("#{@test}/asd1").should be_true
|
74
|
+
File.exists?("#{@test}/a/asd2").should be_true
|
75
|
+
File.exists?("#{@test}/a/s/asd3").should be_true
|
76
|
+
end
|
77
|
+
|
78
|
+
it "should only include qwe 1 and 2 (no 3)" do
|
79
|
+
File.exists?("#{@test}/qwe1").should be_true
|
80
|
+
File.exists?("#{@test}/q/qwe2").should be_true
|
81
|
+
File.exists?("#{@test}/q/w/qwe3").should be_false
|
82
|
+
File.exists?("#{@test}/q/w/e/qwe4").should be_false
|
83
|
+
end
|
84
|
+
|
85
|
+
it "should preserve file content" do
|
86
|
+
File.read("#{@test}/qwe1").should == "qwe"
|
87
|
+
File.read("#{@test}/q/qwe2").should == "qweqwe"
|
88
|
+
File.read("#{@test}/a/s/asd3").should == "asdasdasd"
|
89
|
+
end
|
90
|
+
end
|
91
|
+
|
92
|
+
end
|
@@ -0,0 +1,71 @@
|
|
1
|
+
require File.expand_path(File.dirname(__FILE__) + '/../spec_helper')
|
2
|
+
|
3
|
+
require "fileutils"
|
4
|
+
include FileUtils
|
5
|
+
|
6
|
+
describe "s3 upload with paging" do
|
7
|
+
before(:all) do
|
8
|
+
# need both local and instance vars
|
9
|
+
# instance variables are used in tests
|
10
|
+
# local variables are used in the backup definition (instance vars can't be seen)
|
11
|
+
@root = root = "/tmp/safe-s3"
|
12
|
+
|
13
|
+
# clean state
|
14
|
+
rm_rf @root
|
15
|
+
mkdir_p @root
|
16
|
+
|
17
|
+
# create source tree
|
18
|
+
@src = src = File.join(root, "src")
|
19
|
+
mkdir_p File.join(src, 'dir1', 'dir2', 'dir3')
|
20
|
+
mkdir_p File.join(src, 'dir3', 'dir4', 'dir5')
|
21
|
+
|
22
|
+
File.open(File.join(src, 'file1'), "w") {|f| f.write("dir1") }
|
23
|
+
File.open(File.join(src, 'dir1', 'file2'), "w") {|f| f.write("dir2"*2) }
|
24
|
+
File.open(File.join(src, 'dir1', 'dir2', 'file3'), "w") {|f| f.write("dir3"*3) }
|
25
|
+
|
26
|
+
@dst = dst = File.join(@root, 'backup')
|
27
|
+
mkdir_p @dst
|
28
|
+
|
29
|
+
@now = Time.now
|
30
|
+
@timestamp = @now.strftime("%y%m%d-%H%M")
|
31
|
+
|
32
|
+
stub(Time).now {@now} # Freeze
|
33
|
+
|
34
|
+
Astrails::Safe.safe do
|
35
|
+
local :path => "#{dst}/:kind"
|
36
|
+
|
37
|
+
s3 do
|
38
|
+
key SERVICES_CONFIG['s3']['key']
|
39
|
+
secret SERVICES_CONFIG['s3']['secret']
|
40
|
+
bucket SERVICES_CONFIG['s3']['bucket']
|
41
|
+
path ":kind/" # this is default
|
42
|
+
end
|
43
|
+
|
44
|
+
page do
|
45
|
+
page_size 20
|
46
|
+
end
|
47
|
+
|
48
|
+
tar do
|
49
|
+
archive :test1 do
|
50
|
+
files src
|
51
|
+
end
|
52
|
+
end
|
53
|
+
|
54
|
+
end
|
55
|
+
@backup_file = "archive/archive-test1.#{@timestamp}.tar"
|
56
|
+
@backup_path = File.join(dst, @backup_file)
|
57
|
+
end
|
58
|
+
|
59
|
+
it "should upload all files to s3" do
|
60
|
+
AWS::S3::Base.establish_connection!(:access_key_id => SERVICES_CONFIG['s3']['key'], :secret_access_key => SERVICES_CONFIG['s3']['secret'], :use_ssl => true)
|
61
|
+
|
62
|
+
files = Dir.glob("#{@backup_path}*")
|
63
|
+
|
64
|
+
files.each do |file|
|
65
|
+
file.gsub!("#{@dst}/", '')
|
66
|
+
AWS::S3::S3Object.exists?(file, "df-dev-test").should be_true
|
67
|
+
end
|
68
|
+
|
69
|
+
end
|
70
|
+
|
71
|
+
end
|
metadata
CHANGED
@@ -1,13 +1,13 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: darkofabijan-astrails-safe
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
hash:
|
4
|
+
hash: 5
|
5
5
|
prerelease: false
|
6
6
|
segments:
|
7
7
|
- 0
|
8
8
|
- 2
|
9
|
-
-
|
10
|
-
version: 0.2.
|
9
|
+
- 9
|
10
|
+
version: 0.2.9
|
11
11
|
platform: ruby
|
12
12
|
authors:
|
13
13
|
- Astrails Ltd.
|
@@ -100,6 +100,7 @@ files:
|
|
100
100
|
- lib/astrails/safe/config/builder.rb
|
101
101
|
- lib/astrails/safe/config/node.rb
|
102
102
|
- lib/astrails/safe/gpg.rb
|
103
|
+
- lib/astrails/safe/page.rb
|
103
104
|
- lib/astrails/safe/gzip.rb
|
104
105
|
- lib/astrails/safe/local.rb
|
105
106
|
- lib/astrails/safe/mysqldump.rb
|
@@ -114,6 +115,8 @@ files:
|
|
114
115
|
- lib/astrails/safe/tmp_file.rb
|
115
116
|
- lib/extensions/mktmpdir.rb
|
116
117
|
- spec/integration/archive_integration_spec.rb
|
118
|
+
- spec/integration/archive_paging_integration_spec.rb
|
119
|
+
- spec/integration/s3_paging_integration_spec.rb
|
117
120
|
- spec/integration/cleanup_spec.rb
|
118
121
|
- spec/spec.opts
|
119
122
|
- spec/spec_helper.rb
|