darkofabijan-astrails-safe 0.2.8 → 0.2.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,34 @@
|
|
1
|
+
module Astrails
|
2
|
+
module Safe
|
3
|
+
class Page < Pipe
|
4
|
+
|
5
|
+
protected
|
6
|
+
|
7
|
+
def post_process
|
8
|
+
@backup.compressed = true
|
9
|
+
end
|
10
|
+
|
11
|
+
def pipe
|
12
|
+
#puts "Hi from Cut pipe"
|
13
|
+
#puts "CUT CONFIG #{@config.parent.inspect}"
|
14
|
+
#puts "PIPE: #{@backup.inspect}"
|
15
|
+
"|gzip|split -d -b #{page_size} - #{@backup.filename}#{@backup.extension}."
|
16
|
+
end
|
17
|
+
|
18
|
+
def extension
|
19
|
+
""
|
20
|
+
end
|
21
|
+
|
22
|
+
def active?
|
23
|
+
page_size
|
24
|
+
end
|
25
|
+
|
26
|
+
private
|
27
|
+
|
28
|
+
def page_size
|
29
|
+
@page_size ||= @config[:page, :page_size]
|
30
|
+
end
|
31
|
+
|
32
|
+
end
|
33
|
+
end
|
34
|
+
end
|
@@ -0,0 +1,92 @@
|
|
1
|
+
require File.expand_path(File.dirname(__FILE__) + '/../spec_helper')
|
2
|
+
|
3
|
+
require "fileutils"
|
4
|
+
include FileUtils
|
5
|
+
|
6
|
+
describe "tar backup" do
|
7
|
+
before(:all) do
|
8
|
+
# need both local and instance vars
|
9
|
+
# instance variables are used in tests
|
10
|
+
# local variables are used in the backup definition (instance vars can't be seen)
|
11
|
+
@root = root = "/tmp/safe-test"
|
12
|
+
|
13
|
+
# clean state
|
14
|
+
rm_rf @root
|
15
|
+
mkdir_p @root
|
16
|
+
|
17
|
+
# create source tree
|
18
|
+
@src = src = "#{@root}/src"
|
19
|
+
mkdir_p "#{@src}/q/w/e"
|
20
|
+
mkdir_p "#{@src}/a/s/d"
|
21
|
+
|
22
|
+
File.open("#{@src}/qwe1", "w") {|f| f.write("qwe") }
|
23
|
+
File.open("#{@src}/q/qwe2", "w") {|f| f.write("qwe"*2) }
|
24
|
+
File.open("#{@src}/q/w/qwe3", "w") {|f| f.write("qwe"*3) }
|
25
|
+
File.open("#{@src}/q/w/e/qwe4", "w") {|f| f.write("qwe"*4) }
|
26
|
+
|
27
|
+
File.open("#{@src}/asd1", "w") {|f| f.write("asd") }
|
28
|
+
File.open("#{@src}/a/asd2", "w") {|f| f.write("asd" * 2) }
|
29
|
+
File.open("#{@src}/a/s/asd3", "w") {|f| f.write("asd" * 3) }
|
30
|
+
|
31
|
+
@dst = dst = "#{@root}/backup"
|
32
|
+
mkdir_p @dst
|
33
|
+
|
34
|
+
@now = Time.now
|
35
|
+
@timestamp = @now.strftime("%y%m%d-%H%M")
|
36
|
+
|
37
|
+
stub(Time).now {@now} # Freeze
|
38
|
+
|
39
|
+
Astrails::Safe.safe do
|
40
|
+
local :path => "#{dst}/:kind"
|
41
|
+
page do
|
42
|
+
page_size 20
|
43
|
+
end
|
44
|
+
tar do
|
45
|
+
archive :test1 do
|
46
|
+
files src
|
47
|
+
exclude "#{src}/q/w"
|
48
|
+
exclude "#{src}/q/w/e"
|
49
|
+
end
|
50
|
+
end
|
51
|
+
end
|
52
|
+
|
53
|
+
@backup = "#{dst}/archive/archive-test1.#{@timestamp}.tar"
|
54
|
+
end
|
55
|
+
|
56
|
+
describe "after extracting" do
|
57
|
+
before(:all) do
|
58
|
+
# prepare target dir
|
59
|
+
@target = "#{@root}/test"
|
60
|
+
mkdir_p @target
|
61
|
+
|
62
|
+
backup_file_path = File.dirname(@backup)
|
63
|
+
backup_file_prefix = File.basename(@backup)
|
64
|
+
system "cat #{backup_file_prefix}.* >> #{backup_file_prefix}"
|
65
|
+
|
66
|
+
system "tar -zxvf #{@backup} -C #{@target}"
|
67
|
+
|
68
|
+
@test = "#{@target}/#{@root}/src"
|
69
|
+
puts @test
|
70
|
+
end
|
71
|
+
|
72
|
+
it "should include asd1/2/3" do
|
73
|
+
File.exists?("#{@test}/asd1").should be_true
|
74
|
+
File.exists?("#{@test}/a/asd2").should be_true
|
75
|
+
File.exists?("#{@test}/a/s/asd3").should be_true
|
76
|
+
end
|
77
|
+
|
78
|
+
it "should only include qwe 1 and 2 (no 3)" do
|
79
|
+
File.exists?("#{@test}/qwe1").should be_true
|
80
|
+
File.exists?("#{@test}/q/qwe2").should be_true
|
81
|
+
File.exists?("#{@test}/q/w/qwe3").should be_false
|
82
|
+
File.exists?("#{@test}/q/w/e/qwe4").should be_false
|
83
|
+
end
|
84
|
+
|
85
|
+
it "should preserve file content" do
|
86
|
+
File.read("#{@test}/qwe1").should == "qwe"
|
87
|
+
File.read("#{@test}/q/qwe2").should == "qweqwe"
|
88
|
+
File.read("#{@test}/a/s/asd3").should == "asdasdasd"
|
89
|
+
end
|
90
|
+
end
|
91
|
+
|
92
|
+
end
|
@@ -0,0 +1,71 @@
|
|
1
|
+
require File.expand_path(File.dirname(__FILE__) + '/../spec_helper')
|
2
|
+
|
3
|
+
require "fileutils"
|
4
|
+
include FileUtils
|
5
|
+
|
6
|
+
describe "s3 upload with paging" do
|
7
|
+
before(:all) do
|
8
|
+
# need both local and instance vars
|
9
|
+
# instance variables are used in tests
|
10
|
+
# local variables are used in the backup definition (instance vars can't be seen)
|
11
|
+
@root = root = "/tmp/safe-s3"
|
12
|
+
|
13
|
+
# clean state
|
14
|
+
rm_rf @root
|
15
|
+
mkdir_p @root
|
16
|
+
|
17
|
+
# create source tree
|
18
|
+
@src = src = File.join(root, "src")
|
19
|
+
mkdir_p File.join(src, 'dir1', 'dir2', 'dir3')
|
20
|
+
mkdir_p File.join(src, 'dir3', 'dir4', 'dir5')
|
21
|
+
|
22
|
+
File.open(File.join(src, 'file1'), "w") {|f| f.write("dir1") }
|
23
|
+
File.open(File.join(src, 'dir1', 'file2'), "w") {|f| f.write("dir2"*2) }
|
24
|
+
File.open(File.join(src, 'dir1', 'dir2', 'file3'), "w") {|f| f.write("dir3"*3) }
|
25
|
+
|
26
|
+
@dst = dst = File.join(@root, 'backup')
|
27
|
+
mkdir_p @dst
|
28
|
+
|
29
|
+
@now = Time.now
|
30
|
+
@timestamp = @now.strftime("%y%m%d-%H%M")
|
31
|
+
|
32
|
+
stub(Time).now {@now} # Freeze
|
33
|
+
|
34
|
+
Astrails::Safe.safe do
|
35
|
+
local :path => "#{dst}/:kind"
|
36
|
+
|
37
|
+
s3 do
|
38
|
+
key SERVICES_CONFIG['s3']['key']
|
39
|
+
secret SERVICES_CONFIG['s3']['secret']
|
40
|
+
bucket SERVICES_CONFIG['s3']['bucket']
|
41
|
+
path ":kind/" # this is default
|
42
|
+
end
|
43
|
+
|
44
|
+
page do
|
45
|
+
page_size 20
|
46
|
+
end
|
47
|
+
|
48
|
+
tar do
|
49
|
+
archive :test1 do
|
50
|
+
files src
|
51
|
+
end
|
52
|
+
end
|
53
|
+
|
54
|
+
end
|
55
|
+
@backup_file = "archive/archive-test1.#{@timestamp}.tar"
|
56
|
+
@backup_path = File.join(dst, @backup_file)
|
57
|
+
end
|
58
|
+
|
59
|
+
it "should upload all files to s3" do
|
60
|
+
AWS::S3::Base.establish_connection!(:access_key_id => SERVICES_CONFIG['s3']['key'], :secret_access_key => SERVICES_CONFIG['s3']['secret'], :use_ssl => true)
|
61
|
+
|
62
|
+
files = Dir.glob("#{@backup_path}*")
|
63
|
+
|
64
|
+
files.each do |file|
|
65
|
+
file.gsub!("#{@dst}/", '')
|
66
|
+
AWS::S3::S3Object.exists?(file, "df-dev-test").should be_true
|
67
|
+
end
|
68
|
+
|
69
|
+
end
|
70
|
+
|
71
|
+
end
|
metadata
CHANGED
@@ -1,13 +1,13 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: darkofabijan-astrails-safe
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
hash:
|
4
|
+
hash: 5
|
5
5
|
prerelease: false
|
6
6
|
segments:
|
7
7
|
- 0
|
8
8
|
- 2
|
9
|
-
-
|
10
|
-
version: 0.2.
|
9
|
+
- 9
|
10
|
+
version: 0.2.9
|
11
11
|
platform: ruby
|
12
12
|
authors:
|
13
13
|
- Astrails Ltd.
|
@@ -100,6 +100,7 @@ files:
|
|
100
100
|
- lib/astrails/safe/config/builder.rb
|
101
101
|
- lib/astrails/safe/config/node.rb
|
102
102
|
- lib/astrails/safe/gpg.rb
|
103
|
+
- lib/astrails/safe/page.rb
|
103
104
|
- lib/astrails/safe/gzip.rb
|
104
105
|
- lib/astrails/safe/local.rb
|
105
106
|
- lib/astrails/safe/mysqldump.rb
|
@@ -114,6 +115,8 @@ files:
|
|
114
115
|
- lib/astrails/safe/tmp_file.rb
|
115
116
|
- lib/extensions/mktmpdir.rb
|
116
117
|
- spec/integration/archive_integration_spec.rb
|
118
|
+
- spec/integration/archive_paging_integration_spec.rb
|
119
|
+
- spec/integration/s3_paging_integration_spec.rb
|
117
120
|
- spec/integration/cleanup_spec.rb
|
118
121
|
- spec/spec.opts
|
119
122
|
- spec/spec_helper.rb
|