s3backup 0.5.1 → 0.6.1
Sign up to get free protection for your applications and to get access to all the features.
- data/History.txt +4 -4
- data/Manifest.txt +1 -0
- data/PostInstall.txt +7 -7
- data/lib/s3backup/backup.rb +6 -31
- data/lib/s3backup/manager.rb +232 -0
- data/lib/s3backup/restore.rb +5 -63
- data/lib/s3backup/s3wrapper.rb +41 -147
- data/lib/s3backup/tree_info.rb +62 -0
- data/lib/s3backup.rb +1 -1
- data/script/console +0 -0
- data/script/destroy +0 -0
- data/script/generate +0 -0
- metadata +4 -3
data/History.txt
CHANGED
@@ -1,4 +1,4 @@
|
|
1
|
-
=== 0.0.1 2009-10-21
|
2
|
-
|
3
|
-
* 1 major enhancement:
|
4
|
-
* Initial release
|
1
|
+
=== 0.0.1 2009-10-21
|
2
|
+
|
3
|
+
* 1 major enhancement:
|
4
|
+
* Initial release
|
data/Manifest.txt
CHANGED
data/PostInstall.txt
CHANGED
@@ -1,7 +1,7 @@
|
|
1
|
-
|
2
|
-
For more information on s3backup, see http://s3backup.rubyforge.org
|
3
|
-
|
4
|
-
NOTE: Change this information in PostInstall.txt
|
5
|
-
You can also delete it if you don't want it.
|
6
|
-
|
7
|
-
|
1
|
+
|
2
|
+
For more information on s3backup, see http://s3backup.rubyforge.org
|
3
|
+
|
4
|
+
NOTE: Change this information in PostInstall.txt
|
5
|
+
You can also delete it if you don't want it.
|
6
|
+
|
7
|
+
|
data/lib/s3backup/backup.rb
CHANGED
@@ -1,7 +1,6 @@
|
|
1
|
-
require 'cgi'
|
2
1
|
require 's3backup/s3wrapper'
|
3
2
|
require 's3backup/s3log'
|
4
|
-
require 's3backup/
|
3
|
+
require 's3backup/manager'
|
5
4
|
module S3backup
|
6
5
|
class Backup
|
7
6
|
def initialize(config)
|
@@ -9,21 +8,16 @@ module S3backup
|
|
9
8
|
directories = config["directories"]
|
10
9
|
@directories = directories.map{|d| d=~/\/$/ ? d.chop : d}
|
11
10
|
begin
|
12
|
-
@s3_obj = S3Wrapper.new(config)
|
11
|
+
@s3_obj = S3Wrapper.new(config,true)
|
13
12
|
rescue => err
|
14
13
|
S3log.error(err.backtrace.join("\n")+"\n"+err.message)
|
15
14
|
exit -1
|
16
15
|
end
|
17
|
-
@
|
18
|
-
@directories.each do |dir|
|
19
|
-
@target_infos[:tree_infos].push(TreeInfo.new(dir))
|
20
|
-
S3log.debug("tree_info=#{@target_infos[:tree_infos].inspect}")
|
21
|
-
@target_infos[:attributes].push({:base => dir,:update_date => Date.today})
|
22
|
-
end
|
16
|
+
@manager = Manager.new(@s3_obj,config)
|
23
17
|
end
|
24
18
|
def check_config(config)
|
25
19
|
unless config["directories"]
|
26
|
-
S3log.error("directories doesn't exist in
|
20
|
+
S3log.error("directories doesn't exist in config file.")
|
27
21
|
exit -1
|
28
22
|
end
|
29
23
|
unless config["directories"].class == Array
|
@@ -45,27 +39,8 @@ module S3backup
|
|
45
39
|
def start
|
46
40
|
begin
|
47
41
|
first_flg=false
|
48
|
-
|
49
|
-
|
50
|
-
@s3_obj.create_bucket
|
51
|
-
end
|
52
|
-
@target_infos[:attributes].each_with_index do |attribute,i|
|
53
|
-
base = CGI.escape(attribute[:base])
|
54
|
-
tree_file_name = "tree_"+base+".yml"
|
55
|
-
tree_data = nil
|
56
|
-
unless first_flg
|
57
|
-
#前回のファイル・ツリーをAWS S3から取得
|
58
|
-
tree_data = @s3_obj.get(tree_file_name)
|
59
|
-
end
|
60
|
-
#前回と今回のファイル・ツリーを比較
|
61
|
-
diff_info = @target_infos[:tree_infos][i].diff(TreeInfo.new(tree_data))
|
62
|
-
S3log.debug("diff_info=#{diff_info.inspect}")
|
63
|
-
#更新されたディレクトリをAWS S3にアップロード
|
64
|
-
@s3_obj.store_directories(diff_info[:directory][:add] + diff_info[:directory][:modify])
|
65
|
-
#削除されたディレクトリをAWS S3からdelete
|
66
|
-
@s3_obj.delete_directories(diff_info[:directory][:remove])
|
67
|
-
#今回のファイル・ツリーをAWS S3に登録
|
68
|
-
@s3_obj.post(tree_file_name,@target_infos[:tree_infos][i].dump_yaml)
|
42
|
+
@directories.each do |dir|
|
43
|
+
@manager.differential_copy(dir)
|
69
44
|
end
|
70
45
|
rescue => err
|
71
46
|
S3log.error(err.backtrace.join("\n")+"\n"+err.message)
|
@@ -0,0 +1,232 @@
|
|
1
|
+
require 'cgi'
|
2
|
+
require 'tempfile'
|
3
|
+
require 'fileutils'
|
4
|
+
require 's3backup/s3log'
|
5
|
+
require 's3backup/tree_info'
|
6
|
+
module S3backup
|
7
|
+
class Manager
|
8
|
+
DEFAULT_BUF_READ_SIZE=1024*1024*128
|
9
|
+
def initialize(target,config)
|
10
|
+
@target = target
|
11
|
+
set_config(config)
|
12
|
+
end
|
13
|
+
def set_config(config)
|
14
|
+
if config["password"] and config["password"] != ""
|
15
|
+
unless config["salt"]
|
16
|
+
raise "salt doesn't exist in config file.\n"
|
17
|
+
end
|
18
|
+
unless config["salt"] =~ /[0-9A-Fa-f]{16}/
|
19
|
+
raise "salt format shoud be HexString and length should be 16.\n"
|
20
|
+
end
|
21
|
+
if config["BUF_SIEZE"]
|
22
|
+
size=config["BUF_SIEZE"]
|
23
|
+
if size > 1000*1000*1000*5
|
24
|
+
raise "BUF_SIZE must be less than 5G"
|
25
|
+
end
|
26
|
+
else
|
27
|
+
@buf_size = DEFAULT_BUF_READ_SIZE
|
28
|
+
end
|
29
|
+
@aes = Crypt.new(config["password"],config["salt"])
|
30
|
+
end
|
31
|
+
end
|
32
|
+
#指定されたディレクトリをtar gzip形式で圧縮する
|
33
|
+
def to_tgz(path,dir)
|
34
|
+
#サブディレクトリを圧縮の対象外にする。
|
35
|
+
sub_dir = []
|
36
|
+
Dir.foreach(dir) do |file|
|
37
|
+
next if /^\.+$/ =~ file
|
38
|
+
sub_dir.push(file) if File.directory?(dir+"/"+file)
|
39
|
+
end
|
40
|
+
exclude = ""
|
41
|
+
exclude = exclude + " --exclude=" + sub_dir.join(" --exclude=") if sub_dir.length != 0
|
42
|
+
cmd = "(cd #{File.dirname(dir)};tar -czvf #{path} #{exclude} #{File.basename(dir)} > /dev/null 2>&1)"
|
43
|
+
S3log.debug(cmd)
|
44
|
+
system(cmd)
|
45
|
+
unless $?.success?
|
46
|
+
raise "feiled #{cmd} execute. #{$?.inspect}"
|
47
|
+
end
|
48
|
+
end
|
49
|
+
def from_tgz(path,dir)
|
50
|
+
cmd = "tar -xzvf #{path} -C #{dir} > /dev/null 2>&1"
|
51
|
+
S3log.debug(cmd)
|
52
|
+
system(cmd)
|
53
|
+
unless $?.success?
|
54
|
+
raise "feiled #{cmd} execute. #{$?.inspect}"
|
55
|
+
end
|
56
|
+
end
|
57
|
+
def get_chain(key)
|
58
|
+
data = nil
|
59
|
+
data_set = nil
|
60
|
+
i=1
|
61
|
+
if @aes
|
62
|
+
key = @aes.encrypt(key)
|
63
|
+
end
|
64
|
+
while 1
|
65
|
+
key_name = i.to_s()+"_"+key
|
66
|
+
data = @target.get(key_name)
|
67
|
+
if data == nil
|
68
|
+
break
|
69
|
+
end
|
70
|
+
if i==1
|
71
|
+
data_set = ''
|
72
|
+
end
|
73
|
+
if @aes
|
74
|
+
data = @aes.decrypt(data)
|
75
|
+
end
|
76
|
+
data_set += data
|
77
|
+
i+=1
|
78
|
+
end
|
79
|
+
return data_set
|
80
|
+
end
|
81
|
+
def get_directory(dir,out_dir)
|
82
|
+
data = get_chain(dir)
|
83
|
+
tmp = Tempfile.open("s3backup")
|
84
|
+
tmp.write(data)
|
85
|
+
tmp.close
|
86
|
+
#tgzのファイルをcur_dirに展開
|
87
|
+
from_tgz(tmp.path,out_dir)
|
88
|
+
tmp.close(true)
|
89
|
+
end
|
90
|
+
def get_directories(dirs,prefix,output_dir)
|
91
|
+
prefix_len = prefix.length
|
92
|
+
dirs.each do |dir|
|
93
|
+
parent = File.dirname(dir)
|
94
|
+
p_len = parent.length
|
95
|
+
relative_path = parent.slice(prefix_len,p_len - prefix_len)
|
96
|
+
cur_dir = output_dir + relative_path
|
97
|
+
get_directory(dir,cur_dir)
|
98
|
+
end
|
99
|
+
end
|
100
|
+
def store_directory(dir)
|
101
|
+
tmp = Tempfile.open("s3backup")
|
102
|
+
tmp.close
|
103
|
+
#tgzのファイルをtmp.pathに作成
|
104
|
+
to_tgz(tmp.path,dir)
|
105
|
+
#S3にディレクトリの絶対パスをキーにして、圧縮したデータをストア
|
106
|
+
i=1
|
107
|
+
key = nil
|
108
|
+
if @aes
|
109
|
+
key = @aes.encrypt(dir)
|
110
|
+
else
|
111
|
+
key = dir
|
112
|
+
end
|
113
|
+
#前回のバックアップデータ削除
|
114
|
+
cnt = 1
|
115
|
+
while @target.exists?(cnt.to_s() + "_" + key)
|
116
|
+
@target.delete(cnt.to_s() + "_" + key)
|
117
|
+
cnt+=1
|
118
|
+
end
|
119
|
+
f = File.open(tmp.path,"r")
|
120
|
+
begin
|
121
|
+
while 1
|
122
|
+
key_name = i.to_s()+"_"+key
|
123
|
+
data = f.readpartial(@buf_size)
|
124
|
+
if @aes
|
125
|
+
data = @aes.encrypt(data)
|
126
|
+
end
|
127
|
+
@target.post(key_name,data)
|
128
|
+
i+=1
|
129
|
+
end
|
130
|
+
rescue EOFError
|
131
|
+
end
|
132
|
+
tmp.close(true)
|
133
|
+
end
|
134
|
+
def delete_direcory(dir)
|
135
|
+
if @aes
|
136
|
+
dir = @aes.encrypt(dir)
|
137
|
+
end
|
138
|
+
i=1
|
139
|
+
while @target.delete("#{i}_#{dir}")
|
140
|
+
i+=1
|
141
|
+
end
|
142
|
+
end
|
143
|
+
def differential_copy(dir)
|
144
|
+
#現在のファイル・ツリーを比較
|
145
|
+
tree_info = TreeInfo.new(dir)
|
146
|
+
|
147
|
+
target_tree_name = "tree_"+dir+".yml"
|
148
|
+
tree_data = nil
|
149
|
+
#前回のファイル・ツリーを取得
|
150
|
+
old_tree = TreeInfo.new(@target.get(target_tree_name))
|
151
|
+
|
152
|
+
#前回と今回のファイル・ツリーを比較
|
153
|
+
diff_info = tree_info.diff(old_tree)
|
154
|
+
S3log.debug("diff_info=#{diff_info.inspect}")
|
155
|
+
|
156
|
+
update_dir = diff_info[:directory][:add] + diff_info[:directory][:modify]
|
157
|
+
#更新されたディレクトリをアップロード
|
158
|
+
update_dir.each do |udir|
|
159
|
+
store_directory(udir)
|
160
|
+
udir_info = tree_info.get_dir_info(udir)
|
161
|
+
#前回のファイル・ツリー情報のうち、今回アップデートしたディレクトリ情報ファイル情報を更新
|
162
|
+
old_tree.update_dir(udir,udir_info)
|
163
|
+
#更新したファイル・ツリー情報をアップロード(途中で失敗しても、resumeできるようにするため。)
|
164
|
+
@target.post(target_tree_name,old_tree.dump_yaml)
|
165
|
+
end
|
166
|
+
diff_info[:directory][:remove].each do |rm_dir|
|
167
|
+
delete_direcory(rm_dir)
|
168
|
+
end
|
169
|
+
end
|
170
|
+
def get_target_tree(dir)
|
171
|
+
base_dir = dir
|
172
|
+
tree_data = nil
|
173
|
+
before_base=""
|
174
|
+
#バックアップしたディレクトリよりも下位のディレクトリが指定されることがあるため
|
175
|
+
while 1
|
176
|
+
base = base_dir
|
177
|
+
if base == before_base
|
178
|
+
break
|
179
|
+
end
|
180
|
+
tree_file_name = "tree_"+base+".yml"
|
181
|
+
tree_data = @target.get(tree_file_name)
|
182
|
+
if tree_data
|
183
|
+
break
|
184
|
+
end
|
185
|
+
before_base = base
|
186
|
+
base_dir = File.dirname(base_dir)
|
187
|
+
end
|
188
|
+
unless tree_data
|
189
|
+
return nil
|
190
|
+
end
|
191
|
+
return TreeInfo.new(tree_data)
|
192
|
+
end
|
193
|
+
def get_target_bases
|
194
|
+
files = @target.find(/^tree_.*\.yml/)
|
195
|
+
dirs = files.map do |d|
|
196
|
+
m=/tree_(.*)\.yml/.match(d)
|
197
|
+
next nil unless m
|
198
|
+
m[1]
|
199
|
+
end
|
200
|
+
return dirs.compact
|
201
|
+
end
|
202
|
+
def expand_tree(dir,tree_info,output_dir)
|
203
|
+
tree = tree_info.hierarchie(dir)
|
204
|
+
top = tree[0].keys[0]
|
205
|
+
top_dir = File.dirname(top)
|
206
|
+
tmp_dir = CGI.escape(top_dir)
|
207
|
+
output_dir = output_dir+"/"+tmp_dir
|
208
|
+
FileUtils.mkdir_p(output_dir)
|
209
|
+
tree.each do |node|
|
210
|
+
get_directories(node.keys,top_dir,output_dir)
|
211
|
+
end
|
212
|
+
top_dir_len = top_dir.length
|
213
|
+
(tree.length - 1).downto(0){|n|
|
214
|
+
tree[n].each do |k,v|
|
215
|
+
dir_len = k.length
|
216
|
+
relative_path = k.slice(top_dir_len,dir_len - top_dir_len)
|
217
|
+
dir = output_dir + relative_path
|
218
|
+
File.utime(v[:atime],v[:mtime],dir)
|
219
|
+
end
|
220
|
+
}
|
221
|
+
end
|
222
|
+
def restore(dir,output_dir)
|
223
|
+
tree = get_target_tree(dir)
|
224
|
+
unless tree
|
225
|
+
S3log.warn("#{dir} isn't find in AWS S3. ignore")
|
226
|
+
return
|
227
|
+
end
|
228
|
+
expand_tree(dir,tree,output_dir)
|
229
|
+
S3log.debug("expand_tree=#{tree.inspect}")
|
230
|
+
end
|
231
|
+
end
|
232
|
+
end
|
data/lib/s3backup/restore.rb
CHANGED
@@ -1,8 +1,6 @@
|
|
1
|
-
require 'cgi'
|
2
1
|
require 's3backup/s3wrapper'
|
3
2
|
require 's3backup/s3log'
|
4
|
-
require 's3backup/
|
5
|
-
require 'fileutils'
|
3
|
+
require 's3backup/manager'
|
6
4
|
module S3backup
|
7
5
|
class Restore
|
8
6
|
def initialize(output_dir,config)
|
@@ -10,12 +8,12 @@ module S3backup
|
|
10
8
|
@output_dir = output_dir
|
11
9
|
@directories = config["directories"]
|
12
10
|
begin
|
13
|
-
@s3_obj = S3Wrapper.new(config)
|
11
|
+
@s3_obj = S3Wrapper.new(config,false)
|
14
12
|
rescue => err
|
15
13
|
S3log.error(err.backtrace.join("\n")+"\n"+err.message)
|
16
14
|
exit -1
|
17
15
|
end
|
18
|
-
@
|
16
|
+
@manager = Manager.new(@s3_obj,config)
|
19
17
|
end
|
20
18
|
def check_config(config)
|
21
19
|
if config["directories"]
|
@@ -27,67 +25,11 @@ module S3backup
|
|
27
25
|
config["directories"] = config["directories"].map{|d| d=~/\/$/ ? d.chop : d}
|
28
26
|
end
|
29
27
|
end
|
30
|
-
def get_tree(dir)
|
31
|
-
base_dir = dir
|
32
|
-
tree_data = nil
|
33
|
-
while 1
|
34
|
-
base = CGI.escape(base_dir)
|
35
|
-
tree_file_name = "tree_"+base+".yml"
|
36
|
-
tree_data = @s3_obj.get(tree_file_name)
|
37
|
-
if tree_data or base_dir == "/"
|
38
|
-
break
|
39
|
-
end
|
40
|
-
base_dir = File.dirname(base_dir)
|
41
|
-
end
|
42
|
-
unless tree_data
|
43
|
-
return nil
|
44
|
-
end
|
45
|
-
tree_info = TreeInfo.new(tree_data)
|
46
|
-
return tree_info.hierarchie(dir)
|
47
|
-
end
|
48
|
-
def get_bases
|
49
|
-
files = @s3_obj.find(/^tree_.*\.yml/)
|
50
|
-
dirs = files.map do |d|
|
51
|
-
m=/tree_(.*)\.yml/.match(d)
|
52
|
-
next nil unless m
|
53
|
-
CGI.unescape(m[1])
|
54
|
-
end
|
55
|
-
return dirs.compact
|
56
|
-
end
|
57
|
-
def expand_tree(tree)
|
58
|
-
top = tree[0].keys[0]
|
59
|
-
top_dir = File.dirname(top)
|
60
|
-
tmp_dir = CGI.escape(top_dir)
|
61
|
-
output_dir = @output_dir+"/"+tmp_dir
|
62
|
-
FileUtils.mkdir_p(output_dir)
|
63
|
-
tree.each do |node|
|
64
|
-
@s3_obj.get_directories(node.keys,top_dir,output_dir)
|
65
|
-
end
|
66
|
-
top_dir_len = top_dir.length
|
67
|
-
(tree.length - 1).downto(0){|n|
|
68
|
-
tree[n].each do |k,v|
|
69
|
-
dir_len = k.length
|
70
|
-
relative_path = k.slice(top_dir_len,dir_len - top_dir_len)
|
71
|
-
dir = output_dir + relative_path
|
72
|
-
File.utime(v[:atime],v[:mtime],dir)
|
73
|
-
end
|
74
|
-
}
|
75
|
-
end
|
76
28
|
def start
|
77
29
|
begin
|
78
|
-
|
79
|
-
S3log.error("bucket: #{@s3_obj.bucket} isn't found!")
|
80
|
-
exit -1
|
81
|
-
end
|
82
|
-
@directories = get_bases unless @directories
|
30
|
+
@directories = @manager.get_target_bases unless @directories
|
83
31
|
@directories.each do |dir|
|
84
|
-
|
85
|
-
unless tree
|
86
|
-
s3log.warn("#{dir} isn't find in AWS S3. ignore")
|
87
|
-
next
|
88
|
-
end
|
89
|
-
puts tree.inspect
|
90
|
-
expand_tree(tree)
|
32
|
+
@manager.restore(dir,@output_dir)
|
91
33
|
end
|
92
34
|
rescue => err
|
93
35
|
S3log.error(err.backtrace.join("\n")+"\n"+err.message)
|
data/lib/s3backup/s3wrapper.rb
CHANGED
@@ -1,13 +1,15 @@
|
|
1
|
+
require 'cgi'
|
1
2
|
require 'aws/s3'
|
2
|
-
require 'tempfile'
|
3
3
|
require 's3backup/crypt'
|
4
4
|
module S3backup
|
5
5
|
class S3Wrapper
|
6
|
-
BUF_READ_SIZE=1024*1024*128
|
7
6
|
attr_reader :bucket
|
8
|
-
def initialize(config)
|
7
|
+
def initialize(config,create_flg)
|
8
|
+
@bucket= nil
|
9
|
+
@s3objects =nil
|
9
10
|
#設定ファイルの内容をメンバ変数にセット
|
10
11
|
set_config(config)
|
12
|
+
#設定ファイルの内容をメンバ変数にセット
|
11
13
|
args = {
|
12
14
|
:access_key_id => @access_key_id,
|
13
15
|
:secret_access_key => @secret_access_key
|
@@ -17,77 +19,43 @@ module S3backup
|
|
17
19
|
end
|
18
20
|
#AWS S3に接続
|
19
21
|
AWS::S3::Base.establish_connection!(args)
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
first_flg = true
|
26
|
-
buckets.each do |bucket|
|
27
|
-
if bucket.name == @bucket
|
28
|
-
return true
|
22
|
+
if create_flg
|
23
|
+
begin
|
24
|
+
@bucket = AWS::S3::Bucket.find(@bucket_name)
|
25
|
+
rescue AWS::S3::NoSuchBucket
|
26
|
+
@bucket = create_bucket
|
29
27
|
end
|
28
|
+
else
|
29
|
+
@bucket = AWS::S3::Bucket.find(@bucket_name)
|
30
30
|
end
|
31
|
-
return false
|
32
31
|
end
|
33
32
|
def create_bucket()
|
34
|
-
S3log.info("Bucket.create(#{@
|
33
|
+
S3log.info("Bucket.create(#{@bucket_name})")
|
35
34
|
#Bucket作成
|
36
|
-
ret = AWS::S3::Bucket.create(@
|
35
|
+
ret = AWS::S3::Bucket.create(@bucket_name)
|
37
36
|
unless ret
|
38
37
|
raise "AWS::S3::Bucket create error"
|
39
38
|
end
|
40
39
|
end
|
40
|
+
def rename(orig,dest)
|
41
|
+
AWS::S3::S3Object.rename(orig,dest,@bucket_name)
|
42
|
+
end
|
43
|
+
def exists?(key)
|
44
|
+
key=CGI.escape(key)
|
45
|
+
ret = AWS::S3::S3Object.exists? key,@bucket_name
|
46
|
+
end
|
41
47
|
def get(key)
|
42
48
|
key_name = CGI.escape(key)
|
43
49
|
data = nil
|
44
|
-
if AWS::S3::S3Object.exists? key_name,@
|
45
|
-
data = AWS::S3::S3Object.value(key_name,@
|
46
|
-
if @aes
|
47
|
-
data = @aes.decrypt(data)
|
48
|
-
end
|
50
|
+
if AWS::S3::S3Object.exists? key_name,@bucket_name
|
51
|
+
data = AWS::S3::S3Object.value(key_name,@bucket_name)
|
49
52
|
end
|
50
53
|
return data
|
51
54
|
end
|
52
|
-
def store_directories(dirs)
|
53
|
-
dirs.each do |dir|
|
54
|
-
store_directory(dir)
|
55
|
-
end
|
56
|
-
end
|
57
|
-
def get_directory(dir,out_dir)
|
58
|
-
data = get_chain(dir)
|
59
|
-
tmp = Tempfile.open("s3backup")
|
60
|
-
tmp.write(data)
|
61
|
-
tmp.close
|
62
|
-
#tgzのファイルをcur_dirに展開
|
63
|
-
from_tgz(tmp.path,out_dir)
|
64
|
-
tmp.close(true)
|
65
|
-
end
|
66
|
-
def get_directories(dirs,prefix,output_dir)
|
67
|
-
prefix_len = prefix.length
|
68
|
-
dirs.each do |dir|
|
69
|
-
parent = File.dirname(dir)
|
70
|
-
p_len = parent.length
|
71
|
-
relative_path = parent.slice(prefix_len,p_len - prefix_len)
|
72
|
-
cur_dir = output_dir + relative_path
|
73
|
-
get_directory(dir,cur_dir)
|
74
|
-
end
|
75
|
-
end
|
76
|
-
def delete_directories(dirs)
|
77
|
-
dirs.each do |dir|
|
78
|
-
delete_chain(dir)
|
79
|
-
end
|
80
|
-
end
|
81
55
|
def post(key,val)
|
82
|
-
if val.length > BUF_READ_SIZE
|
83
|
-
raise "max size = #{BUF_READ_SIZE} but size = #{val.size}"
|
84
|
-
end
|
85
56
|
key_name = CGI.escape(key)
|
86
|
-
if @aes
|
87
|
-
val = @aes.encrypt(val)
|
88
|
-
end
|
89
|
-
AWS::S3::S3Object.store(key_name,val,@bucket)
|
90
57
|
S3log.info("S3Object.store(#{key_name})")
|
58
|
+
AWS::S3::S3Object.store(key_name,val,@bucket_name)
|
91
59
|
end
|
92
60
|
def set_config(config)
|
93
61
|
err_msg = ""
|
@@ -107,109 +75,35 @@ module S3backup
|
|
107
75
|
@proxy[:port] = config["proxy_port"] if config["proxy_port"]
|
108
76
|
@proxy[:user] = config["proxy_user"] if config["proxy_user"]
|
109
77
|
@proxy[:password] = config["proxy_password"] if config["proxy_password"]
|
110
|
-
@
|
111
|
-
if config["password"] and config["password"] != ""
|
112
|
-
if config["salt"]
|
113
|
-
if config["salt"] =~ /[0-9A-Fa-f]{16}/
|
114
|
-
@aes = Crypt.new(config["password"],config["salt"])
|
115
|
-
else
|
116
|
-
err_msg += "salt format shoud be HexString and length should be 16.\n"
|
117
|
-
end
|
118
|
-
else
|
119
|
-
err_msg += "salt doesn't exist in config file.\n"
|
120
|
-
end
|
121
|
-
end
|
78
|
+
@bucket_name = config["bucket"]
|
122
79
|
if err_msg != ""
|
123
80
|
raise err_msg
|
124
81
|
end
|
125
82
|
end
|
126
|
-
#指定されたディレクトリをtar gzip形式で圧縮する
|
127
|
-
def to_tgz(path,dir)
|
128
|
-
#サブディレクトリを圧縮の対象外にする。
|
129
|
-
sub_dir = []
|
130
|
-
Dir.foreach(dir) do |file|
|
131
|
-
next if /^\.+$/ =~ file
|
132
|
-
sub_dir.push(file) if File.directory?(dir+"/"+file)
|
133
|
-
end
|
134
|
-
exclude = ""
|
135
|
-
exclude = exclude + " --exclude=" + sub_dir.join(" --exclude=") if sub_dir.length != 0
|
136
|
-
cmd = "(cd #{File.dirname(dir)};tar -czvf #{path} #{exclude} #{File.basename(dir)} > /dev/null 2>&1)"
|
137
|
-
S3log.info(cmd)
|
138
|
-
system(cmd)
|
139
|
-
unless $?.success?
|
140
|
-
raise "feiled #{cmd} execute. #{$?.inspect}"
|
141
|
-
end
|
142
|
-
end
|
143
|
-
def from_tgz(path,dir)
|
144
|
-
cmd = "tar -xzvf #{path} -C #{dir} > /dev/null 2>&1"
|
145
|
-
S3log.info(cmd)
|
146
|
-
system(cmd)
|
147
|
-
unless $?.success?
|
148
|
-
raise "feiled #{cmd} execute. #{$?.inspect}"
|
149
|
-
end
|
150
|
-
end
|
151
83
|
def delete(key)
|
152
|
-
if
|
153
|
-
|
154
|
-
|
155
|
-
key_name = CGI.escape(key)
|
156
|
-
end
|
157
|
-
if AWS::S3::S3Object.exists? key_name
|
158
|
-
S3log.info("S3Object.delete(#{key_name})")
|
159
|
-
AWS::S3::S3Object.delete(key_name,@bucket)
|
84
|
+
if exists? key
|
85
|
+
S3log.info("S3Object.delete(#{CGI.escape(key)})")
|
86
|
+
AWS::S3::S3Object.delete(CGI.escape(key),@bucket_name)
|
160
87
|
return true
|
161
88
|
end
|
162
89
|
return false
|
163
90
|
end
|
164
|
-
def
|
165
|
-
|
166
|
-
|
167
|
-
|
168
|
-
|
169
|
-
|
170
|
-
|
171
|
-
|
172
|
-
|
173
|
-
|
174
|
-
|
175
|
-
|
176
|
-
key_name = @aes.encrypt(key_name)
|
91
|
+
def find(dest)
|
92
|
+
@s3objects = @backet.objects unless @s3objects
|
93
|
+
find_obj=[]
|
94
|
+
keys=@s3objects.map { |o| CGI.unescape(CGI.unescape(o.key.sub("#{@bucket_name}/","")))}
|
95
|
+
keys.each do |key|
|
96
|
+
if dest.class == Regexp
|
97
|
+
if key =~ dest
|
98
|
+
find_obj.push(key)
|
99
|
+
end
|
100
|
+
else
|
101
|
+
if key == dest
|
102
|
+
find_obj.push(key)
|
177
103
|
end
|
178
|
-
post(key_name,f.readpartial(BUF_READ_SIZE))
|
179
|
-
i+=1
|
180
|
-
end
|
181
|
-
rescue EOFError
|
182
|
-
end
|
183
|
-
end
|
184
|
-
def get_chain(key)
|
185
|
-
data = nil
|
186
|
-
data_set = nil
|
187
|
-
i=1
|
188
|
-
while 1
|
189
|
-
key_name = i.to_s()+"_"+key
|
190
|
-
if @aes
|
191
|
-
key_name = @aes.encrypt(key_name)
|
192
|
-
end
|
193
|
-
data = get(key_name)
|
194
|
-
if data == nil
|
195
|
-
break
|
196
|
-
end
|
197
|
-
if i==1
|
198
|
-
data_set = ''
|
199
104
|
end
|
200
|
-
data_set += data
|
201
|
-
i+=1
|
202
105
|
end
|
203
|
-
return
|
204
|
-
end
|
205
|
-
def store_directory(dir)
|
206
|
-
tmp = Tempfile.open("s3backup")
|
207
|
-
tmp.close
|
208
|
-
#tgzのファイルをtmp.pathに作成
|
209
|
-
to_tgz(tmp.path,dir)
|
210
|
-
#S3にディレクトリの絶対パスをキーにして、圧縮したデータをストア
|
211
|
-
post_chain(dir,File.open(tmp.path,"r"))
|
212
|
-
tmp.close(true)
|
106
|
+
return find_obj
|
213
107
|
end
|
214
108
|
end
|
215
109
|
end
|
data/lib/s3backup/tree_info.rb
CHANGED
@@ -3,17 +3,25 @@ module S3backup
|
|
3
3
|
class TreeInfo
|
4
4
|
attr_reader :fileMap
|
5
5
|
def initialize(target)
|
6
|
+
@dir_map={}
|
7
|
+
@orig = nil
|
6
8
|
if target.nil?
|
7
9
|
@fileMap = {:file => Hash.new,:symlink => Hash.new,:directory => Hash.new}
|
8
10
|
elsif File.directory?(target)
|
11
|
+
@orig = {:type=>"directory",:target=>target}
|
9
12
|
@fileMap = {:file => Hash.new,:symlink => Hash.new,:directory => Hash.new}
|
10
13
|
stat = File.stat(target)
|
11
14
|
@fileMap[:directory][target] = {:mtime => stat.mtime, :atime => stat.atime}
|
15
|
+
@dir_map[target] = {:mtime => stat.mtime, :atime => stat.atime,:file=>{},:symlink=>{}}
|
12
16
|
makeFileMap(target)
|
13
17
|
elsif File.file?(target)
|
18
|
+
@orig = {:type=>"file",:target=>target}
|
14
19
|
load_yaml(File.read(target))
|
20
|
+
make_dir_map
|
15
21
|
else
|
22
|
+
@orig = {:type=>"data",:target=>""}
|
16
23
|
load_yaml(target)
|
24
|
+
make_dir_map
|
17
25
|
end
|
18
26
|
end
|
19
27
|
def makeFileMap(dir)
|
@@ -24,16 +32,70 @@ module S3backup
|
|
24
32
|
name = dir + "/" + e
|
25
33
|
if File.directory?(name)
|
26
34
|
stat = File.stat(name)
|
35
|
+
@dir_map[name] = {:mtime => stat.mtime, :atime => stat.atime,:file=>{},:symlink=>{}}
|
27
36
|
@fileMap[:directory][name] = {:mtime => stat.mtime, :atime => stat.atime}
|
28
37
|
makeFileMap(name)
|
29
38
|
elsif File.symlink?(name)
|
39
|
+
@dir_map[dir][:symlink][name] = {:source => File.readlink(name)}
|
30
40
|
@fileMap[:symlink][name] = {:source => File.readlink(name)}
|
31
41
|
else
|
32
42
|
stat = File.stat(name)
|
43
|
+
@dir_map[dir][:file][name] ={:size => stat.size,:date => stat.mtime}
|
33
44
|
@fileMap[:file][name] = {:size => stat.size,:date => stat.mtime}
|
34
45
|
end
|
35
46
|
end
|
36
47
|
end
|
48
|
+
def make_dir_map
|
49
|
+
@fileMap[:directory].each do |k,v|
|
50
|
+
@dir_map[k] = {:mtime => v[:mtime], :atime => v[:atime],:file=>{},:symlink=>{}}
|
51
|
+
end
|
52
|
+
@fileMap[:file].each do |k,v|
|
53
|
+
target = @dir_map[File.dirname(k)]
|
54
|
+
#不整合だけど適当に作る
|
55
|
+
unless target
|
56
|
+
S3log.warn("Tree Data isn't correct.#{@orig.inspect}")
|
57
|
+
target = {:mtime => DateTime.now.to_s,:atime => DateTime.now.to_s,:file=>{},:symlink=>{}}
|
58
|
+
@dir_map[File.dirname(k)] = target
|
59
|
+
end
|
60
|
+
target[:file][k] = {:size => v[:size], :date => v[:date]}
|
61
|
+
end
|
62
|
+
@fileMap[:symlink].each do |k,v|
|
63
|
+
target = @dir_map[File.dirname(k)]
|
64
|
+
#不整合だけど適当に作る
|
65
|
+
unless target
|
66
|
+
S3log.warn("Tree Data isn't correct.#{@orig.inspect}")
|
67
|
+
target = {:mtime => DateTime.now.to_s,:atime => DateTime.now.to_s,:file=>{},:symlink=>{}}
|
68
|
+
@dir_map[File.dirname(k)] = target
|
69
|
+
end
|
70
|
+
target[:symlink][k] = {:source => v[:source]}
|
71
|
+
end
|
72
|
+
end
|
73
|
+
def get_dir_info(name)
|
74
|
+
return @dir_map[name]
|
75
|
+
end
|
76
|
+
def update_dir_map(name,dir_info)
|
77
|
+
@dir_map[name][:file] = dir_info[:file]
|
78
|
+
@dir_map[name][:symlink] = dir_info[:symlink]
|
79
|
+
@dir_map[name][:mtime] = dir_info[:mtime]
|
80
|
+
@dir_map[name][:atime] = dir_info[:atime]
|
81
|
+
end
|
82
|
+
def update_dir(name,dir_info)
|
83
|
+
@dir_map[name] = {:file => {},:symlink=>{}} unless @dir_map[name]
|
84
|
+
@dir_map[name][:file].each do |k,v|
|
85
|
+
@fileMap[:file].delete(k)
|
86
|
+
end
|
87
|
+
@dir_map[name][:symlink].each do |k,v|
|
88
|
+
@fileMap[:symlink].delete(k)
|
89
|
+
end
|
90
|
+
@fileMap[:directory][name] = {:mtime => dir_info[:mtime],:atime =>dir_info[:atime]}
|
91
|
+
dir_info[:file].each do |k,v|
|
92
|
+
@fileMap[:file][k] = v
|
93
|
+
end
|
94
|
+
dir_info[:symlink].each do |k,v|
|
95
|
+
@fileMap[:symlink][k] = v
|
96
|
+
end
|
97
|
+
update_dir_map(name,dir_info)
|
98
|
+
end
|
37
99
|
def load_yaml(data)
|
38
100
|
@fileMap = YAML.load(data)
|
39
101
|
end
|
data/lib/s3backup.rb
CHANGED
data/script/console
CHANGED
File without changes
|
data/script/destroy
CHANGED
File without changes
|
data/script/generate
CHANGED
File without changes
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: s3backup
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.6.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Takeshi Morita
|
@@ -9,7 +9,7 @@ autorequire:
|
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
11
|
|
12
|
-
date: 2009-10-
|
12
|
+
date: 2009-10-24 00:00:00 +09:00
|
13
13
|
default_executable:
|
14
14
|
dependencies:
|
15
15
|
- !ruby/object:Gem::Dependency
|
@@ -55,6 +55,7 @@ files:
|
|
55
55
|
- lib/s3backup/backup.rb
|
56
56
|
- lib/s3backup/cli.rb
|
57
57
|
- lib/s3backup/crypt.rb
|
58
|
+
- lib/s3backup/manager.rb
|
58
59
|
- lib/s3backup/restore.rb
|
59
60
|
- lib/s3backup/s3log.rb
|
60
61
|
- lib/s3backup/s3wrapper.rb
|
@@ -92,7 +93,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
92
93
|
requirements: []
|
93
94
|
|
94
95
|
rubyforge_project: s3backup
|
95
|
-
rubygems_version: 1.3.
|
96
|
+
rubygems_version: 1.3.4
|
96
97
|
signing_key:
|
97
98
|
specification_version: 3
|
98
99
|
summary: S3Backup is a backup tool to local directory to Amazon S3.
|