niky81-s3rbackup 0.2.9 → 0.3.0
Sign up to get free protection for your applications and to get access to all the features.
- data/bin/s3query.rb +22 -27
- data/bin/s3rbackup.rb +1 -1
- data/lib/s3dbsync.rb +79 -195
- metadata +11 -2
data/bin/s3query.rb
CHANGED
@@ -16,7 +16,7 @@ class OptS3rquery
|
|
16
16
|
def self.parse(args)
|
17
17
|
options = {}
|
18
18
|
opts = OptionParser.new do |opts|
|
19
|
-
opts.banner = "Usage: s3query.rb [options] <search|get|unpack|delete|stats
|
19
|
+
opts.banner = "Usage: s3query.rb [options] <search|get|unpack|delete|stats> <parameters> (parameters can be name=test or simply test)"
|
20
20
|
|
21
21
|
opts.on("-s", "search words", String, "Search something") do |name|
|
22
22
|
options[:op] = "search"
|
@@ -74,22 +74,22 @@ class OptS3rquery
|
|
74
74
|
options[:first] = true
|
75
75
|
end
|
76
76
|
|
77
|
-
opts.on("--
|
78
|
-
options[:
|
77
|
+
opts.on("-u", "--config-number NUM", Integer, "Number of config to use if nil use first") do |name|
|
78
|
+
options[:config_num] = name
|
79
79
|
end
|
80
|
-
|
81
|
-
opts.on("--
|
82
|
-
options[:
|
80
|
+
|
81
|
+
opts.on("--initialize", "Inizializza bucket and db") do |name|
|
82
|
+
options[:initialize] = true
|
83
83
|
end
|
84
|
-
|
85
|
-
opts.on("--
|
86
|
-
options[:
|
84
|
+
|
85
|
+
opts.on("--destroy", "Destroy bucket and db") do |name|
|
86
|
+
options[:destroy] = true
|
87
87
|
end
|
88
88
|
|
89
|
-
opts.on("
|
90
|
-
options[:
|
89
|
+
opts.on("--test", "Test something") do |name|
|
90
|
+
options[:test] = true
|
91
91
|
end
|
92
|
-
|
92
|
+
|
93
93
|
opts.on_tail("-h", "--help", "Show this message") do
|
94
94
|
puts opts
|
95
95
|
exit
|
@@ -125,8 +125,6 @@ case command
|
|
125
125
|
results = s3db.find(ARGV, nil, options)
|
126
126
|
results = get_last(results) if options[:last]
|
127
127
|
results = get_first(results) if options[:first]
|
128
|
-
when 'logs'
|
129
|
-
sub_cmd = ARGV.shift
|
130
128
|
end
|
131
129
|
case command
|
132
130
|
when 'search'
|
@@ -160,7 +158,6 @@ case command
|
|
160
158
|
puts "Deleting of #{ret["aws_name"]}"
|
161
159
|
s3db.delete(ret)
|
162
160
|
end
|
163
|
-
s3db.salva_db
|
164
161
|
when 'stats'
|
165
162
|
#get size
|
166
163
|
bucks_s = {}
|
@@ -171,23 +168,21 @@ case command
|
|
171
168
|
else
|
172
169
|
results.each do |ret|
|
173
170
|
bucks_s[ret["bucket"]] ||= 0
|
174
|
-
bucks_s[ret["bucket"]] += ret["size"]
|
171
|
+
bucks_s[ret["bucket"]] += ret["size"].to_i
|
175
172
|
end
|
176
173
|
bucks_s.each do |key,val|
|
177
174
|
puts "#{key}:\t#{sprintf("%.2fMb", val / (1024.0 * 1024.0))}"
|
178
175
|
end
|
179
176
|
end
|
180
|
-
when 'logs'
|
181
|
-
case sub_cmd
|
182
|
-
when 'get'
|
183
|
-
logs = s3db.logs()
|
184
|
-
logs.each do |log|
|
185
|
-
p log
|
186
|
-
end
|
187
|
-
when 'delete'
|
188
|
-
end
|
189
|
-
#get logs bucket
|
190
177
|
else
|
191
|
-
|
178
|
+
if options[:initialize]
|
179
|
+
s3db.initialize_db
|
180
|
+
elsif options[:destroy]
|
181
|
+
s3db.destroy_db
|
182
|
+
elsif options[:test]
|
183
|
+
s3db.test
|
184
|
+
else
|
185
|
+
puts "Some error occurred command #{command} not valid"
|
186
|
+
end
|
192
187
|
end
|
193
188
|
|
data/bin/s3rbackup.rb
CHANGED
@@ -78,4 +78,4 @@ config.current["bucket_log"] = options[:bucket_log] if options[:bucket_log]
|
|
78
78
|
config.current["compression"] = options[:compression] if options[:compression]
|
79
79
|
s3db = S3SyncDb.new(config.current)
|
80
80
|
s3db.bak(ARGV, options[:name], options[:descr])
|
81
|
-
s3db.salva_db
|
81
|
+
#s3db.salva_db
|
data/lib/s3dbsync.rb
CHANGED
@@ -2,6 +2,9 @@ require 'rubygems'
|
|
2
2
|
require 'aws/s3'
|
3
3
|
require 'tempfile'
|
4
4
|
require 'yaml'
|
5
|
+
#nuovo aws
|
6
|
+
require 'right_aws'
|
7
|
+
require 'net/smtp'
|
5
8
|
|
6
9
|
class S3SyncDb
|
7
10
|
include AWS::S3
|
@@ -10,94 +13,21 @@ class S3SyncDb
|
|
10
13
|
@conn = AWS::S3::Base.establish_connection!(
|
11
14
|
:access_key_id => config["access_key_id"],
|
12
15
|
:secret_access_key => config["secret_access_key"])
|
16
|
+
@sdb = RightAws::SdbInterface.new(config["access_key_id"], config["secret_access_key"],
|
17
|
+
{:multi_thread => false, :logger => Logger.new('/tmp/sdb.log')})
|
18
|
+
@s3 = RightAws::S3.new(config["access_key_id"], config["secret_access_key"],
|
19
|
+
{:multi_thread => false, :logger => Logger.new('/tmp/s3.log')})
|
13
20
|
|
14
21
|
@config = config
|
15
|
-
|
16
|
-
@
|
17
|
-
@
|
18
|
-
#apro il db
|
19
|
-
if !File.exists?(@db_file)
|
20
|
-
#download db
|
21
|
-
begin
|
22
|
-
buck_db = Bucket.find(config["bucket_db"])
|
23
|
-
db_file = S3Object.find('s3rbackup_yaml_db', config["bucket_db"])
|
24
|
-
rescue
|
25
|
-
#devo crearlo
|
26
|
-
if config["sync_db"]
|
27
|
-
Bucket.create(config["bucket_db"])
|
28
|
-
end
|
29
|
-
@db = []
|
30
|
-
@version = 0
|
31
|
-
return
|
32
|
-
end
|
33
|
-
#lo copio in locale
|
34
|
-
open(@db_file, 'w') do |file|
|
35
|
-
S3Object.stream('s3rbackup_yaml_db', config["bucket_db"]) do |chunk|
|
36
|
-
file.write chunk
|
37
|
-
end
|
38
|
-
end
|
39
|
-
@version = db_file.metadata[:version].to_i
|
40
|
-
File.open(@db_file_ver, 'w') { |f| f.puts @version.to_s }
|
41
|
-
elsif config["sync_db"]
|
42
|
-
begin
|
43
|
-
buck_db = Bucket.find(config["bucket_db"])
|
44
|
-
db_file = S3Object.find('s3rbackup_yaml_db', config["bucket_db"])
|
45
|
-
rescue
|
46
|
-
#devo crearlo
|
47
|
-
if config["sync_db"]
|
48
|
-
Bucket.create(config["bucket_db"])
|
49
|
-
end
|
50
|
-
@db = YAML::load(File.open(@db_file))
|
51
|
-
@version ||= File.read(@db_file_ver).to_i
|
52
|
-
return
|
53
|
-
end
|
54
|
-
#se esiste e devo fare il sync
|
55
|
-
@version = db_file.metadata[:version].to_i
|
56
|
-
local_ver = File.read(@db_file_ver)
|
57
|
-
if @version > local_ver.to_i
|
58
|
-
#uso il remoto
|
59
|
-
open(@db_file, 'w') do |file|
|
60
|
-
db_file.stream do |chunk|
|
61
|
-
file.write chunk
|
62
|
-
end
|
63
|
-
end
|
64
|
-
File.open(@db_file_ver, 'w') { |f| f.puts @version.to_s }
|
65
|
-
else
|
66
|
-
#posso usare quello locale
|
67
|
-
end
|
68
|
-
end
|
69
|
-
#lo carico
|
70
|
-
@db = YAML::load(File.open(@db_file))
|
71
|
-
@version ||= File.read(@db_file_ver).to_i
|
72
|
-
end
|
73
|
-
|
74
|
-
def salva_locale
|
75
|
-
File.open(@db_file, 'w') { |f| f.puts @db.to_yaml }
|
22
|
+
#d'ora in poi uso il db su aws
|
23
|
+
@domain_db = config['bucket']
|
24
|
+
@bucket = config['bucket']
|
76
25
|
end
|
77
26
|
|
78
|
-
def
|
79
|
-
@
|
80
|
-
|
81
|
-
|
82
|
-
|
83
|
-
def aggiornaS3
|
84
|
-
salva_locale()
|
85
|
-
S3Object.store("s3rbackup_yaml_db", open(@db_file, "r"), @config["bucket_db"])
|
86
|
-
db_file = S3Object.find('s3rbackup_yaml_db', @config["bucket_db"])
|
87
|
-
db_file.metadata[:version] = @version.to_s
|
88
|
-
db_file.store
|
89
|
-
end
|
90
|
-
|
91
|
-
def salva_db
|
92
|
-
salva_locale()
|
93
|
-
nuova_versione()
|
94
|
-
aggiornaS3() if @config["sync_db"]
|
95
|
-
end
|
96
|
-
|
97
|
-
def crea_bucket()
|
98
|
-
#find bucket
|
99
|
-
@bucket = @config["bucket"]
|
100
|
-
@bucket_log = @config["bucket_log"] ? @config["bucket_log"] : "#{@bucket}-logs"
|
27
|
+
def initialize_db
|
28
|
+
puts "Creating #{@domain_db} db..."
|
29
|
+
@sdb.create_domain(@domain_db)
|
30
|
+
puts "Creating #{@bucket} bucket..."
|
101
31
|
begin
|
102
32
|
bbackup = Bucket.find(@bucket)
|
103
33
|
rescue
|
@@ -105,21 +35,24 @@ class S3SyncDb
|
|
105
35
|
raise "Can't create bucket:#{@bucket}"
|
106
36
|
end
|
107
37
|
end
|
108
|
-
|
109
|
-
|
110
|
-
|
111
|
-
|
112
|
-
|
113
|
-
|
114
|
-
|
115
|
-
|
116
|
-
|
117
|
-
|
118
|
-
|
119
|
-
|
38
|
+
end
|
39
|
+
|
40
|
+
def destroy_db
|
41
|
+
puts "Deleting bucket #{@bucket}..."
|
42
|
+
bucket1 = @s3.bucket(@bucket)
|
43
|
+
bucket1.delete(true)
|
44
|
+
puts "Deleting db #{@domain_db}..."
|
45
|
+
@sdb.delete_domain(@domain_db)
|
46
|
+
end
|
47
|
+
|
48
|
+
def test
|
49
|
+
#buk = @s3.buckets
|
50
|
+
buk = @s3.buckets.map{|b| b.name}
|
51
|
+
puts "Buckets on S3: #{buk.join(', ')}"
|
52
|
+
end
|
120
53
|
|
54
|
+
#s3 sdb
|
121
55
|
def bak(dirs, name, descr)
|
122
|
-
crea_bucket()
|
123
56
|
name = dirs[0] if !name
|
124
57
|
tf = Tempfile.new("s3rbackup")
|
125
58
|
tf_l = Tempfile.new("s3rbackup-listfile")
|
@@ -147,23 +80,24 @@ class S3SyncDb
|
|
147
80
|
doc = {}
|
148
81
|
doc["name"] = name
|
149
82
|
doc["bucket"] = @config["bucket"]
|
150
|
-
doc["datetime"] =
|
83
|
+
doc["datetime"] = "'#{DateTime.now}'"
|
84
|
+
#doc["date"] = "#{DateTime.now.strftime("%Y%m%d")}"
|
151
85
|
doc["current_path"] = `pwd`.gsub("\n","").to_s
|
152
|
-
doc["description"] = descr
|
86
|
+
doc["description"] = descr ? descr : "n/a"
|
153
87
|
doc["host"] = `hostname`.gsub("\n","").to_s
|
154
88
|
doc["user"] = `whoami`.gsub("\n","").to_s
|
155
89
|
doc["size"] = File.size(file_path)
|
156
90
|
doc["compression"] = @config["compression"]
|
157
91
|
doc["archive"] = "tar"
|
158
|
-
doc["files"] = filez.join("")
|
159
|
-
|
160
|
-
aws_name = "#{doc["name"]}
|
92
|
+
#doc["files"] = filez.join("")
|
93
|
+
#@db << doc
|
94
|
+
aws_name = "#{doc["name"]}##{`date +%Y%m%d_%H.%M.%S`}".gsub("\n","")
|
161
95
|
doc["aws_name"] = aws_name
|
162
|
-
|
96
|
+
@sdb.put_attributes @domain_db, aws_name, doc
|
97
|
+
#TODO aggiungere md5
|
163
98
|
|
164
99
|
# Store it!
|
165
100
|
options = {}
|
166
|
-
#options[:access] = :public_read if @public
|
167
101
|
options["x-amz-meta-host"] = doc["host"]
|
168
102
|
options["x-amz-meta-user"] = doc["user"]
|
169
103
|
options["x-amz-meta-descrizione"] = doc["description"]
|
@@ -171,12 +105,6 @@ class S3SyncDb
|
|
171
105
|
options["x-amz-meta-size"] = doc["size"]
|
172
106
|
options["x-amz-meta-compression"] = doc["compression"]
|
173
107
|
options["x-amz-meta-archive"] = doc["archive"]
|
174
|
-
#options["x-amz-meta-files"] = doc["files"]
|
175
|
-
|
176
|
-
|
177
|
-
# options["x-amz-meta-sha1_hash"] = `sha1sum #{file}`.split[0] if @save_hash
|
178
|
-
# options["x-amz-meta-mtime"] = fstat.mtime.getutc.to_i if @save_time
|
179
|
-
# options["x-amz-meta-size"] = fstat.size if @save_size
|
180
108
|
|
181
109
|
store = S3Object.store(aws_name, open(file_path), @config["bucket"], options)
|
182
110
|
obj = S3Object.find(aws_name, @config["bucket"])
|
@@ -185,108 +113,56 @@ class S3SyncDb
|
|
185
113
|
doc[key] = val
|
186
114
|
end
|
187
115
|
#TODO aggiungere check
|
188
|
-
#
|
116
|
+
send_mail("S3rbackup - Saved #{doc["name"]}", (doc.to_a.map {|val| "#{val[0]}: #{val[1]}"}).join("\n") + "\n\nFiles:\n\t#{filez.join("\t")}")
|
189
117
|
end
|
190
118
|
|
191
119
|
def find(words, bucket = nil, cmd_opt = {})
|
192
120
|
option = {}
|
193
121
|
words_search = []
|
122
|
+
date = 0
|
194
123
|
words.each do |word|
|
124
|
+
date -= 1
|
195
125
|
case word
|
196
126
|
when /.*=.*/
|
197
127
|
#opzione
|
198
|
-
|
199
|
-
option["="][word.split("=")[0]] = word.split("=")[1]
|
128
|
+
words_search << word
|
200
129
|
when /.*>.*/
|
201
130
|
#opzione
|
202
|
-
|
203
|
-
option[">"][word.split(">")[0]] = word.split(">")[1]
|
131
|
+
words_search << word
|
204
132
|
when /.*<.*/
|
205
133
|
#opzione
|
206
|
-
option["<"] ||= {}
|
207
|
-
option["<"][word.split("<")[0]] = word.split("<")[1]
|
208
|
-
else
|
209
134
|
words_search << word
|
210
|
-
|
211
|
-
|
212
|
-
|
213
|
-
@db.each do |item|
|
214
|
-
option.each do |key,opts|
|
215
|
-
opts.each do |campo,val|
|
216
|
-
case key
|
217
|
-
when "="
|
218
|
-
case item[campo].class.to_s
|
219
|
-
when "Time"
|
220
|
-
results << item if item[campo] = Time.parse(val)
|
221
|
-
when "Fixnum"
|
222
|
-
results << item if item[campo] = val.to_i
|
223
|
-
else
|
224
|
-
results << item if item[campo] =~ /.*#{val}.*/
|
225
|
-
end
|
226
|
-
when "<"
|
227
|
-
case item[campo].class.to_s
|
228
|
-
when "Time"
|
229
|
-
results << item if item[campo] < Time.parse(val)
|
230
|
-
when "Fixnum"
|
231
|
-
results << item if item[campo] < val.to_i
|
232
|
-
else
|
233
|
-
results << item if item[campo] < val
|
234
|
-
end
|
235
|
-
when ">"
|
236
|
-
case item[campo].class.to_s
|
237
|
-
when "Time"
|
238
|
-
results << item if item[campo] > Time.parse(val)
|
239
|
-
when "Fixnum"
|
240
|
-
results << item if item[campo] > val.to_i
|
241
|
-
else
|
242
|
-
results << item if item[campo] > val
|
243
|
-
end
|
135
|
+
else
|
136
|
+
if word == "datetime"
|
137
|
+
date = 2
|
244
138
|
end
|
245
|
-
|
246
|
-
|
247
|
-
|
248
|
-
|
249
|
-
|
250
|
-
end
|
251
|
-
end
|
252
|
-
#results.sort!
|
253
|
-
and_results = []
|
254
|
-
if results.nitems > 0
|
255
|
-
prev = results[0]
|
256
|
-
results.each do |res|
|
257
|
-
test = results.select {|t| t == res}
|
258
|
-
#test = results.select {|t| t["aws_name"] == res["aws_name"]}
|
259
|
-
if words.nitems == test.nitems
|
260
|
-
and_results << res
|
139
|
+
if date == 0
|
140
|
+
#ci siamo uso i doppi apici
|
141
|
+
words_search << "'\\'#{word}\\''"
|
142
|
+
else
|
143
|
+
words_search << "'#{word}'"
|
261
144
|
end
|
262
|
-
end
|
263
145
|
end
|
264
|
-
results = and_results
|
265
146
|
end
|
266
|
-
|
267
|
-
|
268
|
-
|
269
|
-
|
270
|
-
|
147
|
+
if words_search.nitems == 1
|
148
|
+
#ho solo una parola uso startwith sul nome
|
149
|
+
search = "['aws_name' starts-with #{words_search[0]}] union ['description' starts-with #{words_search[0]}]"
|
150
|
+
elsif words_search.nitems == 0
|
151
|
+
#devo caricare tutto
|
152
|
+
search = "['datetime' > '\\'1970\\'']"
|
153
|
+
else
|
154
|
+
search = "[#{words_search.join(" ")}]"
|
271
155
|
end
|
272
|
-
|
273
|
-
|
274
|
-
|
275
|
-
|
276
|
-
|
277
|
-
|
278
|
-
|
279
|
-
|
280
|
-
#hres[:name].sort! {|x,y| x["date"] <=> y["date"]}
|
281
|
-
end
|
156
|
+
#p search
|
157
|
+
results = []
|
158
|
+
@sdb.query(@domain_db, search) do |result|
|
159
|
+
result[:items].each do |item|
|
160
|
+
hattr = @sdb.get_attributes(@domain_db, item)[:attributes]
|
161
|
+
hattr_ok = {}
|
162
|
+
hattr.each do |key,val|
|
163
|
+
hattr_ok[key] = val[0]
|
282
164
|
end
|
283
|
-
|
284
|
-
results = []
|
285
|
-
#p hres
|
286
|
-
hres.each do |key,val|
|
287
|
-
val.sort! {|x,y| x["date"] <=> y["date"]}
|
288
|
-
results << val[val.nitems - 1] if cmd_opt[:newer]
|
289
|
-
results << val[0] if cmd_opt[:older]
|
165
|
+
results << hattr_ok
|
290
166
|
end
|
291
167
|
end
|
292
168
|
results.sort! {|x,y| x["datetime"] <=> y["datetime"]}
|
@@ -339,12 +215,20 @@ class S3SyncDb
|
|
339
215
|
|
340
216
|
def delete(item)
|
341
217
|
S3Object.delete item["aws_name"], item["bucket"]
|
342
|
-
@
|
218
|
+
@sdb.delete_attributes(@domain_db, item["aws_name"])
|
343
219
|
end
|
344
220
|
|
345
|
-
def
|
346
|
-
|
221
|
+
def send_mail(subj, msg)
|
222
|
+
if(@config['mail_to'] and @config['mail_to'] != "")
|
223
|
+
msg = [ "Subject: #{subj}\n", "\n", "#{msg}\n" ]
|
224
|
+
Net::SMTP.start('localhost') do |smtp|
|
225
|
+
mail_to = @config['mail_to'].split(",").map {|t| t.strip}
|
226
|
+
ret = smtp.sendmail( msg, @config['mail_from'], mail_to)
|
227
|
+
#ret = smtp.sendmail( msg, "nik@nikpad.ath.cx", ['sacchi.nicola@gmail.com'] )
|
228
|
+
end
|
229
|
+
end
|
347
230
|
end
|
231
|
+
|
348
232
|
end
|
349
233
|
|
350
234
|
class Configure
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: niky81-s3rbackup
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.3.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Nicola Sacchi
|
@@ -9,7 +9,7 @@ autorequire:
|
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
11
|
|
12
|
-
date: 2008-
|
12
|
+
date: 2008-09-26 00:00:00 -07:00
|
13
13
|
default_executable:
|
14
14
|
dependencies:
|
15
15
|
- !ruby/object:Gem::Dependency
|
@@ -21,6 +21,15 @@ dependencies:
|
|
21
21
|
- !ruby/object:Gem::Version
|
22
22
|
version: 0.0.0
|
23
23
|
version:
|
24
|
+
- !ruby/object:Gem::Dependency
|
25
|
+
name: right_aws
|
26
|
+
version_requirement:
|
27
|
+
version_requirements: !ruby/object:Gem::Requirement
|
28
|
+
requirements:
|
29
|
+
- - ">"
|
30
|
+
- !ruby/object:Gem::Version
|
31
|
+
version: 0.0.0
|
32
|
+
version:
|
24
33
|
- !ruby/object:Gem::Dependency
|
25
34
|
name: OptionParser
|
26
35
|
version_requirement:
|