niky81-s3rbackup 0.2.0 → 0.2.1
Sign up to get free protection for your applications and to get access to all the features.
- data/README +6 -1
- data/s3dbsync.rb +74 -19
- data/s3query.rb +18 -6
- data/s3rbackup.rb +2 -0
- metadata +1 -1
data/README
CHANGED
@@ -8,6 +8,9 @@ Local database with metadata of every file, database can be also automatically
|
|
8
8
|
saved to S3.
|
9
9
|
Query to database for file searching.
|
10
10
|
|
11
|
+
- Install
|
12
|
+
sudo gem install niky81-s3rbackup -s http://gems.github.com
|
13
|
+
|
11
14
|
- Configure
|
12
15
|
You need a file called ~/.s3rbackup/config.yml in your home directory with:
|
13
16
|
|
@@ -40,8 +43,10 @@ To query your database and get backup files it's based on command:
|
|
40
43
|
|
41
44
|
To search a word on database:
|
42
45
|
s3query.rb search test
|
43
|
-
To search
|
46
|
+
To search file only by name on database:
|
44
47
|
s3query.rb search name=test
|
48
|
+
To search only file bigger than 1Mb on database:
|
49
|
+
s3query.rb search size>1000000
|
45
50
|
to get all file that match name=test:
|
46
51
|
s3query.rb get name=test
|
47
52
|
to get and extract all file that match name=test
|
data/s3dbsync.rb
CHANGED
@@ -122,8 +122,14 @@ class S3SyncDb
|
|
122
122
|
crea_bucket()
|
123
123
|
name = dirs[0] if !name
|
124
124
|
tf = Tempfile.new("s3rbackup")
|
125
|
-
|
126
|
-
|
125
|
+
tf_l = Tempfile.new("s3rbackup-listfile")
|
126
|
+
tar = `tar -cv #{dirs.join(" ")} 2>#{tf_l.path} | bzip2 -9 > #{tf.path}`
|
127
|
+
|
128
|
+
filez = []
|
129
|
+
File.open(tf_l.path, 'r').each_line do |fh|
|
130
|
+
filez << fh
|
131
|
+
end
|
132
|
+
|
127
133
|
doc = {}
|
128
134
|
doc["name"] = name
|
129
135
|
doc["bucket"] = @config["bucket"]
|
@@ -135,21 +141,32 @@ class S3SyncDb
|
|
135
141
|
doc["size"] = File.size(tf.path)
|
136
142
|
doc["compression"] = "bz2"
|
137
143
|
doc["archive"] = "tar"
|
144
|
+
doc["files"] = filez.join("")
|
138
145
|
@db << doc
|
139
146
|
aws_name = "#{doc["name"]}_#{`date +%Y%m%d_%H.%M.%S`}_#{@db.find_index(doc)}".gsub("\n","")
|
140
147
|
doc["aws_name"] = aws_name
|
141
148
|
#FIXME Controllare che in db venga salvato aws_name
|
142
149
|
|
143
|
-
|
150
|
+
# Store it!
|
151
|
+
options = {}
|
152
|
+
#options[:access] = :public_read if @public
|
153
|
+
options["x-amz-meta-host"] = doc["host"]
|
154
|
+
options["x-amz-meta-user"] = doc["user"]
|
155
|
+
options["x-amz-meta-descrizione"] = doc["description"]
|
156
|
+
options["x-amz-meta-current_path"] = doc["current_path"]
|
157
|
+
options["x-amz-meta-size"] = doc["size"]
|
158
|
+
options["x-amz-meta-compression"] = doc["compression"]
|
159
|
+
options["x-amz-meta-archive"] = doc["archive"]
|
160
|
+
#options["x-amz-meta-files"] = doc["files"]
|
161
|
+
|
162
|
+
|
163
|
+
# options["x-amz-meta-sha1_hash"] = `sha1sum #{file}`.split[0] if @save_hash
|
164
|
+
# options["x-amz-meta-mtime"] = fstat.mtime.getutc.to_i if @save_time
|
165
|
+
# options["x-amz-meta-size"] = fstat.size if @save_size
|
166
|
+
|
167
|
+
store = S3Object.store(aws_name, open(tf.path), @config["bucket"], options)
|
144
168
|
obj = S3Object.find(aws_name, @config["bucket"])
|
145
|
-
obj.
|
146
|
-
obj.metadata[:user] = doc["user"]
|
147
|
-
obj.metadata[:descrizione] = doc["description"]
|
148
|
-
obj.metadata[:current_path] = doc["current_path"]
|
149
|
-
obj.metadata[:size] = doc["size"]
|
150
|
-
obj.metadata[:compression] = doc["compression"]
|
151
|
-
obj.metadata[:archive] = doc["archive"]
|
152
|
-
obj.store
|
169
|
+
#obj.store
|
153
170
|
obj.about.each do |key,val|
|
154
171
|
doc[key] = val
|
155
172
|
end
|
@@ -161,18 +178,56 @@ class S3SyncDb
|
|
161
178
|
option = {}
|
162
179
|
words_search = []
|
163
180
|
words.each do |word|
|
164
|
-
|
165
|
-
|
166
|
-
|
167
|
-
|
168
|
-
|
181
|
+
case word
|
182
|
+
when /.*=.*/
|
183
|
+
#opzione
|
184
|
+
option["="] ||= {}
|
185
|
+
option["="][word.split("=")[0]] = word.split("=")[1]
|
186
|
+
when /.*>.*/
|
187
|
+
#opzione
|
188
|
+
option[">"] ||= {}
|
189
|
+
option[">"][word.split(">")[0]] = word.split(">")[1]
|
190
|
+
when /.*<.*/
|
191
|
+
#opzione
|
192
|
+
option["<"] ||= {}
|
193
|
+
option["<"][word.split("<")[0]] = word.split("<")[1]
|
194
|
+
else
|
195
|
+
words_search << word
|
169
196
|
end
|
170
197
|
end
|
171
198
|
results = []
|
172
199
|
@db.each do |item|
|
173
|
-
option.each do |key,
|
174
|
-
|
175
|
-
|
200
|
+
option.each do |key,opts|
|
201
|
+
opts.each do |campo,val|
|
202
|
+
case key
|
203
|
+
when "="
|
204
|
+
case item[campo].class.to_s
|
205
|
+
when "Time"
|
206
|
+
results << item if item[campo] = Time.parse(val)
|
207
|
+
when "Fixnum"
|
208
|
+
results << item if item[campo] = val.to_i
|
209
|
+
else
|
210
|
+
results << item if item[campo] =~ /.*#{val}.*/
|
211
|
+
end
|
212
|
+
when "<"
|
213
|
+
case item[campo].class.to_s
|
214
|
+
when "Time"
|
215
|
+
results << item if item[campo] < Time.parse(val)
|
216
|
+
when "Fixnum"
|
217
|
+
results << item if item[campo] < val.to_i
|
218
|
+
else
|
219
|
+
results << item if item[campo] < val
|
220
|
+
end
|
221
|
+
when ">"
|
222
|
+
case item[campo].class.to_s
|
223
|
+
when "Time"
|
224
|
+
results << item if item[campo] > Time.parse(val)
|
225
|
+
when "Fixnum"
|
226
|
+
results << item if item[campo] > val.to_i
|
227
|
+
else
|
228
|
+
results << item if item[campo] > val
|
229
|
+
end
|
230
|
+
end
|
176
231
|
end
|
177
232
|
end
|
178
233
|
words_search.each do |word|
|
data/s3query.rb
CHANGED
@@ -1,3 +1,5 @@
|
|
1
|
+
#!/usr/bin/ruby
|
2
|
+
|
1
3
|
require 'rubygems'
|
2
4
|
require 'optparse'
|
3
5
|
|
@@ -60,6 +62,10 @@ class OptS3rquery
|
|
60
62
|
options[:first] = true
|
61
63
|
end
|
62
64
|
|
65
|
+
opts.on("--detail", "Detailed output (stats)") do |name|
|
66
|
+
options[:detail] = true
|
67
|
+
end
|
68
|
+
|
63
69
|
opts.on("--size", "Get size") do |name|
|
64
70
|
options[:first] = true
|
65
71
|
end
|
@@ -150,12 +156,18 @@ case command
|
|
150
156
|
when 'stats'
|
151
157
|
#get size
|
152
158
|
bucks_s = {}
|
153
|
-
|
154
|
-
|
155
|
-
|
156
|
-
|
157
|
-
|
158
|
-
|
159
|
+
if options[:detail]
|
160
|
+
results.each do |ret|
|
161
|
+
puts "#{ret["name"]}\t#{sprintf("%.2fMb", ret["size"].to_i / (1024.0 * 1024.0))}"
|
162
|
+
end
|
163
|
+
else
|
164
|
+
results.each do |ret|
|
165
|
+
bucks_s[ret["bucket"]] ||= 0
|
166
|
+
bucks_s[ret["bucket"]] += ret["size"]
|
167
|
+
end
|
168
|
+
bucks_s.each do |key,val|
|
169
|
+
puts "#{key}:\t#{sprintf("%.2fMb", val / (1024.0 * 1024.0))}"
|
170
|
+
end
|
159
171
|
end
|
160
172
|
when 'logs'
|
161
173
|
case sub_cmd
|
data/s3rbackup.rb
CHANGED