niky81-s3rbackup 0.2.0 → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/README +6 -1
- data/s3dbsync.rb +74 -19
- data/s3query.rb +18 -6
- data/s3rbackup.rb +2 -0
- metadata +1 -1
data/README
CHANGED
@@ -8,6 +8,9 @@ Local database with metadata of every file, database can be also automatically
|
|
8
8
|
saved to S3.
|
9
9
|
Query to database for file searching.
|
10
10
|
|
11
|
+
- Install
|
12
|
+
sudo gem install niky81-s3rbackup -s http://gems.github.com
|
13
|
+
|
11
14
|
- Configure
|
12
15
|
You need a file called ~/.s3rbackup/config.yml in your home directory with:
|
13
16
|
|
@@ -40,8 +43,10 @@ To query your database and get backup files it's based on command:
|
|
40
43
|
|
41
44
|
To search a word on database:
|
42
45
|
s3query.rb search test
|
43
|
-
To search
|
46
|
+
To search file only by name on database:
|
44
47
|
s3query.rb search name=test
|
48
|
+
To search only file bigger than 1Mb on database:
|
49
|
+
s3query.rb search size>1000000
|
45
50
|
to get all file that match name=test:
|
46
51
|
s3query.rb get name=test
|
47
52
|
to get and extract all file that match name=test
|
data/s3dbsync.rb
CHANGED
@@ -122,8 +122,14 @@ class S3SyncDb
|
|
122
122
|
crea_bucket()
|
123
123
|
name = dirs[0] if !name
|
124
124
|
tf = Tempfile.new("s3rbackup")
|
125
|
-
|
126
|
-
|
125
|
+
tf_l = Tempfile.new("s3rbackup-listfile")
|
126
|
+
tar = `tar -cv #{dirs.join(" ")} 2>#{tf_l.path} | bzip2 -9 > #{tf.path}`
|
127
|
+
|
128
|
+
filez = []
|
129
|
+
File.open(tf_l.path, 'r').each_line do |fh|
|
130
|
+
filez << fh
|
131
|
+
end
|
132
|
+
|
127
133
|
doc = {}
|
128
134
|
doc["name"] = name
|
129
135
|
doc["bucket"] = @config["bucket"]
|
@@ -135,21 +141,32 @@ class S3SyncDb
|
|
135
141
|
doc["size"] = File.size(tf.path)
|
136
142
|
doc["compression"] = "bz2"
|
137
143
|
doc["archive"] = "tar"
|
144
|
+
doc["files"] = filez.join("")
|
138
145
|
@db << doc
|
139
146
|
aws_name = "#{doc["name"]}_#{`date +%Y%m%d_%H.%M.%S`}_#{@db.find_index(doc)}".gsub("\n","")
|
140
147
|
doc["aws_name"] = aws_name
|
141
148
|
#FIXME Controllare che in db venga salvato aws_name
|
142
149
|
|
143
|
-
|
150
|
+
# Store it!
|
151
|
+
options = {}
|
152
|
+
#options[:access] = :public_read if @public
|
153
|
+
options["x-amz-meta-host"] = doc["host"]
|
154
|
+
options["x-amz-meta-user"] = doc["user"]
|
155
|
+
options["x-amz-meta-descrizione"] = doc["description"]
|
156
|
+
options["x-amz-meta-current_path"] = doc["current_path"]
|
157
|
+
options["x-amz-meta-size"] = doc["size"]
|
158
|
+
options["x-amz-meta-compression"] = doc["compression"]
|
159
|
+
options["x-amz-meta-archive"] = doc["archive"]
|
160
|
+
#options["x-amz-meta-files"] = doc["files"]
|
161
|
+
|
162
|
+
|
163
|
+
# options["x-amz-meta-sha1_hash"] = `sha1sum #{file}`.split[0] if @save_hash
|
164
|
+
# options["x-amz-meta-mtime"] = fstat.mtime.getutc.to_i if @save_time
|
165
|
+
# options["x-amz-meta-size"] = fstat.size if @save_size
|
166
|
+
|
167
|
+
store = S3Object.store(aws_name, open(tf.path), @config["bucket"], options)
|
144
168
|
obj = S3Object.find(aws_name, @config["bucket"])
|
145
|
-
obj.
|
146
|
-
obj.metadata[:user] = doc["user"]
|
147
|
-
obj.metadata[:descrizione] = doc["description"]
|
148
|
-
obj.metadata[:current_path] = doc["current_path"]
|
149
|
-
obj.metadata[:size] = doc["size"]
|
150
|
-
obj.metadata[:compression] = doc["compression"]
|
151
|
-
obj.metadata[:archive] = doc["archive"]
|
152
|
-
obj.store
|
169
|
+
#obj.store
|
153
170
|
obj.about.each do |key,val|
|
154
171
|
doc[key] = val
|
155
172
|
end
|
@@ -161,18 +178,56 @@ class S3SyncDb
|
|
161
178
|
option = {}
|
162
179
|
words_search = []
|
163
180
|
words.each do |word|
|
164
|
-
|
165
|
-
|
166
|
-
|
167
|
-
|
168
|
-
|
181
|
+
case word
|
182
|
+
when /.*=.*/
|
183
|
+
#opzione
|
184
|
+
option["="] ||= {}
|
185
|
+
option["="][word.split("=")[0]] = word.split("=")[1]
|
186
|
+
when /.*>.*/
|
187
|
+
#opzione
|
188
|
+
option[">"] ||= {}
|
189
|
+
option[">"][word.split(">")[0]] = word.split(">")[1]
|
190
|
+
when /.*<.*/
|
191
|
+
#opzione
|
192
|
+
option["<"] ||= {}
|
193
|
+
option["<"][word.split("<")[0]] = word.split("<")[1]
|
194
|
+
else
|
195
|
+
words_search << word
|
169
196
|
end
|
170
197
|
end
|
171
198
|
results = []
|
172
199
|
@db.each do |item|
|
173
|
-
option.each do |key,
|
174
|
-
|
175
|
-
|
200
|
+
option.each do |key,opts|
|
201
|
+
opts.each do |campo,val|
|
202
|
+
case key
|
203
|
+
when "="
|
204
|
+
case item[campo].class.to_s
|
205
|
+
when "Time"
|
206
|
+
results << item if item[campo] = Time.parse(val)
|
207
|
+
when "Fixnum"
|
208
|
+
results << item if item[campo] = val.to_i
|
209
|
+
else
|
210
|
+
results << item if item[campo] =~ /.*#{val}.*/
|
211
|
+
end
|
212
|
+
when "<"
|
213
|
+
case item[campo].class.to_s
|
214
|
+
when "Time"
|
215
|
+
results << item if item[campo] < Time.parse(val)
|
216
|
+
when "Fixnum"
|
217
|
+
results << item if item[campo] < val.to_i
|
218
|
+
else
|
219
|
+
results << item if item[campo] < val
|
220
|
+
end
|
221
|
+
when ">"
|
222
|
+
case item[campo].class.to_s
|
223
|
+
when "Time"
|
224
|
+
results << item if item[campo] > Time.parse(val)
|
225
|
+
when "Fixnum"
|
226
|
+
results << item if item[campo] > val.to_i
|
227
|
+
else
|
228
|
+
results << item if item[campo] > val
|
229
|
+
end
|
230
|
+
end
|
176
231
|
end
|
177
232
|
end
|
178
233
|
words_search.each do |word|
|
data/s3query.rb
CHANGED
@@ -1,3 +1,5 @@
|
|
1
|
+
#!/usr/bin/ruby
|
2
|
+
|
1
3
|
require 'rubygems'
|
2
4
|
require 'optparse'
|
3
5
|
|
@@ -60,6 +62,10 @@ class OptS3rquery
|
|
60
62
|
options[:first] = true
|
61
63
|
end
|
62
64
|
|
65
|
+
opts.on("--detail", "Detailed output (stats)") do |name|
|
66
|
+
options[:detail] = true
|
67
|
+
end
|
68
|
+
|
63
69
|
opts.on("--size", "Get size") do |name|
|
64
70
|
options[:first] = true
|
65
71
|
end
|
@@ -150,12 +156,18 @@ case command
|
|
150
156
|
when 'stats'
|
151
157
|
#get size
|
152
158
|
bucks_s = {}
|
153
|
-
|
154
|
-
|
155
|
-
|
156
|
-
|
157
|
-
|
158
|
-
|
159
|
+
if options[:detail]
|
160
|
+
results.each do |ret|
|
161
|
+
puts "#{ret["name"]}\t#{sprintf("%.2fMb", ret["size"].to_i / (1024.0 * 1024.0))}"
|
162
|
+
end
|
163
|
+
else
|
164
|
+
results.each do |ret|
|
165
|
+
bucks_s[ret["bucket"]] ||= 0
|
166
|
+
bucks_s[ret["bucket"]] += ret["size"]
|
167
|
+
end
|
168
|
+
bucks_s.each do |key,val|
|
169
|
+
puts "#{key}:\t#{sprintf("%.2fMb", val / (1024.0 * 1024.0))}"
|
170
|
+
end
|
159
171
|
end
|
160
172
|
when 'logs'
|
161
173
|
case sub_cmd
|
data/s3rbackup.rb
CHANGED