niky81-s3rbackup 0.3.4 → 0.4.0

data/bin/s3query.rb

@@ -16,7 +16,7 @@ class OptS3rquery
 	def self.parse(args)
 		options = {}
 		opts = OptionParser.new do |opts|
-			opts.banner = "Usage: s3query.rb [options] <search|get|unpack|delete|stats> <parameters> (parameters can be name=test or simply test)"
+			opts.banner = "Usage: s3query.rb [options] <search|get|unpack|delete|stats|delete_older> <parameters> (parameters can be name=test or simply test)"
 		
 			opts.on("-s", "search words", String, "Search something") do |name|
 				options[:op] = "search"
@@ -78,6 +78,10 @@ class OptS3rquery
 				options[:config_num] = name
 			end
 			
+			opts.on("-e", "--item-to-keep NUM", Integer, "Number of item to keep after deleting the olders (delete_older)") do |name|
+				options[:delete_older] = name
+			end
+			
 			opts.on("--initialize", "Inizializza bucket and db") do |name|
 				options[:initialize] = true
 			end
@@ -85,7 +89,7 @@ class OptS3rquery
 			opts.on("--destroy", "Destroy bucket and db") do |name|
 				options[:destroy] = true
 			end
-
+			
 			opts.on("--test", "Test something") do |name|
 				options[:test] = true
 			end
@@ -129,7 +133,7 @@ s3db = S3SyncDb.new(config.current)
 
 command = ARGV.shift
 case command
-	when /[search|get|unpack|delete|stats]/
+	when /[search|get|unpack|delete|stats|delete_older]/
 		results = s3db.find(ARGV, nil, options)
 		results = get_last(results) if options[:last]
 		results = get_first(results) if options[:first]
@@ -166,6 +170,27 @@ case command
 			puts "Deleting of #{ret["aws_name"]}"
 			s3db.delete(ret)
 		end
+	when 'delete_older'
+		#cancella i piu' vecchi mantenendo options[:delete_older] item
+		# per ongi nome trovato
+		if options[:delete_older] == nil
+			options[:delete_older] = 2
+		end
+		group = {}
+		results.each do |ret|
+			group[ret["name"]] ||= []
+			group[ret["name"]] << ret
+		end
+		group.each do |key, arr_ret|
+			arr_ret.sort! {|x,y| x["datetime"] <=> y["datetime"] }
+			options[:delete_older].times do |num|
+				arr_ret.pop
+			end 
+			arr_ret.each do |ret|
+				puts "Deleting of #{ret["aws_name"]}"
+				s3db.delete(ret)
+			end
+		end
 	when 'stats'
 		#get size
 		bucks_s = {}

data/bin/s3rbackup.rb

@@ -40,6 +40,10 @@ class OptS3rbackup
 				options[:log] = true
 			end
 		
+			opts.on("--md5", "Check md5, upload only if md5 differ") do |name|
+				options[:md5_check] = true
+			end
+
 			opts.on("--nolog", "Log disabled") do |name|
 				options[:log] = false
 			end
@@ -77,5 +81,5 @@ config.current["log"] = options[:log] if options[:log] != nil
 config.current["bucket_log"] = options[:bucket_log] if options[:bucket_log]
 config.current["compression"] = options[:compression] if options[:compression]
 s3db = S3SyncDb.new(config.current)
-s3db.bak(ARGV,  options[:name],  options[:descr])
+s3db.bak(ARGV,  options[:name],  options[:descr], options[:md5_check])
 #s3db.salva_db

data/lib/s3dbsync.rb

@@ -60,7 +60,7 @@ class S3SyncDb
 	end
 
 	#s3 sdb
-	def bak(dirs, name, descr)
+	def bak(dirs, name, descr, md5_check = false)
 		name = dirs[0] if !name
 		tf = Tempfile.new("s3rbackup")
 		tf_l = Tempfile.new("s3rbackup-listfile")
@@ -95,45 +95,58 @@ class S3SyncDb
 		doc["host"] = `hostname`.gsub("\n","").to_s
 		doc["user"] = `whoami`.gsub("\n","").to_s
 		doc["size"] = File.size(file_path)
+		doc["md5"] = `md5sum #{file_path} | cut -d " " -f 1`.gsub("\n", "")
 		doc["compression"] = @config["compression"]
 		doc["archive"] = "tar"
 		#@db << doc
 		aws_name = "#{doc["name"]}##{`date +%Y%m%d_%H.%M.%S`}".gsub("\n","")
 		doc["aws_name"] = aws_name
-		@sdb.put_attributes @domain_db, aws_name, doc
-		if @config['files_db']
-			#tutti in uno
-			filez_name = []
-			filez.each do |fil|
-				begin
-					filez_name << File.basename(fil.gsub("\n", ""))
-				rescue
+
+		carica = true
+		if md5_check
+			search = "['md5' = '#{doc["md5"]}']"
+			@sdb.query(@domain_db, search) do |result|
+				result[:items].each do |item|
+					hattr = @sdb.get_attributes(@domain_db, item)[:attributes]
+					if hattr["name"][0] == doc["name"]
+						carica = false
+					end
 				end
 			end
-			@sdb.put_attributes @config['files_db'], aws_name, { 
-																:files_full => filez.map {|t| t.gsub("\n","")}, 
-																:files_name => filez_name}
 		end
-
-		#TODO aggiungere md5
-  # Store it!
-		options = {}
-		options["x-amz-meta-host"] = doc["host"]
-		options["x-amz-meta-user"] = doc["user"]
-		options["x-amz-meta-descrizione"] = doc["description"]
-		options["x-amz-meta-current_path"] = doc["current_path"]
-		options["x-amz-meta-size"] = doc["size"]
-		options["x-amz-meta-compression"] = doc["compression"]
-		options["x-amz-meta-archive"] = doc["archive"]
-
-		store = S3Object.store(aws_name, open(file_path), @config["bucket"], options)
-		#obj = S3Object.find(aws_name, @config["bucket"])
-		#obj.store
-		#obj.about.each do |key,val|
-		#	doc[key] = val
-		#end
-		#TODO aggiungere check
-		send_mail("S3rbackup - Saved #{doc["name"]}", (doc.to_a.map {|val| "#{val[0]}: #{val[1]}"}).join("\n") + "\n\nFiles:\n\t#{filez.join("\t")}")
+		if carica 
+			@sdb.put_attributes @domain_db, aws_name, doc
+			if @config['files_db']
+				#tutti in uno
+				filez_name = []
+				filez.each do |fil|
+					begin
+						filez_name << File.basename(fil.gsub("\n", ""))
+					rescue
+					end
+				end
+				@sdb.put_attributes @config['files_db'], aws_name, { 
+																	:files_full => filez.map {|t| t.gsub("\n","")}, 
+																	:files_name => filez_name}
+			end
+	
+			#TODO aggiungere md5
+	  # Store it!
+			options = {}
+	#		options["x-amz-meta-host"] = doc["host"]
+	#		options["x-amz-meta-user"] = doc["user"]
+	#		options["x-amz-meta-descrizione"] = doc["description"]
+	#		options["x-amz-meta-current_path"] = doc["current_path"]
+	#		options["x-amz-meta-size"] = doc["size"]
+	#		options["x-amz-meta-compression"] = doc["compression"]
+	#		options["x-amz-meta-archive"] = doc["archive"]
+	
+			store = S3Object.store(aws_name, open(file_path), @config["bucket"], options)
+			#TODO aggiungere check
+			send_mail("S3rbackup - Saved #{doc["name"]}", (doc.to_a.map {|val| "#{val[0]}: #{val[1]}"}).join("\n") + "\n\nFiles:\n\t#{filez.join("\t")}")
+		else
+			send_mail("S3rbackup - Not Saved #{doc["name"]} - md5 already exist", (doc.to_a.map {|val| "#{val[0]}: #{val[1]}"}).join("\n") + "\n\nFiles:\n\t#{filez.join("\t")}")
+		end
 	end
 
 	def find(words, bucket = nil, cmd_opt = {})

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: niky81-s3rbackup
 version: !ruby/object:Gem::Version 
-  version: 0.3.4
+  version: 0.4.0
 platform: ruby
 authors: 
 - Nicola Sacchi
@@ -9,7 +9,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2008-09-28 00:00:00 -07:00
+date: 2008-11-28 00:00:00 -08:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency