niky81-s3rbackup 0.2.9 → 0.3.0

data/bin/s3query.rb

@@ -16,7 +16,7 @@ class OptS3rquery
 	def self.parse(args)
 		options = {}
 		opts = OptionParser.new do |opts|
-			opts.banner = "Usage: s3query.rb [options] <search|get|unpack|delete|stats|logs> <parameters> (parameters can be name=test or simply test)"
+			opts.banner = "Usage: s3query.rb [options] <search|get|unpack|delete|stats> <parameters> (parameters can be name=test or simply test)"
 		
 			opts.on("-s", "search words", String, "Search something") do |name|
 				options[:op] = "search"
@@ -74,22 +74,22 @@ class OptS3rquery
 				options[:first] = true
 			end
 	
-			opts.on("--log", "Log enabled") do |name|
-				options[:log] = true
+			opts.on("-u", "--config-number NUM", Integer, "Number of config to use if nil use first") do |name|
+				options[:config_num] = name
 			end
-		
-			opts.on("--nolog", "Log disabled") do |name|
-				options[:log] = false
+			
+			opts.on("--initialize", "Inizializza bucket and db") do |name|
+				options[:initialize] = true
 			end
-
-			opts.on("--bucket-log", String, "Bucket log NAME") do |name|
-				options[:bucket_log] = name
+			
+			opts.on("--destroy", "Destroy bucket and db") do |name|
+				options[:destroy] = true
 			end
 
-			opts.on("-u", "--config-number NUM", Integer, "Number of config to use if nil use first") do |name|
-				options[:config_num] = name
+			opts.on("--test", "Test something") do |name|
+				options[:test] = true
 			end
-	
+
 			opts.on_tail("-h", "--help", "Show this message") do
 				puts opts
 				exit
@@ -125,8 +125,6 @@ case command
 		results = s3db.find(ARGV, nil, options)
 		results = get_last(results) if options[:last]
 		results = get_first(results) if options[:first]
-	when 'logs'
-		sub_cmd = ARGV.shift
 end
 case command
 	when 'search'
@@ -160,7 +158,6 @@ case command
 			puts "Deleting of #{ret["aws_name"]}"
 			s3db.delete(ret)
 		end
-		s3db.salva_db
 	when 'stats'
 		#get size
 		bucks_s = {}
@@ -171,23 +168,21 @@ case command
 		else
 			results.each do |ret|
 				bucks_s[ret["bucket"]] ||= 0
-				bucks_s[ret["bucket"]] += ret["size"]
+				bucks_s[ret["bucket"]] += ret["size"].to_i
 			end
 			bucks_s.each do |key,val|
 				puts "#{key}:\t#{sprintf("%.2fMb", val / (1024.0 * 1024.0))}"
 			end
 		end
-	when 'logs'
-		case sub_cmd
-			when 'get'
-				logs = s3db.logs()
-				logs.each do |log|
-					p log
-				end
-			when 'delete'
-		end
-		#get logs bucket
 	else
-		puts "Some error occurred command #{command} not valid"
+		if options[:initialize]
+			s3db.initialize_db
+		elsif	options[:destroy]
+			s3db.destroy_db
+		elsif options[:test]
+			s3db.test
+		else
+			puts "Some error occurred command #{command} not valid"
+		end
 end
 

data/bin/s3rbackup.rb

@@ -78,4 +78,4 @@ config.current["bucket_log"] = options[:bucket_log] if options[:bucket_log]
 config.current["compression"] = options[:compression] if options[:compression]
 s3db = S3SyncDb.new(config.current)
 s3db.bak(ARGV,  options[:name],  options[:descr])
-s3db.salva_db
+#s3db.salva_db

data/lib/s3dbsync.rb

@@ -2,6 +2,9 @@ require 'rubygems'
 require 'aws/s3'
 require 'tempfile'
 require 'yaml'
+#nuovo aws
+require 'right_aws'
+require 'net/smtp'
 
 class S3SyncDb
 	include AWS::S3
@@ -10,94 +13,21 @@ class S3SyncDb
 		@conn = AWS::S3::Base.establish_connection!(
 	    					:access_key_id     => config["access_key_id"],
 			    			:secret_access_key => config["secret_access_key"])
+		@sdb = RightAws::SdbInterface.new(config["access_key_id"], config["secret_access_key"], 
+															{:multi_thread => false, :logger => Logger.new('/tmp/sdb.log')})
+		@s3 =  RightAws::S3.new(config["access_key_id"], config["secret_access_key"], 
+															{:multi_thread => false, :logger => Logger.new('/tmp/s3.log')})
 
 		@config = config
-		@db_file = "#{ENV['HOME']}/.s3rbackup/#{config['db_file']}"
-		@db_file = "#{ENV['HOME']}/.s3rbackup/s3db.yml" if !@db_file
-		@db_file_ver = "#{@db_file}.ver"
-		#apro il db
-		if !File.exists?(@db_file)
-			#download db
-			begin
-				buck_db = Bucket.find(config["bucket_db"])
-			  db_file = S3Object.find('s3rbackup_yaml_db', config["bucket_db"])
-			rescue
-				#devo crearlo
-				if config["sync_db"]
-					Bucket.create(config["bucket_db"])
-				end
-				@db = []
-				@version = 0
-				return
-			end
-			#lo copio in locale
-			open(@db_file, 'w') do |file|
-				S3Object.stream('s3rbackup_yaml_db', config["bucket_db"]) do |chunk|
-					file.write chunk
-		    end
-		  end
-			@version = db_file.metadata[:version].to_i
-			File.open(@db_file_ver, 'w') { |f| f.puts @version.to_s }
-		elsif config["sync_db"]
-			begin
-				buck_db = Bucket.find(config["bucket_db"])
-				db_file = S3Object.find('s3rbackup_yaml_db', config["bucket_db"])
-			rescue
-				#devo crearlo
-				if config["sync_db"]
-					Bucket.create(config["bucket_db"])
-				end
-				@db = YAML::load(File.open(@db_file))
-				@version ||= File.read(@db_file_ver).to_i
-				return
-			end
-			#se esiste e devo fare il sync
-			@version = db_file.metadata[:version].to_i
-			local_ver = File.read(@db_file_ver)
-			if @version > local_ver.to_i
-				#uso il remoto
-				open(@db_file, 'w') do |file|
-					db_file.stream do |chunk|
-						file.write chunk
-			    end
-			  end
-				File.open(@db_file_ver, 'w') { |f| f.puts @version.to_s }
-			else
-				#posso usare quello locale
-			end
-		end
-		#lo carico
-		@db = YAML::load(File.open(@db_file))
-		@version ||= File.read(@db_file_ver).to_i
-	end
-
-	def salva_locale
-		File.open(@db_file, 'w') { |f| f.puts @db.to_yaml }
+		#d'ora in poi uso il db su aws
+		@domain_db = config['bucket']
+		@bucket = config['bucket']
 	end
 
-	def nuova_versione
-		@version += 1
-		File.open(@db_file_ver, 'w') { |f| f.puts @version.to_s }
-	end
-
-	def aggiornaS3
-		salva_locale()
-		S3Object.store("s3rbackup_yaml_db", open(@db_file, "r"), @config["bucket_db"])
-		db_file = S3Object.find('s3rbackup_yaml_db', @config["bucket_db"])
-		db_file.metadata[:version] = @version.to_s
-		db_file.store
-	end
-
-	def salva_db
-		salva_locale()
-		nuova_versione()
-		aggiornaS3() if @config["sync_db"]
-	end
-
-	def crea_bucket()
-		#find bucket
-		@bucket = @config["bucket"]
-		@bucket_log = @config["bucket_log"] ? @config["bucket_log"] : "#{@bucket}-logs"
+	def initialize_db
+		puts "Creating #{@domain_db} db..."
+		@sdb.create_domain(@domain_db) 
+		puts "Creating #{@bucket} bucket..."
 		begin
 			bbackup = Bucket.find(@bucket)
 		rescue
@@ -105,21 +35,24 @@ class S3SyncDb
 				raise "Can't create bucket:#{@bucket}"
 			end
 		end
-		if @config["log"]
-			begin
-				blog = Bucket.find(@bucket_log)
-			rescue
-				Bucket.create(@bucket_log)
-			end
-			Bucket.enable_logging_for(
-				@bucket, 'target_bucket' => @bucket_log)
-		else
-			Bucket.disable_logging_for(@bucket) 
-		end
-	end	
+	end
+
+	def destroy_db
+		puts "Deleting bucket #{@bucket}..."
+		bucket1 = @s3.bucket(@bucket)
+		bucket1.delete(true)
+		puts "Deleting db #{@domain_db}..."
+		@sdb.delete_domain(@domain_db)
+	end
+
+	def test
+		#buk = @s3.buckets
+		buk = @s3.buckets.map{|b| b.name}
+ 		puts "Buckets on S3: #{buk.join(', ')}"
+	end
 
+	#s3 sdb
 	def bak(dirs, name, descr)
-		crea_bucket()
 		name = dirs[0] if !name
 		tf = Tempfile.new("s3rbackup")
 		tf_l = Tempfile.new("s3rbackup-listfile")
@@ -147,23 +80,24 @@ class S3SyncDb
 		doc = {}
 		doc["name"] = name
 		doc["bucket"] = @config["bucket"]
-		doc["datetime"] = Time.now
+		doc["datetime"] = "'#{DateTime.now}'"
+		#doc["date"] = "#{DateTime.now.strftime("%Y%m%d")}"
 		doc["current_path"] = `pwd`.gsub("\n","").to_s
-		doc["description"] = descr
+		doc["description"] = descr ? descr : "n/a"
 		doc["host"] = `hostname`.gsub("\n","").to_s
 		doc["user"] = `whoami`.gsub("\n","").to_s
 		doc["size"] = File.size(file_path)
 		doc["compression"] = @config["compression"]
 		doc["archive"] = "tar"
-		doc["files"] = filez.join("")
-		@db << doc
-		aws_name = "#{doc["name"]}_#{`date +%Y%m%d_%H.%M.%S`}_#{@db.index(doc)}".gsub("\n","")
+		#doc["files"] = filez.join("")
+		#@db << doc
+		aws_name = "#{doc["name"]}##{`date +%Y%m%d_%H.%M.%S`}".gsub("\n","")
 		doc["aws_name"] = aws_name
-		#FIXME Controllare che in db venga salvato aws_name
+		@sdb.put_attributes @domain_db, aws_name, doc
+		#TODO aggiungere md5
 
   # Store it!
 		options = {}
-		#options[:access] = :public_read if @public
 		options["x-amz-meta-host"] = doc["host"]
 		options["x-amz-meta-user"] = doc["user"]
 		options["x-amz-meta-descrizione"] = doc["description"]
@@ -171,12 +105,6 @@ class S3SyncDb
 		options["x-amz-meta-size"] = doc["size"]
 		options["x-amz-meta-compression"] = doc["compression"]
 		options["x-amz-meta-archive"] = doc["archive"]
-		#options["x-amz-meta-files"] = doc["files"]
-
-
-	#       options["x-amz-meta-sha1_hash"] = `sha1sum #{file}`.split[0] if @save_hash
-	#         options["x-amz-meta-mtime"] = fstat.mtime.getutc.to_i if @save_time
-	#           options["x-amz-meta-size"] = fstat.size if @save_size
 
 		store = S3Object.store(aws_name, open(file_path), @config["bucket"], options)
 		obj = S3Object.find(aws_name, @config["bucket"])
@@ -185,108 +113,56 @@ class S3SyncDb
 			doc[key] = val
 		end
 		#TODO aggiungere check
-		#TODO aggiornare db
+		send_mail("S3rbackup - Saved #{doc["name"]}", (doc.to_a.map {|val| "#{val[0]}: #{val[1]}"}).join("\n") + "\n\nFiles:\n\t#{filez.join("\t")}")
 	end
 
 	def find(words, bucket = nil, cmd_opt = {})
 		option = {}
 		words_search = []
+		date = 0
 		words.each do |word|
+			date -= 1
 			case word 
 				when /.*=.*/
 					#opzione
-					option["="] ||= {}
-					option["="][word.split("=")[0]] = word.split("=")[1]
+					words_search << word
 				when /.*>.*/
 					#opzione
-					option[">"] ||= {}
-					option[">"][word.split(">")[0]] = word.split(">")[1]
+					words_search << word
 				when /.*<.*/
 					#opzione
-					option["<"] ||= {}
-					option["<"][word.split("<")[0]] = word.split("<")[1]
-				else
 					words_search << word
-			end
-		end
-		results = []
-		@db.each do |item|
-			option.each do |key,opts|
-				opts.each do |campo,val|
-					case key
-						when "="
-							case item[campo].class.to_s
-								when "Time"
-									results << item if item[campo] = Time.parse(val) 
-								when "Fixnum"
-									results << item if item[campo] = val.to_i
-								else
-									results << item if item[campo] =~ /.*#{val}.*/
-							end
-						when "<"
-							case item[campo].class.to_s
-								when "Time"
-									results << item if item[campo] < Time.parse(val) 
-								when "Fixnum"
-									results << item if item[campo] < val.to_i
-								else
-									results << item if item[campo] < val
-							end
-						when ">"
-							case item[campo].class.to_s
-								when "Time"
-									results << item if item[campo] > Time.parse(val) 
-								when "Fixnum"
-									results << item if item[campo] > val.to_i
-								else
-									results << item if item[campo] > val
-							end
+				else
+					if word == "datetime"
+						date = 2
 					end
-				end
-			end
-			words_search.each do |word|
-				if item.values.join(" ") =~ /.*#{word}.*/
-					results << item
-				end
-			end
-			#results.sort!
-			and_results = []
-			if results.nitems > 0
-				prev = results[0]
-				results.each do |res|
-					test = results.select {|t| t == res}
-					#test = results.select {|t| t["aws_name"] == res["aws_name"]}
-					if words.nitems == test.nitems
-						and_results << res
+					if date == 0
+						#ci siamo uso i doppi apici
+						words_search << "'\\'#{word}\\''"
+					else
+						words_search << "'#{word}'"
 					end
-				end
 			end
-			results = and_results
 		end
-		#caso in cui voglio tutto
-		if words.nitems == 0
-			@db.each do |item|
-				results << item
-			end
+		if words_search.nitems == 1
+			#ho solo una parola uso startwith sul nome
+			search = "['aws_name' starts-with #{words_search[0]}] union ['description' starts-with #{words_search[0]}]"
+		elsif words_search.nitems == 0
+			#devo caricare tutto
+			search = "['datetime' > '\\'1970\\'']"
+		else
+			search = "[#{words_search.join(" ")}]"
 		end
-		results.uniq!
-		hres = {}
-		if cmd_opt[:newer] or cmd_opt[:older]
-			results.each do |item|
-				results.each do |item2|
-					if item["name"] == item2["name"]
-						hres[item["name"]] ||= []
-						hres[item["name"]] << item
-						#hres[:name].sort! {|x,y| x["date"] <=> y["date"]}
-					end
+		#p search
+		results = []
+		@sdb.query(@domain_db, search) do |result|
+			result[:items].each do |item|
+				hattr = @sdb.get_attributes(@domain_db, item)[:attributes]
+				hattr_ok = {}
+				hattr.each do |key,val|
+					hattr_ok[key] = val[0]
 				end
-			end
-			results = []
-			#p hres
-			hres.each do |key,val|
-				val.sort! {|x,y| x["date"] <=> y["date"]}
-				results << val[val.nitems - 1] if cmd_opt[:newer]
-				results << val[0] if cmd_opt[:older]
+				results << hattr_ok
 			end
 		end
 		results.sort! {|x,y| x["datetime"] <=> y["datetime"]}
@@ -339,12 +215,20 @@ class S3SyncDb
 
 	def delete(item)
 		S3Object.delete item["aws_name"], item["bucket"]
-		@db.delete(item)
+		@sdb.delete_attributes(@domain_db, item["aws_name"])
 	end
 
-	def log()
-		return Bucket.logs(@config["bucket_log"])
+	def send_mail(subj, msg)
+		if(@config['mail_to'] and @config['mail_to'] != "")
+			msg = [ "Subject: #{subj}\n", "\n", "#{msg}\n" ]
+			Net::SMTP.start('localhost') do |smtp|
+				mail_to = @config['mail_to'].split(",").map {|t| t.strip} 
+			  ret = smtp.sendmail( msg,  @config['mail_from'], mail_to)
+			  #ret = smtp.sendmail( msg,  "nik@nikpad.ath.cx", ['sacchi.nicola@gmail.com'] )
+			end
+		end
 	end
+
 end
 
 class Configure

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: niky81-s3rbackup
 version: !ruby/object:Gem::Version 
-  version: 0.2.9
+  version: 0.3.0
 platform: ruby
 authors: 
 - Nicola Sacchi
@@ -9,7 +9,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2008-06-27 00:00:00 -07:00
+date: 2008-09-26 00:00:00 -07:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -21,6 +21,15 @@ dependencies:
       - !ruby/object:Gem::Version 
         version: 0.0.0
     version: 
+- !ruby/object:Gem::Dependency 
+  name: right_aws
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">"
+      - !ruby/object:Gem::Version 
+        version: 0.0.0
+    version: 
 - !ruby/object:Gem::Dependency 
   name: OptionParser
   version_requirement: