ricardochimal-taps 0.2.9 → 0.2.12

data/README.rdoc

@@ -27,7 +27,6 @@ or when you want to push a local database to a taps server
 * Blob data may not transfer properly, I suspect that SQLite3 is modifying some native ruby objects.
 * Foreign Keys get lost in the schema transfer
 * Large tables (>1 million rows with a large number of columns) get slower as the offset gets larger. This is due to it being inefficient having large offset values in queries.
-* Character encoding problems should be more transparent but they're not, it usually involves looking at the server log to see the error.
 
 == Meta
 

data/Rakefile

@@ -12,7 +12,7 @@ begin
     s.add_dependency 'activerecord', '= 2.2.2'
     s.add_dependency 'thor', '= 0.9.9'
     s.add_dependency 'rest-client', '~> 0.9.0'
-    s.add_dependency 'sequel', '~> 2.10.0'
+    s.add_dependency 'sequel', '~> 2.11.0'
     s.add_dependency 'sqlite3-ruby', '~> 1.2.0'
 
     s.rubyforge_project = "taps"

data/VERSION.yml

@@ -1,4 +1,4 @@
 --- 
-:patch: 9
+:patch: 12
 :major: 0
 :minor: 2

data/bin/taps

@@ -5,7 +5,7 @@ gem 'activerecord', '= 2.2.2'
 gem 'thor', '= 0.9.9'
 gem 'rest-client', '~> 0.9.0'
 gem 'sinatra', '~> 0.9.0'
-gem 'sequel', '~> 2.10.0'
+gem 'sequel', '~> 2.11.0'
 gem 'sqlite3-ruby', '~> 1.2.0'
 
 require File.dirname(__FILE__) + '/../lib/taps/cli'

data/lib/taps/cli.rb

@@ -36,6 +36,11 @@ class Cli < Thor
 		clientxfer(:cmd_send, database_url, remote_url)
 	end
 
+	desc "version", "Taps version"
+	def version
+		puts Taps.version
+	end
+
 	def clientxfer(method, database_url, remote_url)
 		if options[:chunksize]
 			Taps::Config.chunksize = options[:chunksize] < 100 ? 100 : options[:chunksize]

data/lib/taps/client_session.rb

@@ -70,11 +70,21 @@ class ClientSession
 	end
 
 	def cmd_send
-		verify_server
-		cmd_send_schema
-		cmd_send_data
-		cmd_send_indexes
-		cmd_send_reset_sequences
+		begin
+			verify_server
+			cmd_send_schema
+			cmd_send_data
+			cmd_send_indexes
+			cmd_send_reset_sequences
+		rescue RestClient::Exception => e
+			if e.respond_to?(:response)
+				puts "!!! Caught Server Exception"
+				puts "#{e.response.body}"
+				exit(1)
+			else
+				raise
+			end
+		end
 	end
 
 	def cmd_send_indexes
@@ -108,20 +118,22 @@ class ClientSession
 		db.tables.each do |table_name|
 			table = db[table_name]
 			count = table.count
-			columns = table.columns
-			order = columns.include?(:id) ? :id : columns.first
+			primary_key = db.primary_key(table_name.to_sym)
+			order = primary_key ? [primary_key.to_sym] : table.columns
 			chunksize = self.default_chunksize
 
 			progress = ProgressBar.new(table_name.to_s, count)
 
 			offset = 0
 			loop do
-				rows = Taps::Utils.format_data(table.order(order).limit(chunksize, offset).all)
-				break if rows == { }
+				row_size = 0
+				chunksize = Taps::Utils.calculate_chunksize(chunksize) do |c|
+					rows = Taps::Utils.format_data(table.order(*order).limit(c, offset).all)
+					break if rows == { }
 
-				gzip_data = Taps::Utils.gzip(Marshal.dump(rows))
+					row_size = rows[:data].size
+					gzip_data = Taps::Utils.gzip(Marshal.dump(rows))
 
-				chunksize = Taps::Utils.calculate_chunksize(chunksize) do
 					begin
 						session_resource["tables/#{table_name}"].post(gzip_data, http_headers({
 							:content_type => 'application/octet-stream',
@@ -135,8 +147,10 @@ class ClientSession
 					end
 				end
 
-				progress.inc(rows[:data].size)
-				offset += rows[:data].size
+				progress.inc(row_size)
+				offset += row_size
+
+				break if row_size == 0
 			end
 
 			progress.finish
@@ -156,11 +170,21 @@ class ClientSession
 	end
 
 	def cmd_receive
-		verify_server
-		cmd_receive_schema
-		cmd_receive_data
-		cmd_receive_indexes
-		cmd_reset_sequences
+		begin
+			verify_server
+			cmd_receive_schema
+			cmd_receive_data
+			cmd_receive_indexes
+			cmd_reset_sequences
+		rescue RestClient::Exception => e
+			if e.respond_to?(:response)
+				puts "!!! Caught Server Exception"
+				puts "#{e.response.body}"
+				exit(1)
+			else
+				raise
+			end
+		end
 	end
 
 	def cmd_receive_data
@@ -199,8 +223,8 @@ class ClientSession
 
 	def fetch_table_rows(table_name, chunksize, offset)
 		response = nil
-		chunksize = Taps::Utils.calculate_chunksize(chunksize) do
-			response = session_resource["tables/#{table_name}/#{chunksize}?offset=#{offset}"].get(http_headers)
+		chunksize = Taps::Utils.calculate_chunksize(chunksize) do |c|
+			response = session_resource["tables/#{table_name}/#{c}?offset=#{offset}"].get(http_headers)
 		end
 		raise CorruptedData unless Taps::Utils.valid_data?(response.to_s, response.headers[:taps_checksum])
 

data/lib/taps/db_session.rb

@@ -1,3 +1,5 @@
+require 'thread'
+
 Sequel::Model.db = Sequel.connect(Taps::Config.taps_database_url)
 
 class DbSession < Sequel::Model
@@ -9,17 +11,51 @@ class DbSession < Sequel::Model
 		timestamp :last_access
 	end
 
+	@@connections = {}
+	@@mutex = Mutex.new
+
 	def connection
-		@@connections ||= {}
-		@@connections[key] ||= Sequel.connect(database_url)
+		@@mutex.synchronize {
+			conn =
+				if @@connections.key?(key)
+					@@connections[key].first
+				else
+					Sequel.connect(database_url)
+				end
+			@@connections[key] = [conn, Time.now]
+			return conn
+		}
 	end
 
 	def disconnect
-		if defined? @@connections and @@connections[key]
-			@@connections[key].disconnect
-			@@connections.delete key
-		end
+		@@mutex.synchronize {
+			if @@connections.key?(key)
+				conn, time = @@connections.delete(key)
+				conn.disconnect
+			end
+		}
 	end
+
+	# Removes connections that have not been accessed within the
+	# past thirty seconds.
+	def self.cleanup
+		@@mutex.synchronize {
+			now = Time.now
+			@@connections.each do |key, (conn, time)|
+				if now - time > 30
+					@@connections.delete(key)
+					conn.disconnect
+				end
+			end
+		}
+	end
+
+	Thread.new {
+		while true
+			sleep 30
+			cleanup
+		end
+	}.run
 end
 
 DbSession.create_table! unless DbSession.table_exists?

data/lib/taps/server.rb

@@ -10,7 +10,8 @@ class Server < Sinatra::Default
 	end
 
 	error do
-		"Application error"
+		e = request.env['sinatra.error']
+		"Taps Server Error: #{e}"
 	end
 
 	before do
@@ -116,9 +117,9 @@ class Server < Sinatra::Default
 
 		db = session.connection
 		table = db[params[:table].to_sym]
-		columns = table.columns
-		order = columns.include?(:id) ? :id : columns.first
-		raw_data = Marshal.dump(Taps::Utils.format_data(table.order(order).limit(chunk, offset).all))
+		primary_key = db.primary_key(params[:table].to_sym)
+		order = primary_key ? [primary_key.to_sym] : table.columns
+		raw_data = Marshal.dump(Taps::Utils.format_data(table.order(*order).limit(chunk, offset).all))
 		gzip_data = Taps::Utils.gzip(raw_data)
 		response['Taps-Checksum'] = Taps::Utils.checksum(gzip_data).to_s
 		response['Content-Type'] = "application/octet-stream"

data/lib/taps/utils.rb

@@ -41,19 +41,32 @@ module Utils
 	end
 
 	def calculate_chunksize(old_chunksize)
-		t1 = Time.now
-		yield
+		chunksize = old_chunksize
+
+		retries = 0
+		begin
+			t1 = Time.now
+			yield chunksize
+		rescue Errno::EPIPE
+			retries += 1
+			raise if retries > 1
+			# we got disconnected, the chunksize could be too large
+			# so we're resetting to a very small value
+			chunksize = 100
+			retry
+		end
+
 		t2 = Time.now
 
 		diff = t2 - t1
 		new_chunksize = if diff > 3.0
-			(old_chunksize / 3).ceil
+			(chunksize / 3).ceil
 		elsif diff > 1.1
-			old_chunksize - 100
+			chunksize - 100
 		elsif diff < 0.8
-			old_chunksize * 2
+			chunksize * 2
 		else
-			old_chunksize + 100
+			chunksize + 100
 		end
 		new_chunksize = 100 if new_chunksize < 100
 		new_chunksize

data/spec/client_session_spec.rb

@@ -69,7 +69,7 @@ describe Taps::ClientSession do
 	it "fetches table rows given a chunksize and offset from taps server" do
 		@data = { :header => [ :x, :y ], :data => [ [1, 2], [3, 4] ] }
 		@gzip_data = Taps::Utils.gzip(Marshal.dump(@data))
-		Taps::Utils.stubs(:calculate_chunksize).with(1000).yields.returns(1000)
+		Taps::Utils.stubs(:calculate_chunksize).with(1000).yields(1000).returns(1000)
 
 		@response = mock('response')
 		@client.session_resource.stubs(:[]).with('tables/mytable/1000?offset=0').returns(mock('table resource'))

data/spec/utils_spec.rb

@@ -41,5 +41,9 @@ describe Taps::Utils do
 		Time.stubs(:now).returns(10.0).returns(11.1)
 		Taps::Utils.calculate_chunksize(1000) { }.should == 1100
 	end
+
+	it "will reset the chunksize to a small value if we got a broken pipe exception" do
+		Taps::Utils.calculate_chunksize(1000) { |c| raise Errno::EPIPE if c == 1000; c.should == 100 }.should == 200
+	end
 end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: ricardochimal-taps
 version: !ruby/object:Gem::Version 
-  version: 0.2.9
+  version: 0.2.12
 platform: ruby
 authors: 
 - Ricardo Chimal, Jr.
@@ -10,7 +10,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2009-03-18 00:00:00 -07:00
+date: 2009-03-31 00:00:00 -07:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -61,7 +61,7 @@ dependencies:
     requirements: 
     - - ~>
       - !ruby/object:Gem::Version 
-        version: 2.10.0
+        version: 2.11.0
     version: 
 - !ruby/object:Gem::Dependency 
   name: sqlite3-ruby
@@ -85,20 +85,20 @@ extra_rdoc_files: []
 files: 
 - spec/base.rb
 - spec/schema_spec.rb
-- spec/utils_spec.rb
-- spec/client_session_spec.rb
 - spec/server_spec.rb
-- lib/taps/db_session.rb
+- spec/client_session_spec.rb
+- spec/utils_spec.rb
 - lib/taps/progress_bar.rb
-- lib/taps/cli.rb
+- lib/taps/client_session.rb
+- lib/taps/server.rb
+- lib/taps/utils.rb
 - lib/taps/adapter_hacks.rb
 - lib/taps/adapter_hacks/invalid_text_limit.rb
 - lib/taps/adapter_hacks/non_rails_schema_dump.rb
 - lib/taps/schema.rb
 - lib/taps/config.rb
-- lib/taps/client_session.rb
-- lib/taps/server.rb
-- lib/taps/utils.rb
+- lib/taps/cli.rb
+- lib/taps/db_session.rb
 - README.rdoc
 - LICENSE
 - VERSION.yml