taps 0.2.10 → 0.2.11

data/README.rdoc

@@ -27,7 +27,6 @@ or when you want to push a local database to a taps server
 * Blob data may not transfer properly, I suspect that SQLite3 is modifying some native ruby objects.
 * Foreign Keys get lost in the schema transfer
 * Large tables (>1 million rows with a large number of columns) get slower as the offset gets larger. This is due to it being inefficient having large offset values in queries.
-* Character encoding problems should be more transparent but they're not, it usually involves looking at the server log to see the error.
 
 == Meta
 

data/VERSION.yml

@@ -1,4 +1,4 @@
 --- 
-:patch: 10
+:patch: 11
 :major: 0
 :minor: 2

data/lib/taps/cli.rb

@@ -36,6 +36,11 @@ class Cli < Thor
 		clientxfer(:cmd_send, database_url, remote_url)
 	end
 
+	desc "version", "Taps version"
+	def version
+		puts Taps.version
+	end
+
 	def clientxfer(method, database_url, remote_url)
 		if options[:chunksize]
 			Taps::Config.chunksize = options[:chunksize] < 100 ? 100 : options[:chunksize]

data/lib/taps/client_session.rb

@@ -70,11 +70,21 @@ class ClientSession
 	end
 
 	def cmd_send
-		verify_server
-		cmd_send_schema
-		cmd_send_data
-		cmd_send_indexes
-		cmd_send_reset_sequences
+		begin
+			verify_server
+			cmd_send_schema
+			cmd_send_data
+			cmd_send_indexes
+			cmd_send_reset_sequences
+		rescue RestClient::Exception => e
+			if e.respond_to?(:response)
+				puts "!!! Caught Server Exception"
+				puts "#{e.response.body}"
+				exit(1)
+			else
+				raise
+			end
+		end
 	end
 
 	def cmd_send_indexes
@@ -116,12 +126,14 @@ class ClientSession
 
 			offset = 0
 			loop do
-				rows = Taps::Utils.format_data(table.order(order).limit(chunksize, offset).all)
-				break if rows == { }
+				row_size = 0
+				chunksize = Taps::Utils.calculate_chunksize(chunksize) do |c|
+					rows = Taps::Utils.format_data(table.order(order).limit(c, offset).all)
+					break if rows == { }
 
-				gzip_data = Taps::Utils.gzip(Marshal.dump(rows))
+					row_size = rows[:data].size
+					gzip_data = Taps::Utils.gzip(Marshal.dump(rows))
 
-				chunksize = Taps::Utils.calculate_chunksize(chunksize) do
 					begin
 						session_resource["tables/#{table_name}"].post(gzip_data, http_headers({
 							:content_type => 'application/octet-stream',
@@ -135,8 +147,10 @@ class ClientSession
 					end
 				end
 
-				progress.inc(rows[:data].size)
-				offset += rows[:data].size
+				progress.inc(row_size)
+				offset += row_size
+
+				break if row_size == 0
 			end
 
 			progress.finish
@@ -156,11 +170,21 @@ class ClientSession
 	end
 
 	def cmd_receive
-		verify_server
-		cmd_receive_schema
-		cmd_receive_data
-		cmd_receive_indexes
-		cmd_reset_sequences
+		begin
+			verify_server
+			cmd_receive_schema
+			cmd_receive_data
+			cmd_receive_indexes
+			cmd_reset_sequences
+		rescue RestClient::Exception => e
+			if e.respond_to?(:response)
+				puts "!!! Caught Server Exception"
+				puts "#{e.response.body}"
+				exit(1)
+			else
+				raise
+			end
+		end
 	end
 
 	def cmd_receive_data
@@ -199,8 +223,8 @@ class ClientSession
 
 	def fetch_table_rows(table_name, chunksize, offset)
 		response = nil
-		chunksize = Taps::Utils.calculate_chunksize(chunksize) do
-			response = session_resource["tables/#{table_name}/#{chunksize}?offset=#{offset}"].get(http_headers)
+		chunksize = Taps::Utils.calculate_chunksize(chunksize) do |c|
+			response = session_resource["tables/#{table_name}/#{c}?offset=#{offset}"].get(http_headers)
 		end
 		raise CorruptedData unless Taps::Utils.valid_data?(response.to_s, response.headers[:taps_checksum])
 

data/lib/taps/server.rb

@@ -10,7 +10,8 @@ class Server < Sinatra::Default
 	end
 
 	error do
-		"Application error"
+		e = request.env['sinatra.error']
+		"Taps Server Error: #{e}"
 	end
 
 	before do

data/lib/taps/utils.rb

@@ -41,19 +41,32 @@ module Utils
 	end
 
 	def calculate_chunksize(old_chunksize)
-		t1 = Time.now
-		yield
+		chunksize = old_chunksize
+
+		retries = 0
+		begin
+			t1 = Time.now
+			yield chunksize
+		rescue Errno::EPIPE
+			retries += 1
+			raise if retries > 1
+			# we got disconnected, the chunksize could be too large
+			# so we're resetting to a very small value
+			chunksize = 100
+			retry
+		end
+
 		t2 = Time.now
 
 		diff = t2 - t1
 		new_chunksize = if diff > 3.0
-			(old_chunksize / 3).ceil
+			(chunksize / 3).ceil
 		elsif diff > 1.1
-			old_chunksize - 100
+			chunksize - 100
 		elsif diff < 0.8
-			old_chunksize * 2
+			chunksize * 2
 		else
-			old_chunksize + 100
+			chunksize + 100
 		end
 		new_chunksize = 100 if new_chunksize < 100
 		new_chunksize

data/spec/client_session_spec.rb

@@ -69,7 +69,7 @@ describe Taps::ClientSession do
 	it "fetches table rows given a chunksize and offset from taps server" do
 		@data = { :header => [ :x, :y ], :data => [ [1, 2], [3, 4] ] }
 		@gzip_data = Taps::Utils.gzip(Marshal.dump(@data))
-		Taps::Utils.stubs(:calculate_chunksize).with(1000).yields.returns(1000)
+		Taps::Utils.stubs(:calculate_chunksize).with(1000).yields(1000).returns(1000)
 
 		@response = mock('response')
 		@client.session_resource.stubs(:[]).with('tables/mytable/1000?offset=0').returns(mock('table resource'))

data/spec/utils_spec.rb

@@ -41,5 +41,9 @@ describe Taps::Utils do
 		Time.stubs(:now).returns(10.0).returns(11.1)
 		Taps::Utils.calculate_chunksize(1000) { }.should == 1100
 	end
+
+	it "will reset the chunksize to a small value if we got a broken pipe exception" do
+		Taps::Utils.calculate_chunksize(1000) { |c| raise Errno::EPIPE if c == 1000; c.should == 100 }.should == 200
+	end
 end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: taps
 version: !ruby/object:Gem::Version 
-  version: 0.2.10
+  version: 0.2.11
 platform: ruby
 authors: 
 - Ricardo Chimal, Jr.
@@ -10,7 +10,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2009-03-20 00:00:00 -07:00
+date: 2009-03-23 00:00:00 -07:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -85,18 +85,18 @@ extra_rdoc_files: []
 files: 
 - spec/base.rb
 - spec/schema_spec.rb
-- spec/utils_spec.rb
-- spec/client_session_spec.rb
 - spec/server_spec.rb
+- spec/client_session_spec.rb
+- spec/utils_spec.rb
 - lib/taps/db_session.rb
 - lib/taps/progress_bar.rb
-- lib/taps/cli.rb
+- lib/taps/client_session.rb
 - lib/taps/adapter_hacks.rb
 - lib/taps/adapter_hacks/invalid_text_limit.rb
 - lib/taps/adapter_hacks/non_rails_schema_dump.rb
 - lib/taps/schema.rb
 - lib/taps/config.rb
-- lib/taps/client_session.rb
+- lib/taps/cli.rb
 - lib/taps/server.rb
 - lib/taps/utils.rb
 - README.rdoc