matthewtodd-taps 0.2.19

data/lib/taps/utils.rb

@@ -0,0 +1,115 @@
+require 'zlib'
+require 'stringio'
+require 'time'
+require 'tempfile'
+
+module Taps
+module Utils
+	extend self
+
+	def checksum(data)
+		Zlib.crc32(data)
+	end
+
+	def valid_data?(data, crc32)
+		Zlib.crc32(data) == crc32.to_i
+	end
+
+	def gzip(data)
+		io = StringIO.new
+		gz = Zlib::GzipWriter.new(io)
+		gz.write data
+		gz.close
+		io.string
+	end
+
+	def gunzip(gzip_data)
+		io = StringIO.new(gzip_data)
+		gz = Zlib::GzipReader.new(io)
+		data = gz.read
+		gz.close
+		data
+	end
+
+	def format_data(data, string_columns)
+		return {} if data.size == 0
+		header = data[0].keys
+		only_data = data.collect do |row|
+			row = blobs_to_string(row, string_columns)
+			header.collect { |h| row[h] }
+		end
+		{ :header => header, :data => only_data }
+	end
+
+	# mysql text and blobs fields are handled the same way internally
+	# this is not true for other databases so we must check if the field is
+	# actually text and manually convert it back to a string
+	def incorrect_blobs(db, table)
+		return [] unless db.class.to_s == "Sequel::MySQL::Database"
+
+		columns = []
+		db.schema(table).each do |data|
+			column, cdata = data
+			columns << column if cdata[:db_type] =~ /text/
+		end
+		columns
+	end
+
+	def blobs_to_string(row, columns)
+		return row if columns.size == 0
+		columns.each do |c|
+			row[c] = row[c].to_s if row[c].kind_of?(Sequel::SQL::Blob)
+		end
+		row
+	end
+
+	def calculate_chunksize(old_chunksize)
+		chunksize = old_chunksize
+
+		retries = 0
+		begin
+			t1 = Time.now
+			yield chunksize
+		rescue Errno::EPIPE
+			retries += 1
+			raise if retries > 1
+			# we got disconnected, the chunksize could be too large
+			# so we're resetting to a very small value
+			chunksize = 100
+			retry
+		end
+
+		t2 = Time.now
+
+		diff = t2 - t1
+		new_chunksize = if diff > 3.0
+			(chunksize / 3).ceil
+		elsif diff > 1.1
+			chunksize - 100
+		elsif diff < 0.8
+			chunksize * 2
+		else
+			chunksize + 100
+		end
+		new_chunksize = 100 if new_chunksize < 100
+		new_chunksize
+	end
+
+	def primary_key(db, table)
+		if db.respond_to?(:primary_key)
+			db.primary_key(table)
+		else
+			db.schema(table).select { |c| c[1][:primary_key] }.map { |c| c.first }.shift
+		end
+	end
+
+	def order_by(db, table)
+		pkey = primary_key(db, table)
+		if pkey
+			[pkey.to_sym]
+		else
+			db[table].columns
+		end
+	end
+end
+end

data/spec/base.rb

@@ -0,0 +1,21 @@
+require 'rubygems'
+require 'bacon'
+require 'mocha'
+
+class Bacon::Context
+	include Mocha::Standalone
+
+	alias_method :old_it, :it
+	def it(description)
+		old_it(description) do
+			mocha_setup
+			yield
+			mocha_verify
+			mocha_teardown
+		end
+	end
+end
+
+require File.dirname(__FILE__) + '/../lib/taps/config'
+Taps::Config.taps_database_url = 'sqlite://test.db'
+Sequel.connect(Taps::Config.taps_database_url)

data/spec/client_session_spec.rb

@@ -0,0 +1,88 @@
+require File.dirname(__FILE__) + '/base'
+require File.dirname(__FILE__) + '/../lib/taps/client_session'
+
+describe Taps::ClientSession do
+	before do
+		@client = Taps::ClientSession.new('sqlite://my.db', 'http://example.com:3000', 1000)
+		@client.stubs(:session_resource).returns(mock('session resource'))
+	end
+
+	it "starts a session and yields the session object to the block" do
+		Taps::ClientSession.start('x', 'y', 1000) do |session|
+			session.database_url.should == 'x'
+			session.remote_url.should == 'y'
+			session.default_chunksize.should == 1000
+		end
+	end
+
+	it "opens the local db connection via sequel and the database url" do
+		Sequel.expects(:connect).with('sqlite://my.db').returns(:con)
+		@client.db.should == :con
+	end
+
+	it "creates a restclient resource to the remote server" do
+		@client.server.url.should == 'http://example.com:3000'
+	end
+
+	it "verifies the db version, receive the schema, data, indexes, then reset the sequences" do
+		@client.expects(:verify_server)
+		@client.expects(:cmd_receive_schema)
+		@client.expects(:cmd_receive_data)
+		@client.expects(:cmd_receive_indexes)
+		@client.expects(:cmd_reset_sequences)
+		@client.cmd_receive.should.be.nil
+	end
+
+	it "checks the version of the server by seeing if it has access" do
+		@client.stubs(:server).returns(mock('server'))
+		@request = mock('request')
+		@client.server.expects(:[]).with('/').returns(@request)
+		@request.expects(:get).with({:taps_version => Taps.compatible_version})
+
+		lambda { @client.verify_server }.should.not.raise
+	end
+
+	it "receives data from a remote taps server" do
+		@client.stubs(:puts)
+		@progressbar = mock('progressbar')
+		ProgressBar.stubs(:new).with('mytable', 2).returns(@progressbar)
+		@progressbar.stubs(:inc)
+		@progressbar.stubs(:finish)
+		@mytable = mock('mytable')
+		@client.expects(:fetch_remote_tables_info).returns([ { :mytable => 2 }, 2 ])
+		@client.stubs(:db).returns(mock('db'))
+		@client.db.stubs(:[]).with(:mytable).returns(@mytable)
+		@client.expects(:fetch_table_rows).with(:mytable, 1000, 0).returns([ 1000, { :header => [:x, :y], :data => [[1, 2], [3, 4]] } ])
+		@client.expects(:fetch_table_rows).with(:mytable, 1000, 2).returns([ 1000, { }])
+		@mytable.expects(:import).with([:x, :y], [[1, 2], [3, 4]])
+
+		lambda { @client.cmd_receive_data }.should.not.raise
+	end
+
+	it "fetches remote tables info from taps server" do
+		@marshal_data = Marshal.dump({ :mytable => 2 })
+		@client.session_resource.stubs(:[]).with('tables').returns(mock('tables'))
+		@client.session_resource['tables'].stubs(:get).with(:taps_version => Taps.compatible_version).returns(@marshal_data)
+		@client.fetch_remote_tables_info.should == [ { :mytable => 2 }, 2 ]
+	end
+
+	it "fetches table rows given a chunksize and offset from taps server" do
+		@data = { :header => [ :x, :y ], :data => [ [1, 2], [3, 4] ] }
+		@gzip_data = Taps::Utils.gzip(Marshal.dump(@data))
+		Taps::Utils.stubs(:calculate_chunksize).with(1000).yields(1000).returns(1000)
+
+		@response = mock('response')
+		@client.session_resource.stubs(:[]).with('tables/mytable/1000?offset=0').returns(mock('table resource'))
+		@client.session_resource['tables/mytable/1000?offset=0'].expects(:get).with(:taps_version => Taps.compatible_version).returns(@response)
+		@response.stubs(:to_s).returns(@gzip_data)
+		@response.stubs(:headers).returns({ :taps_checksum => Taps::Utils.checksum(@gzip_data) })
+		@client.fetch_table_rows('mytable', 1000, 0).should == [ 1000, { :header => [:x, :y], :data => [[1, 2], [3, 4]] } ]
+	end
+
+	it "hides the password in urls" do
+		@client.safe_url("postgres://postgres:password@localhost/mydb").should == "postgres://postgres:[hidden]@localhost/mydb"
+		@client.safe_url("postgres://postgres@localhost/mydb").should == "postgres://postgres@localhost/mydb"
+		@client.safe_url("http://x:y@localhost:5000").should == "http://x:[hidden]@localhost:5000"
+	end
+end
+

data/spec/schema_spec.rb

@@ -0,0 +1,45 @@
+require File.dirname(__FILE__) + '/base'
+require File.dirname(__FILE__) + '/../lib/taps/schema'
+
+describe Taps::Schema do
+	before do
+		Taps::AdapterHacks.stubs(:load)
+		@connection = mock("AR connection")
+		ActiveRecord::Base.stubs(:connection).returns(@connection)
+	end
+
+	it "parses a database url and returns a config hash for activerecord" do
+		Taps::Schema.create_config("postgres://myuser:mypass@localhost/mydb").should == {
+			'adapter' => 'postgresql',
+			'database' => 'mydb',
+			'username' => 'myuser',
+			'password' => 'mypass',
+			'host' => 'localhost'
+		}
+	end
+
+	it "translates sqlite in the database url to sqlite3" do
+		Taps::Schema.create_config("sqlite://mydb")['adapter'].should == 'sqlite3'
+	end
+
+	it "translates sqlite database path" do
+		Taps::Schema.create_config("sqlite://pathtodb/mydb")['database'].should == 'pathtodb/mydb'
+		Taps::Schema.create_config("sqlite:///pathtodb/mydb")['database'].should == '/pathtodb/mydb'
+		Taps::Schema.create_config("sqlite:///pathtodb/mydb?encoding=utf8")['database'].should == '/pathtodb/mydb'
+	end
+
+	it "connects activerecord to the database" do
+		Taps::Schema.expects(:create_config).with("postgres://myuser:mypass@localhost/mydb").returns("db config")
+		ActiveRecord::Base.expects(:establish_connection).with("db config").returns(true)
+		Taps::Schema.connection("postgres://myuser:mypass@localhost/mydb").should == true
+	end
+
+	it "resets the db tables' primary keys" do
+		Taps::Schema.stubs(:connection)
+		ActiveRecord::Base.connection.expects(:respond_to?).with(:reset_pk_sequence!).returns(true)
+		ActiveRecord::Base.connection.stubs(:tables).returns(['table1'])
+		ActiveRecord::Base.connection.expects(:reset_pk_sequence!).with('table1')
+		should.not.raise { Taps::Schema.reset_db_sequences("postgres://myuser:mypass@localhost/mydb") }
+	end
+end
+

data/spec/server_spec.rb

@@ -0,0 +1,33 @@
+require File.dirname(__FILE__) + '/base'
+require 'sinatra'
+require 'sinatra/test/bacon'
+
+require File.dirname(__FILE__) + '/../lib/taps/server'
+
+require 'pp'
+
+describe Taps::Server do
+	before do
+		Taps::Config.login = 'taps'
+		Taps::Config.password = 'tpass'
+
+		@app = Taps::Server
+		@auth_header = "Basic " + ["taps:tpass"].pack("m*")
+	end
+
+	it "asks for http basic authentication" do
+		get '/'
+		status.should == 401
+	end
+
+	it "verifies the client taps version" do
+		get('/', { }, { 'HTTP_AUTHORIZATION' => @auth_header, 'HTTP_TAPS_VERSION' => Taps.compatible_version })
+		status.should == 200
+	end
+
+	it "yells loudly if the client taps version doesn't match" do
+		get('/', { }, { 'HTTP_AUTHORIZATION' => @auth_header, 'HTTP_TAPS_VERSION' => '0.0.1' })
+		status.should == 417
+	end
+end
+

data/spec/utils_spec.rb

@@ -0,0 +1,61 @@
+require File.dirname(__FILE__) + '/base'
+require File.dirname(__FILE__) + '/../lib/taps/utils'
+
+describe Taps::Utils do
+	it "gunzips a string" do
+		@hello_world = "\037\213\b\000R\261\207I\000\003\313H\315\311\311W(\317/\312I\001\000\205\021J\r\v\000\000\000"
+		Taps::Utils.gunzip(@hello_world).should == "hello world"
+	end
+
+	it "gzips and gunzips a string and returns the same string" do
+		Taps::Utils.gunzip(Taps::Utils.gzip("hello world")).should == "hello world"
+	end
+
+	it "generates a checksum using crc32" do
+		Taps::Utils.checksum("hello world").should == Zlib.crc32("hello world")
+	end
+
+	it "formats a data hash into one hash that contains an array of headers and an array of array of data" do
+		first_row = { :x => 1, :y => 1 }
+		first_row.stubs(:keys).returns([:x, :y])
+		Taps::Utils.format_data([ first_row, { :x => 2, :y => 2 } ], []).should == { :header => [ :x, :y ], :data => [ [1, 1], [2, 2] ] }
+	end
+
+	it "scales chunksize down slowly when the time delta of the block is just over a second" do
+		Time.stubs(:now).returns(10.0).returns(11.5)
+		Taps::Utils.calculate_chunksize(1000) { }.should == 900
+	end
+
+	it "scales chunksize down fast when the time delta of the block is over 3 seconds" do
+		Time.stubs(:now).returns(10.0).returns(15.0)
+		Taps::Utils.calculate_chunksize(3000) { }.should == 1000
+	end
+
+	it "scales up chunksize fast when the time delta of the block is under 0.8 seconds" do
+		Time.stubs(:now).returns(10.0).returns(10.7)
+		Taps::Utils.calculate_chunksize(1000) { }.should == 2000
+	end
+
+	it "scales up chunksize slow when the time delta of the block is between 0.8 and 1.1 seconds" do
+		Time.stubs(:now).returns(10.0).returns(10.8)
+		Taps::Utils.calculate_chunksize(1000) { }.should == 1100
+
+		Time.stubs(:now).returns(10.0).returns(11.1)
+		Taps::Utils.calculate_chunksize(1000) { }.should == 1100
+	end
+
+	it "will reset the chunksize to a small value if we got a broken pipe exception" do
+		Taps::Utils.calculate_chunksize(1000) { |c| raise Errno::EPIPE if c == 1000; c.should == 100 }.should == 200
+	end
+
+	it "returns a list of columns that are text fields if the database is mysql" do
+		@db = mock("db")
+		@db.class.stubs(:to_s).returns("Sequel::MySQL::Database")
+		@db.stubs(:schema).with(:mytable).returns([
+			[:id, { :db_type => "int" }],
+			[:mytext, { :db_type => "text" }]
+		])
+		Taps::Utils.incorrect_blobs(@db, :mytable).should == [:mytext]
+	end
+end
+

metadata

@@ -0,0 +1,151 @@
+--- !ruby/object:Gem::Specification 
+name: matthewtodd-taps
+version: !ruby/object:Gem::Version 
+  version: 0.2.19
+platform: ruby
+authors: 
+- Ricardo Chimal, Jr.
+- Adam Wiggins
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2009-11-08 00:00:00 +03:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: sinatra
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - "="
+      - !ruby/object:Gem::Version 
+        version: 0.9.2
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: activerecord
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        version: 2.3.4
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: thor
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - "="
+      - !ruby/object:Gem::Version 
+        version: 0.9.9
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: rest-client
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 1.0.1
+    - - <
+      - !ruby/object:Gem::Version 
+        version: 1.1.0
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: sequel
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 3.0.0
+    - - <
+      - !ruby/object:Gem::Version 
+        version: 3.1.0
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: sqlite3-ruby
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        version: 1.2.0
+    version: 
+description: A simple database agnostic import/export app to transfer data to/from a remote database.
+email: ricardo@heroku.com
+executables: 
+- taps
+- schema
+extensions: []
+
+extra_rdoc_files: 
+- LICENSE
+- README.rdoc
+files: 
+- LICENSE
+- README.rdoc
+- Rakefile
+- VERSION.yml
+- bin/schema
+- bin/schema.cmd
+- bin/taps
+- lib/taps/adapter_hacks.rb
+- lib/taps/adapter_hacks/invalid_binary_limit.rb
+- lib/taps/adapter_hacks/invalid_text_limit.rb
+- lib/taps/adapter_hacks/mysql_invalid_primary_key.rb
+- lib/taps/adapter_hacks/non_rails_schema_dump.rb
+- lib/taps/cli.rb
+- lib/taps/client_session.rb
+- lib/taps/config.rb
+- lib/taps/db_session.rb
+- lib/taps/progress_bar.rb
+- lib/taps/schema.rb
+- lib/taps/server.rb
+- lib/taps/utils.rb
+- spec/base.rb
+- spec/client_session_spec.rb
+- spec/schema_spec.rb
+- spec/server_spec.rb
+- spec/utils_spec.rb
+has_rdoc: true
+homepage: http://github.com/matthewtodd/taps
+licenses: []
+
+post_install_message: 
+rdoc_options: 
+- --charset=UTF-8
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+requirements: []
+
+rubyforge_project: taps
+rubygems_version: 1.3.5
+signing_key: 
+specification_version: 3
+summary: simple database import/export app
+test_files: 
+- spec/base.rb
+- spec/client_session_spec.rb
+- spec/schema_spec.rb
+- spec/server_spec.rb
+- spec/utils_spec.rb