taps 0.2.26 → 0.3.0

data/lib/taps/schema.rb

@@ -1,93 +1,75 @@
-require 'active_record'
-require 'active_support'
-require 'stringio'
-require 'uri'
+require 'sequel'
+require 'sequel/extensions/schema_dumper'
+require 'sequel/extensions/migration'
 
-require File.dirname(__FILE__) + '/adapter_hacks'
+require 'json'
 
 module Taps
 module Schema
 	extend self
 
-	def create_config(url)
-		uri = URI.parse(url)
-		adapter = uri.scheme
-		adapter = 'postgresql' if adapter == 'postgres'
-		adapter = 'sqlite3' if adapter == 'sqlite'
-		config = {
-			'adapter' => adapter,
-			'database' => uri.path.blank? ? uri.host : uri.path.split('/')[1],
-			'username' => uri.user,
-			'password' => uri.password,
-			'host' => uri.host,
-		}
-		config = sqlite_config(url) if config['adapter'] == 'sqlite3'
-		config
+	def dump(database_url)
+		db = Sequel.connect(database_url)
+		db.dump_schema_migration(:indexes => false)
 	end
 
-	def sqlite_config(url)
-		m = %r{(sqlite3?)://(.+)}.match(url)
-		database = m[2]
-		database, q = database.split('?')
-		{ 'adapter' => 'sqlite3', 'database' => database }
+	def dump_table(database_url, table)
+		Sequel.connect(database_url) do |db|
+			<<END_MIG
+Class.new(Sequel::Migration) do
+	def up
+		#{db.dump_table_schema(table, :indexes => false)}
 	end
-
-	def connection(database_url)
-		config = create_config(database_url)
-		c = ActiveRecord::Base.establish_connection(config)
-		Taps::AdapterHacks.load(config['adapter'])
-		c
+end
+END_MIG
+		end
 	end
 
-	def dump(database_url)
-		connection(database_url)
-
-		stream = StringIO.new
-		ActiveRecord::SchemaDumper.ignore_tables = []
-		ActiveRecord::SchemaDumper.dump(ActiveRecord::Base.connection, stream)
-		stream.string
+	def indexes(database_url)
+		db = Sequel.connect(database_url)
+		db.dump_indexes_migration
 	end
 
-	def dump_without_indexes(database_url)
-		schema = dump(database_url)
-		schema.split("\n").collect do |line|
-			if line =~ /^\s+add_index/
-				line = "##{line}"
+	def indexes_individual(database_url)
+		idxs = {}
+		Sequel.connect(database_url) do |db|
+			tables = db.tables
+			tables.each do |table|
+				idxs[table] = db.send(:dump_table_indexes, table, :add_index, {}).split("\n")
 			end
-			line
-		end.join("\n")
-	end
+		end
 
-	def indexes(database_url)
-		schema = dump(database_url)
-		schema.split("\n").collect do |line|
-			line if line =~ /^\s+add_index/
-		end.uniq.join("\n")
+		idxs.each do |table, indexes|
+			idxs[table] = indexes.map do |idx|
+				<<END_MIG
+Class.new(Sequel::Migration) do
+	def up
+		#{idx}
+	end
+end
+END_MIG
+			end
+		end
+		idxs.to_json
 	end
 
 	def load(database_url, schema)
-		connection(database_url)
-		eval(schema)
+		Sequel.connect(database_url) do |db|
+			eval(schema).apply(db, :up)
+		end
 	end
 
 	def load_indexes(database_url, indexes)
-		connection(database_url)
-
-		schema =<<EORUBY
-ActiveRecord::Schema.define do
-	#{indexes}
-end
-EORUBY
-		eval(schema)
+		Sequel.connect(database_url) do |db|
+			eval(indexes).apply(db, :up)
+		end
 	end
 
 	def reset_db_sequences(database_url)
-		connection(database_url)
-
-		if ActiveRecord::Base.connection.respond_to?(:reset_pk_sequence!)
-			ActiveRecord::Base.connection.tables.each do |table|
-				ActiveRecord::Base.connection.reset_pk_sequence!(table)
-			end
+		db = Sequel.connect(database_url)
+		return unless db.respond_to?(:reset_primary_key_sequence)
+		db.tables.each do |table|
+			db.reset_primary_key_sequence(table)
 		end
 	end
 end

data/lib/taps/server.rb

@@ -1,17 +1,20 @@
 require 'sinatra/base'
-require File.dirname(__FILE__) + '/config'
-require File.dirname(__FILE__) + '/utils'
-require File.dirname(__FILE__) + '/db_session'
+require 'taps/config'
+require 'taps/utils'
+require 'taps/db_session'
+require 'taps/data_stream'
 
 module Taps
-class Server < Sinatra::Default
+class Server < Sinatra::Base
 	use Rack::Auth::Basic do |login, password|
 		login == Taps::Config.login && password == Taps::Config.password
 	end
 
+	use Rack::Deflater unless ENV['NO_DEFLATE']
+
 	error do
 		e = request.env['sinatra.error']
-		"Taps Server Error: #{e}"
+		"Taps Server Error: #{e}\n#{e.backtrace}"
 	end
 
 	before do
@@ -27,38 +30,46 @@ class Server < Sinatra::Default
 
 	post '/sessions' do
 		key = rand(9999999999).to_s
-		database_url = Taps::Config.database_url || request.body.string
+
+		if ENV['NO_DEFAULT_DATABASE_URL']
+			database_url = request.body.string
+		else
+			database_url = Taps::Config.database_url || request.body.string
+		end
 
 		DbSession.create(:key => key, :database_url => database_url, :started_at => Time.now, :last_access => Time.now)
 
 		"/sessions/#{key}"
 	end
 
-	post '/sessions/:key/tables/:table' do
+	post '/sessions/:key/push/table' do
 		session = DbSession.filter(:key => params[:key]).first
 		halt 404 unless session
 
-		gzip_data = request.body.read
-		halt 412 unless Taps::Utils.valid_data?(gzip_data, request.env['HTTP_TAPS_CHECKSUM'])
+		json = DataStream.parse_json(params[:json])
 
-		rows = Marshal.load(Taps::Utils.gunzip(gzip_data))
-
-		db = session.connection
-		table = db[params[:table].to_sym]
-		table.import(rows[:header], rows[:data])
+		size = 0
+		session.conn do |db|
+			begin
+				stream = DataStream.factory(db, json[:state])
+				size = stream.fetch_remote_in_server(params)
+			rescue Taps::DataStream::CorruptedData
+				halt 412
+			end
+		end
 
-		"#{rows[:data].size}"
+		# TODO: return the stream's state with the size
+		size.to_s
 	end
 
-	post '/sessions/:key/reset_sequences' do
+	post '/sessions/:key/push/reset_sequences' do
 		session = DbSession.filter(:key => params[:key]).first
 		halt 404 unless session
 
-		schema_app = File.dirname(__FILE__) + '/../../bin/schema'
 		Taps::Utils.schema_bin(:reset_db_sequences, session.database_url)
 	end
 
-	post '/sessions/:key/schema' do
+	post '/sessions/:key/push/schema' do
 		session = DbSession.filter(:key => params[:key]).first
 		halt 404 unless session
 
@@ -66,7 +77,7 @@ class Server < Sinatra::Default
 		Taps::Utils.load_schema(session.database_url, schema_data)
 	end
 
-	post '/sessions/:key/indexes' do
+	post '/sessions/:key/push/indexes' do
 		session = DbSession.filter(:key => params[:key]).first
 		halt 404 unless session
 
@@ -74,63 +85,78 @@ class Server < Sinatra::Default
 		Taps::Utils.load_indexes(session.database_url, index_data)
 	end
 
-	get '/sessions/:key/schema' do
+	post '/sessions/:key/pull/schema' do
 		session = DbSession.filter(:key => params[:key]).first
 		halt 404 unless session
 
-		schema_app = File.dirname(__FILE__) + '/../../bin/schema'
-		Taps::Utils.schema_bin(:dump, session.database_url)
+		Taps::Utils.schema_bin(:dump_table, session.database_url, params[:table_name])
 	end
 
-	get '/sessions/:key/indexes' do
+	get '/sessions/:key/pull/indexes' do
 		session = DbSession.filter(:key => params[:key]).first
 		halt 404 unless session
 
-		schema_app = File.dirname(__FILE__) + '/../../bin/schema'
-		Taps::Utils.schema_bin(:indexes, session.database_url)
+		content_type 'application/json'
+		Taps::Utils.schema_bin(:indexes_individual, session.database_url)
 	end
 
-	get '/sessions/:key/tables' do
+	get '/sessions/:key/pull/table_names' do
 		session = DbSession.filter(:key => params[:key]).first
 		halt 404 unless session
 
-		db = session.connection
-		tables = db.tables
-
-		tables_with_counts = tables.inject({}) do |accum, table|
-			accum[table] = db[table].count
-			accum
+		tables = []
+		session.conn do |db|
+			tables = db.tables
 		end
 
-		Marshal.dump(tables_with_counts)
+		content_type 'application/json'
+		tables.to_json
+	end
+
+	post '/sessions/:key/pull/table_count' do
+		session = DbSession.filter(:key => params[:key]).first
+		halt 404 unless session
+
+		count = 0
+		session.conn do |db|
+			count = db[ params[:table].to_sym ].count
+		end
+		count.to_s
 	end
 
-	get '/sessions/:key/tables/:table/:chunk' do
+	post '/sessions/:key/pull/table' do
 		session = DbSession.filter(:key => params[:key]).first
 		halt 404 unless session
 
-		chunk = params[:chunk].to_i
-		chunk = 500 if chunk < 1
-
-		offset = params[:offset].to_i
-		offset = 0 if offset < 0
-
-		db = session.connection
-		table = db[params[:table].to_sym]
-		order = Taps::Utils.order_by(db, params[:table].to_sym)
-		string_columns = Taps::Utils.incorrect_blobs(db, params[:table].to_sym)
-		raw_data = Marshal.dump(Taps::Utils.format_data(table.order(*order).limit(chunk, offset).all, string_columns))
-		gzip_data = Taps::Utils.gzip(raw_data)
-		response['Taps-Checksum'] = Taps::Utils.checksum(gzip_data).to_s
-		response['Content-Type'] = "application/octet-stream"
-		gzip_data
+		encoded_data = nil
+		stream = nil
+
+		session.conn do |db|
+			state = JSON.parse(params[:state]).symbolize_keys
+			stream = Taps::DataStream.factory(db, state)
+			encoded_data = stream.fetch.first
+		end
+
+		checksum = Taps::Utils.checksum(encoded_data).to_s
+		json = { :checksum => checksum, :state => stream.to_hash }.to_json
+
+		content, content_type_value = Taps::Multipart.create do |r|
+			r.attach :name => :encoded_data,
+				:payload => encoded_data,
+				:content_type => 'application/octet-stream'
+			r.attach :name => :json,
+				:payload => json,
+				:content_type => 'application/json'
+		end
+
+		content_type content_type_value
+		content
 	end
 
 	delete '/sessions/:key' do
 		session = DbSession.filter(:key => params[:key]).first
 		halt 404 unless session
 
-		session.disconnect
 		session.destroy
 
 		"ok"

data/lib/taps/utils.rb

@@ -8,7 +8,9 @@ module Utils
 	extend self
 
 	def windows?
-		RUBY_PLATFORM =~ /mswin32|mingw32/
+		return @windows if defined?(@windows)
+		require 'rbconfig'
+		@windows = !!(::Config::CONFIG['host_os'] =~ /mswin|mingw/)
 	end
 
 	def bin(cmd)
@@ -24,24 +26,18 @@ module Utils
 		Zlib.crc32(data) == crc32.to_i
 	end
 
-	def gzip(data)
-		io = StringIO.new
-		gz = Zlib::GzipWriter.new(io)
-		gz.write data
-		gz.close
-		io.string
+	def base64encode(data)
+		[data].pack("m")
 	end
 
-	def gunzip(gzip_data)
-		io = StringIO.new(gzip_data)
-		gz = Zlib::GzipReader.new(io)
-		data = gz.read
-		gz.close
-		data
+	def base64decode(data)
+		data.unpack("m").first
 	end
 
-	def format_data(data, string_columns)
+	def format_data(data, opts={})
 		return {} if data.size == 0
+		string_columns = opts[:string_columns] || []
+
 		header = data[0].keys
 		only_data = data.collect do |row|
 			row = blobs_to_string(row, string_columns)
@@ -54,7 +50,7 @@ module Utils
 	# this is not true for other databases so we must check if the field is
 	# actually text and manually convert it back to a string
 	def incorrect_blobs(db, table)
-		return [] unless db.class.to_s == "Sequel::MySQL::Database"
+		return [] if (db.url =~ /mysql:\/\//).nil?
 
 		columns = []
 		db.schema(table).each do |data|
@@ -81,7 +77,7 @@ module Utils
 			t1 = Time.now
 			time_in_db = yield chunksize
 			time_in_db = time_in_db.to_f rescue 0
-		rescue Errno::EPIPE, RestClient::RequestFailed
+		rescue Errno::EPIPE, RestClient::RequestFailed, RestClient::RequestTimeout
 			retries += 1
 			raise if retries > 2
 
@@ -126,21 +122,26 @@ module Utils
 	end
 
 	def schema_bin(*args)
-		`#{File.dirname(__FILE__)}/../../bin/#{bin('schema')} #{args.join(' ')}`
+		`#{File.dirname(__FILE__)}/../../bin/#{bin('schema')} #{args.map { |a| "'#{a}'" }.join(' ')}`
 	end
 
 	def primary_key(db, table)
 		if db.respond_to?(:primary_key)
 			db.primary_key(table)
 		else
-			db.schema(table).select { |c| c[1][:primary_key] }.map { |c| c.first }.shift
+			db.schema(table).select { |c| c[1][:primary_key] }.map { |c| c.first.to_sym }
 		end
 	end
 
+	def single_integer_primary_key(db, table)
+		keys = db.schema(table).select { |c| c[1][:primary_key] and c[1][:type] == :integer }
+		not keys.nil? and keys.size == 1
+	end
+
 	def order_by(db, table)
 		pkey = primary_key(db, table)
 		if pkey
-			[pkey.to_sym]
+			pkey.kind_of?(Array) ? pkey : [pkey.to_sym]
 		else
 			db[table].columns
 		end

data/spec/base.rb

@@ -4,6 +4,8 @@ require 'mocha'
 require 'rack/test'
 require 'tempfile'
 
+$:.unshift File.dirname(__FILE__) + "/../lib"
+
 class Bacon::Context
 	include Mocha::Standalone
 	include Rack::Test::Methods
@@ -19,6 +21,6 @@ class Bacon::Context
 	end
 end
 
-require File.dirname(__FILE__) + '/../lib/taps/config'
+require 'taps/config'
 Taps::Config.taps_database_url = "sqlite://#{Tempfile.new('test.db').path}"
 Sequel.connect(Taps::Config.taps_database_url)