taps 0.2.26 → 0.3.0

data/README.rdoc

@@ -22,11 +22,15 @@ or when you want to push a local database to a taps server
 
   $ taps push postgres://dbuser:dbpassword@localhost/dbname http://httpuser:httppassword@example.com:5000
 
+or when you want to transfer only 1 table
+
+  $ taps push postgres://dbuser:dbpassword@localhost/dbname http://httpuser:httppassword@example.com:5000 --filter mytable
+
 == Known Issues
 
-* Blob data may not transfer properly, I suspect that SQLite3 is modifying some native ruby objects.
 * Foreign Keys get lost in the schema transfer
-* Large tables (>1 million rows with a large number of columns) get slower as the offset gets larger. This is due to it being inefficient having large offset values in queries.
+* Tables without primary keys will be incredibly slow to transfer. This is due to it being inefficient having large offset values in queries.
+* Multiple schemas are currently not supported
 
 == Meta
 
@@ -39,3 +43,5 @@ Early research and inspiration by Blake Mizerany
 Released under the MIT License: http://www.opensource.org/licenses/mit-license.php
 
 http://github.com/ricardochimal/taps
+
+Special Thanks to Sequel for making this tool possible http://sequel.rubyforge.org/

data/Rakefile

@@ -6,17 +6,18 @@ begin
     s.email = "ricardo@heroku.com"
     s.homepage = "http://github.com/ricardochimal/taps"
     s.description = "A simple database agnostic import/export app to transfer data to/from a remote database."
-    s.authors = ["Ricardo Chimal, Jr.", "Adam Wiggins"]
+    s.authors = ["Ricardo Chimal, Jr."]
 
-    s.add_dependency 'sinatra', '= 0.9.2'
-    s.add_dependency 'activerecord', '= 2.2.2'
-    s.add_dependency 'thor', '= 0.9.9'
-    s.add_dependency 'rest-client', '~> 1.3.0'
-    s.add_dependency 'sequel', '>= 3.0.0', '< 3.1.0'
+    s.rubygems_version = %q{1.3.5}
+
+    s.add_dependency 'json_pure', '~> 1.2.0'
+    s.add_dependency 'sinatra', '~> 1.0.0'
+    s.add_dependency 'rest-client', '~> 1.4.0'
+    s.add_dependency 'sequel', '~> 3.10.0'
     s.add_dependency 'sqlite3-ruby', '~> 1.2.0'
+    s.add_dependency 'rack', '>= 1.0.1'
 
     s.rubyforge_project = "taps"
-    s.rubygems_version = '1.3.1'
 
     s.files = FileList['spec/*.rb'] + FileList['lib/**/*.rb'] + ['README.rdoc', 'LICENSE', 'VERSION.yml', 'Rakefile'] + FileList['bin/*']
     s.executables = ['taps', 'schema']
@@ -29,13 +30,17 @@ rescue LoadError => e
   end
 end
 
-require 'rake/rdoctask'
-Rake::RDocTask.new do |rdoc|
-  rdoc.rdoc_dir = 'rdoc'
-  rdoc.title = 'taps'
-  rdoc.options << '--line-numbers' << '--inline-source'
-  rdoc.rdoc_files.include('README*')
-  rdoc.rdoc_files.include('lib/**/*.rb')
+begin
+  require 'rake/rdoctask'
+  Rake::RDocTask.new do |rdoc|
+    rdoc.rdoc_dir = 'rdoc'
+    rdoc.title = 'taps'
+    rdoc.options << '--line-numbers' << '--inline-source'
+    rdoc.rdoc_files.include('README*')
+    rdoc.rdoc_files.include('lib/**/*.rb')
+  end
+rescue LoadError
+   puts "Rdoc is not available"
 end
 
 begin
@@ -46,7 +51,7 @@ begin
     t.verbose = true
   end
 rescue LoadError
-  puts "RCov is not available. In order to run rcov, you must: sudo gem install spicycode-rcov"
+  puts "RCov is not available. In order to run rcov, you must: sudo gem install rcov"
 end
 
 desc "Run all specs; requires the bacon gem"

data/VERSION.yml

@@ -1,5 +1,5 @@
 --- 
-:major: 0
-:minor: 2
 :build: 
-:patch: 26
+:minor: 3
+:patch: 0
+:major: 0

data/bin/schema

@@ -1,9 +1,11 @@
 #!/usr/bin/env ruby
 
 require 'rubygems'
-gem 'activerecord', '= 2.2.2'
+gem 'sequel', '~> 3.10.0'
 
-require File.dirname(__FILE__) + '/../lib/taps/schema'
+$:.unshift File.dirname(__FILE__) + '/../lib'
+
+require 'taps/schema'
 
 cmd = ARGV.shift.strip rescue ''
 database_url = ARGV.shift.strip rescue ''
@@ -12,7 +14,9 @@ def show_usage_and_exit
 	puts <<EOTXT
 	schema console <database_url>
 	schema dump <database_url>
+	schema dump_table <database_url> <table>
 	schema indexes <database_url>
+	schema indexes_individual <database_url>
 	schema reset_db_sequences <database_url>
 	schema load <database_url> <schema_file>
 	schema load_indexes <database_url> <indexes_file>
@@ -22,9 +26,14 @@ end
 
 case cmd
 when 'dump'
-	puts Taps::Schema.dump_without_indexes(database_url)
+	puts Taps::Schema.dump(database_url)
+when 'dump_table'
+	table = ARGV.shift.strip
+	puts Taps::Schema.dump_table(database_url, table)
 when 'indexes'
 	puts Taps::Schema.indexes(database_url)
+when 'indexes_individual'
+	puts Taps::Schema.indexes_individual(database_url)
 when 'load_indexes'
 	filename = ARGV.shift.strip rescue ''
 	indexes = File.read(filename) rescue show_usage_and_exit
@@ -36,8 +45,7 @@ when 'load'
 when 'reset_db_sequences'
 	Taps::Schema.reset_db_sequences(database_url)
 when 'console'
-	Taps::Schema.connection(database_url)
-	$db = ActiveRecord::Base.connection
+	$db = Sequel.connect(database_url)
 	require 'irb'
 	require 'irb/completion'
 	IRB.start

data/bin/taps

@@ -1,13 +1,6 @@
 #!/usr/bin/env ruby
 
-require 'rubygems'
-gem 'activerecord', '= 2.2.2'
-gem 'thor', '= 0.9.9'
-gem 'rest-client', '~> 1.3.0'
-gem 'sinatra', '= 0.9.2'
-gem 'sequel', '>= 3.0.0', '< 3.1.0'
-gem 'sqlite3-ruby', '~> 1.2.0'
+$:.unshift File.dirname(__FILE__) + '/../lib'
+require 'taps/cli'
 
-require File.dirname(__FILE__) + '/../lib/taps/cli'
-
-Taps::Cli.start
+Taps::Cli.new(ARGV.dup).run

data/lib/taps/cli.rb

@@ -1,63 +1,183 @@
-require 'thor'
+require 'optparse'
 require 'tempfile'
-require File.dirname(__FILE__) + '/config'
+require 'json'
+require 'taps/monkey'
+require 'taps/config'
+require 'taps/log'
 
 Taps::Config.taps_database_url = ENV['TAPS_DATABASE_URL'] || "sqlite://#{Tempfile.new('taps.db').path}"
 
 module Taps
-class Cli < Thor
-	desc "server <local_database_url> <login> <password>", "Start a taps database import/export server"
-	method_options(:port => :numeric)
-	def server(database_url, login, password)
-		Taps::Config.database_url = database_url
-		Taps::Config.login = login
-		Taps::Config.password = password
+class Cli
+	attr_accessor :argv
 
-		port = options[:port] || 5000
+	def initialize(argv)
+		@argv = argv
+	end
 
-		Taps::Config.verify_database_url
+	def run
+		method = (argv.shift || 'help').to_sym
+		if [:pull, :push, :server, :version].include? method
+			send(method)
+		else
+			help
+		end
+	end
 
-		require File.dirname(__FILE__) + '/server'
-		Taps::Server.run!({
-			:port => port,
-			:environment => :production,
-			:logging => true
-		})
+	def pull
+		opts = clientoptparse(:pull)
+		Taps.log.level = Logger::DEBUG if opts[:debug]
+		if opts[:resume_filename]
+			clientresumexfer(:pull, opts)
+		else
+			clientxfer(:pull, opts)
+		end
 	end
 
-	desc "pull <local_database_url> <remote_url>", "Pull a database from a taps server"
-	method_options(:chunksize => :numeric)
-	def pull(database_url, remote_url)
-		clientxfer(:cmd_receive, database_url, remote_url)
+	def push
+		opts = clientoptparse(:push)
+		Taps.log.level = Logger::DEBUG if opts[:debug]
+		if opts[:resume_filename]
+			clientresumexfer(:push, opts)
+		else
+			clientxfer(:push, opts)
+		end
 	end
 
-	desc "push <local_database_url> <remote_url>", "Push a database to a taps server"
-	method_options(:chunksize => :numeric)
-	def push(database_url, remote_url)
-		clientxfer(:cmd_send, database_url, remote_url)
+	def server
+		opts = serveroptparse
+		Taps.log.level = Logger::DEBUG if opts[:debug]
+		Taps::Config.database_url = opts[:database_url]
+		Taps::Config.login = opts[:login]
+		Taps::Config.password = opts[:password]
+
+		Taps::Config.verify_database_url
+		require 'taps/server'
+		Taps::Server.run!({
+			:port => opts[:port],
+			:environment => :production,
+			:logging => true,
+			:dump_errors => true,
+		})
 	end
 
-	desc "version", "Taps version"
 	def version
 		puts Taps.version
 	end
 
-	def clientxfer(method, database_url, remote_url)
-		if options[:chunksize]
-			Taps::Config.chunksize = options[:chunksize] < 100 ? 100 : options[:chunksize]
-		else
-			Taps::Config.chunksize = 1000
+	def help
+		puts <<EOHELP
+Options
+=======
+server    Start a taps database import/export server
+pull      Pull a database from a taps server
+push      Push a database to a taps server
+version   Taps version
+
+Add '-h' to any command to see their usage
+EOHELP
+	end
+
+	def serveroptparse
+		opts={:port => 5000, :database_url => nil, :login => nil, :password => nil, :debug => false}
+		OptionParser.new do |o|
+			o.banner = "Usage: #{File.basename($0)} server [OPTIONS] <local_database_url> <login> <password>"
+			o.define_head "Start a taps database import/export server"
+
+			o.on("-p", "--port=N", "Server Port") { |v| opts[:port] = v.to_i if v.to_i > 0 }
+			o.on("-d", "--debug", "Enable Debug Messages") { |v| opts[:debug] = true }
+			o.parse!(argv)
+
+			opts[:database_url] = argv.shift
+			opts[:login] = argv.shift
+			opts[:password] = argv.shift
+
+			if opts[:database_url].nil?
+				$stderr.puts "Missing Database URL"
+				puts o
+				exit 1
+			end
+			if opts[:login].nil?
+				$stderr.puts "Missing Login"
+				puts o
+				exit 1
+			end
+			if opts[:password].nil?
+				$stderr.puts "Missing Password"
+				puts o
+				exit 1
+			end
 		end
-		Taps::Config.database_url = database_url
-		Taps::Config.remote_url = remote_url
+		opts
+	end
 
-		Taps::Config.verify_database_url
+	def clientoptparse(cmd)
+		opts={:default_chunksize => 1000, :database_url => nil, :remote_url => nil, :debug => false, :resume_filename => nil, :disable_compresion => false}
+		OptionParser.new do |o|
+			o.banner = "Usage: #{File.basename($0)} #{cmd} [OPTIONS] <local_database_url> <remote_url>"
+
+			case cmd
+			when :pull
+				o.define_head "Pull a database from a taps server"
+			when :push
+				o.define_head "Push a database to a taps server"
+			end
+
+			o.on("-r", "--resume=file", "Resume a Taps Session from a stored file") { |v| opts[:resume_filename] = v }
+			o.on("-c", "--chunksize=N", "Initial Chunksize") { |v| opts[:default_chunksize] = (v.to_i < 10 ? 10 : v.to_i) }
+			o.on("-g", "--disable-compression", "Disable Compression") { |v| opts[:disable_compression] = true }
+			o.on("-f", "--filter=regex", "Regex Filter for tables") { |v| opts[:table_filter] = v }
+			o.on("-d", "--debug", "Enable Debug Messages") { |v| opts[:debug] = true }
+			o.parse!(argv)
 
-		require File.dirname(__FILE__) + '/client_session'
+			opts[:database_url] = argv.shift
+			opts[:remote_url] = argv.shift
 
-		Taps::ClientSession.quickstart do |session|
-			session.send(method)
+			if opts[:database_url].nil?
+				$stderr.puts "Missing Database URL"
+				puts o
+				exit 1
+			end
+			if opts[:remote_url].nil?
+				$stderr.puts "Missing Remote Taps URL"
+				puts o
+				exit 1
+			end
 		end
+
+		opts
 	end
+
+	def clientxfer(method, opts)
+		database_url = opts.delete(:database_url)
+		remote_url = opts.delete(:remote_url)
+
+		Taps::Config.verify_database_url(database_url)
+
+		require 'taps/operation'
+
+		Taps::Operation.factory(method, database_url, remote_url, opts).run
+	end
+
+	def clientresumexfer(method, opts)
+		session = JSON.parse(File.read(opts.delete(:resume_filename)))
+		session.symbolize_recursively!
+
+		database_url = opts.delete(:database_url)
+		remote_url = opts.delete(:remote_url) || session.delete(:remote_url)
+
+		Taps::Config.verify_database_url(database_url)
+
+		require 'taps/operation'
+
+		newsession = session.merge({
+			:default_chunksize => opts[:default_chunksize],
+			:disable_compression => opts[:disable_compression],
+			:resume => true,
+		})
+
+		Taps::Operation.factory(method, database_url, remote_url, newsession).run
+	end
+
 end
 end

data/lib/taps/config.rb

@@ -8,21 +8,32 @@ module Taps
 	end
 
 	def self.version
-		"#{version_yml[:major]}.#{version_yml[:minor]}.#{version_yml[:patch]}"
+		version = "#{version_yml[:major]}.#{version_yml[:minor]}.#{version_yml[:patch]}"
+		version += ".#{version_yml[:build]}" if version_yml[:build]
+		version
 	end
 
 	def self.compatible_version
 		"#{version_yml[:major]}.#{version_yml[:minor]}"
 	end
 
+	def self.exiting=(val)
+		@@exiting = val
+	end
+
+	def exiting?
+		(@@exiting ||= false) == true
+	end
+
 	class Config
 		class << self
 			attr_accessor :taps_database_url
 			attr_accessor :login, :password, :database_url, :remote_url
 			attr_accessor :chunksize
 
-			def verify_database_url
-				db = Sequel.connect(self.database_url)
+			def verify_database_url(db_url=nil)
+				db_url ||= self.database_url
+				db = Sequel.connect(db_url)
 				db.tables
 				db.disconnect
 			rescue Object => e

data/lib/taps/data_stream.rb

@@ -0,0 +1,299 @@
+require 'taps/monkey'
+require 'taps/multipart'
+require 'taps/utils'
+require 'taps/log'
+require 'json'
+
+module Taps
+
+class DataStream
+	class CorruptedData < Exception; end
+
+	attr_reader :db, :state
+
+	def initialize(db, state)
+		@db = db
+		@state = {
+			:offset => 0,
+			:avg_chunksize => 0,
+			:num_chunksize => 0,
+			:total_chunksize => 0,
+		}.merge(state)
+		@complete = false
+	end
+
+	def log
+		Taps.log
+	end
+
+	def error=(val)
+		state[:error] = val
+	end
+
+	def error
+		state[:error] || false
+	end
+
+	def table_name
+		state[:table_name].to_sym
+	end
+
+	def to_hash
+		state.merge(:klass => self.class.to_s)
+	end
+
+	def to_json
+		to_hash.to_json
+	end
+
+	def string_columns
+		@string_columns ||= Taps::Utils.incorrect_blobs(db, table_name)
+	end
+
+	def table
+		@table ||= db[table_name]
+	end
+
+	def order_by(name=nil)
+		@order_by ||= begin
+			name ||= table_name
+			Taps::Utils.order_by(db, name)
+		end
+	end
+
+	def increment(row_count)
+		state[:offset] += row_count
+	end
+
+	# keep a record of the average chunksize within the first few hundred thousand records, after chunksize
+	# goes below 100 or maybe if offset is > 1000
+	def fetch_rows
+		state[:chunksize] = fetch_chunksize
+		ds = table.order(*order_by).limit(state[:chunksize], state[:offset])
+		log.debug "DataStream#fetch_rows SQL -> #{ds.sql}"
+		rows = Taps::Utils.format_data(ds.all,
+			:string_columns => string_columns)
+		update_chunksize_stats
+		rows
+	end
+
+	def max_chunksize_training
+		20
+	end
+
+	def fetch_chunksize
+		chunksize = state[:chunksize]
+		return chunksize if state[:num_chunksize] < max_chunksize_training
+		return chunksize if state[:avg_chunksize] == 0
+		return chunksize if state[:error]
+		state[:avg_chunksize] > chunksize ? state[:avg_chunksize] : chunksize
+	end
+
+	def update_chunksize_stats
+		return if state[:num_chunksize] >= max_chunksize_training
+		state[:total_chunksize] += state[:chunksize]
+		state[:num_chunksize] += 1
+		state[:avg_chunksize] = state[:total_chunksize] / state[:num_chunksize] rescue state[:chunksize]
+	end
+
+	def encode_rows(rows)
+		Taps::Utils.base64encode(Marshal.dump(rows))
+	end
+
+	def fetch
+		log.debug "DataStream#fetch state -> #{state.inspect}"
+
+		t1 = Time.now
+		rows = fetch_rows
+		encoded_data = encode_rows(rows)
+		t2 = Time.now
+		elapsed_time = t2 - t1
+
+		@complete = rows == { }
+
+		[encoded_data, (@complete ? 0 : rows[:data].size), elapsed_time]
+	end
+
+	def complete?
+		@complete
+	end
+
+	def fetch_remote(resource, headers)
+		params = fetch_from_resource(resource, headers)
+		encoded_data = params[:encoded_data]
+		json = params[:json]
+
+		rows = parse_encoded_data(encoded_data, json[:checksum])
+		@complete = rows == { }
+
+		# update local state
+		state.merge!(json[:state].merge(:chunksize => state[:chunksize]))
+
+		unless @complete
+			import_rows(rows)
+			rows[:data].size
+		else
+			0
+		end
+	end
+
+	# this one is used inside the server process
+	def fetch_remote_in_server(params)
+		json = self.class.parse_json(params[:json])
+		encoded_data = params[:encoded_data]
+
+		rows = parse_encoded_data(encoded_data, json[:checksum])
+		@complete = rows == { }
+
+		unless @complete
+			import_rows(rows)
+			rows[:data].size
+		else
+			0
+		end
+	end
+
+	def fetch_from_resource(resource, headers)
+		res = nil
+		log.debug "DataStream#fetch_from_resource state -> #{state.inspect}"
+		state[:chunksize] = Taps::Utils.calculate_chunksize(state[:chunksize]) do |c|
+			state[:chunksize] = c
+			res = resource.post({:state => self.to_json}, headers)
+		end
+
+		begin
+			params = Taps::Multipart.parse(res)
+			params[:json] = self.class.parse_json(params[:json]) if params.has_key?(:json)
+			return params
+		rescue JSON::Parser
+			raise DataStream::CorruptedData.new("Invalid JSON Received")
+		end
+	end
+
+	def self.parse_json(json)
+		hash = JSON.parse(json).symbolize_keys
+		hash[:state].symbolize_keys! if hash.has_key?(:state)
+		hash
+	end
+
+	def parse_encoded_data(encoded_data, checksum)
+		raise DataStream::CorruptedData.new("Checksum Failed") unless Taps::Utils.valid_data?(encoded_data, checksum)
+
+		begin
+			return Marshal.load(Taps::Utils.base64decode(encoded_data))
+		rescue Object => e
+			unless ENV['NO_DUMP_MARSHAL_ERRORS']
+				puts "Error encountered loading data, wrote the data chunk to dump.#{Process.pid}.dat"
+				File.open("dump.#{Process.pid}.dat", "w") { |f| f.write(encoded_data) }
+			end
+			raise
+		end
+	end
+
+	def import_rows(rows)
+		table.import(rows[:header], rows[:data])
+		state[:offset] += rows[:data].size
+	end
+
+	def self.factory(db, state)
+		if defined?(Sequel::MySQL) && Sequel::MySQL.respond_to?(:convert_invalid_date_time=)
+			Sequel::MySQL.convert_invalid_date_time = :nil
+		end
+
+		if state.has_key?(:klass)
+			return eval(state[:klass]).new(db, state)
+		end
+
+		if Taps::Utils.single_integer_primary_key(db, state[:table_name].to_sym)
+			DataStreamKeyed.new(db, state)
+		else
+			DataStream.new(db, state)
+		end
+	end
+end
+
+
+class DataStreamKeyed < DataStream
+	attr_accessor :buffer
+
+	def initialize(db, state)
+		super(db, state)
+		@state = { :primary_key => order_by(state[:table_name]).first, :filter => 0 }.merge(state)
+		@buffer = []
+	end
+
+	def primary_key
+		state[:primary_key].to_sym
+	end
+
+	def buffer_limit
+		if state[:last_fetched] and state[:last_fetched] < state[:filter] and self.buffer.size == 0
+			state[:last_fetched]
+		else
+			state[:filter]
+		end
+	end
+
+	def calc_limit(chunksize)
+		# we want to not fetch more than is needed while we're
+		# inside sinatra but locally we can select more than
+		# is strictly needed
+		if defined?(Sinatra)
+			(chunksize * 1.1).ceil
+		else
+			(chunksize * 3).ceil
+		end
+	end
+
+	def load_buffer(chunksize)
+		# make sure BasicObject is not polluted by subsequent requires
+		Sequel::BasicObject.remove_methods!
+
+		num = 0
+		loop do
+			limit = calc_limit(chunksize)
+			# we have to use local variables in order for the virtual row filter to work correctly
+			key = primary_key
+			buf_limit = buffer_limit
+			ds = table.order(*order_by).filter { key > buf_limit }.limit(limit)
+			log.debug "DataStreamKeyed#load_buffer SQL -> #{ds.sql}"
+			data = ds.all
+			self.buffer += data
+			num += data.size
+			if data.size > 0
+				# keep a record of the last primary key value in the buffer
+				state[:filter] = self.buffer.last[ primary_key ]
+			end
+
+			break if num >= chunksize or data.size == 0
+		end
+	end
+
+	def fetch_buffered(chunksize)
+		load_buffer(chunksize) if self.buffer.size < chunksize
+		rows = buffer.slice(0, chunksize)
+		state[:last_fetched] = if rows.size > 0
+			rows.last[ primary_key ]
+		else
+			nil
+		end
+		rows
+	end
+
+	def import_rows(rows)
+		table.import(rows[:header], rows[:data])
+	end
+
+	def fetch_rows
+		chunksize = state[:chunksize]
+		Taps::Utils.format_data(fetch_buffered(chunksize) || [],
+			:string_columns => string_columns)
+	end
+
+	def increment(row_count)
+		# pop the rows we just successfully sent off the buffer
+		@buffer.slice!(0, row_count)
+	end
+end
+
+end