taps 0.2.1

data/LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 2008 Ricardo Chimal, Jr
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.rdoc

@@ -0,0 +1,35 @@
+= Taps -- simple database import/export app
+
+A simple database agnostic import/export app to transfer data to/from a remote database.
+
+== Usage: Server
+
+  $ taps server postgres://localdbuser:localdbpass@localhost/dbname httpuser httppassword
+
+== Usage: Client
+
+When you want to pull down a database from a taps server
+
+  $ taps pull postgres://dbuser:dbpassword@localhost/dbname http://httpuser:httppassword@example.com:5000
+
+or when you want to push a local database to a taps server
+
+  $ taps push postgres://dbuser:dbpassword@localhost/dbname http://httpuser:httppassword@example.com:5000
+
+== Known Issues
+
+* Blob Data does not transfer properly.
+* Foreign Keys get lost in the schema transfer
+* Large tables (>1 million rows with a large number of columns) get slower as the offset gets larger. This is due to it being inefficient having large offset values in queries.
+
+== Meta
+
+Maintained by Ricardo Chimal, Jr. (ricardo at heroku dot com)
+
+Written by Ricardo Chimal, Jr. (ricardo at heroku dot com) and Adam Wiggins (adam at heroku dot com)
+
+Early research and inspiration by Blake Mizerany
+
+Released under the MIT License: http://www.opensource.org/licenses/mit-license.php
+
+http://github.com/ricardochimal/taps

data/Rakefile

@@ -0,0 +1,53 @@
+begin
+  require 'jeweler'
+  Jeweler::Tasks.new do |s|
+    s.name = "taps"
+    s.summary = %Q{simple database import/export app}
+    s.email = "ricardo@heroku.com"
+    s.homepage = "http://github.com/ricardochimal/taps"
+    s.description = "A simple database agnostic import/export app to transfer data to/from a remote database."
+    s.authors = ["Ricardo Chimal, Jr.", "Adam Wiggins"]
+
+    s.add_dependency 'sinatra', '~> 0.9.0'
+    s.add_dependency 'activerecord', '= 2.2.2'
+    s.add_dependency 'thor', '= 0.9.9'
+    s.add_dependency 'rest-client', '~> 0.9.0'
+    s.add_dependency 'sequel', '~> 2.10.0'
+    s.add_dependency 'sqlite3-ruby', '~> 1.2.0'
+
+    s.rubyforge_project = "taps"
+    s.rubygems_version = '1.3.1'
+
+    s.files = FileList['spec/*.rb'] + FileList['lib/**/*.rb'] + ['README.rdoc', 'LICENSE', 'VERSION.yml', 'Rakefile']
+    s.executables = ['taps', 'schema']
+  end
+rescue LoadError
+  puts "Jeweler not available. Install it with: sudo gem install technicalpickles-jeweler -s http://gems.github.com"
+end
+
+require 'rake/rdoctask'
+Rake::RDocTask.new do |rdoc|
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title = 'taps'
+  rdoc.options << '--line-numbers' << '--inline-source'
+  rdoc.rdoc_files.include('README*')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+end
+
+begin
+  require 'rcov/rcovtask'
+  Rcov::RcovTask.new do |t|
+    t.libs << 'spec'
+    t.test_files = FileList['spec/*_spec.rb']
+    t.verbose = true
+  end
+rescue LoadError
+  puts "RCov is not available. In order to run rcov, you must: sudo gem install spicycode-rcov"
+end
+
+desc "Run all specs"
+task :spec do
+  system "bacon #{File.dirname(__FILE__)}/spec/*_spec.rb"
+end
+
+task :default => :spec

data/VERSION.yml

@@ -0,0 +1,4 @@
+--- 
+:major: 0
+:minor: 2
+:patch: 1

data/bin/schema

@@ -0,0 +1,38 @@
+#!/usr/bin/env ruby
+
+require 'rubygems'
+gem 'activerecord', '= 2.2.2'
+
+require File.dirname(__FILE__) + '/../lib/taps/schema'
+
+cmd = ARGV.shift.strip rescue ''
+database_url = ARGV.shift.strip rescue ''
+
+def show_usage_and_exit
+	puts <<EOTXT
+	schema dump <database_url>
+	schema indexes <database_url>
+	schema reset_db_sequences <database_url>
+	schema load <database_url> <schema_file>
+	schema load_indexes <database_url> <indexes_file>
+EOTXT
+	exit(1)
+end
+
+if cmd == 'dump'
+	puts Taps::Schema.dump_without_indexes(database_url)
+elsif cmd == 'indexes'
+	puts Taps::Schema.indexes(database_url)
+elsif cmd == 'load_indexes'
+	filename = ARGV.shift.strip rescue ''
+	indexes = File.read(filename) rescue show_usage_and_exit
+	Taps::Schema.load_indexes(database_url, indexes)
+elsif cmd == 'load'
+	filename = ARGV.shift.strip rescue ''
+	schema = File.read(filename) rescue show_usage_and_exit
+	Taps::Schema.load(database_url, schema)
+elsif cmd == 'reset_db_sequences'
+	Taps::Schema.reset_db_sequences(database_url)
+else
+	show_usage_and_exit
+end

data/bin/taps

@@ -0,0 +1,13 @@
+#!/usr/bin/env ruby
+
+require 'rubygems'
+gem 'activerecord', '= 2.2.2'
+gem 'thor', '= 0.9.9'
+gem 'rest-client', '~> 0.9.0'
+gem 'sinatra', '~> 0.9.0'
+gem 'sequel', '~> 2.10.0'
+gem 'sqlite3-ruby', '~> 1.2.0'
+
+require File.dirname(__FILE__) + '/../lib/taps/cli'
+
+Taps::Cli.start

data/lib/taps/cli.rb

@@ -0,0 +1,57 @@
+require 'thor'
+require File.dirname(__FILE__) + '/config'
+
+Taps::Config.taps_database_url = 'sqlite://taps.db'
+
+module Taps
+class Cli < Thor
+	desc "server <local_database_url> <login> <password>", "Start a taps database import/export server"
+	method_options(:port => :numeric)
+	def server(database_url, login, password)
+		Taps::Config.database_url = database_url
+		Taps::Config.login = login
+		Taps::Config.password = password
+
+		port = options[:port] || 5000
+
+		Taps::Config.verify_database_url
+
+		require File.dirname(__FILE__) + '/server'
+		Taps::Server.run!({
+			:port => port,
+			:environment => :production,
+			:logging => true
+		})
+	end
+
+	desc "pull <local_database_url> <remote_url>", "Pull a database from a taps server"
+	method_options(:chunksize => :numeric)
+	def pull(database_url, remote_url)
+		clientxfer(:cmd_receive, database_url, remote_url)
+	end
+
+	desc "push <local_database_url> <remote_url>", "Push a database to a taps server"
+	method_options(:chunksize => :numeric)
+	def push(database_url, remote_url)
+		clientxfer(:cmd_send, database_url, remote_url)
+	end
+
+	def clientxfer(method, database_url, remote_url)
+		if options[:chunksize]
+			Taps::Config.chunksize = options[:chunksize] < 100 ? 100 : options[:chunksize]
+		else
+			Taps::Config.chunksize = 1000
+		end
+		Taps::Config.database_url = database_url
+		Taps::Config.remote_url = remote_url
+
+		Taps::Config.verify_database_url
+
+		require File.dirname(__FILE__) + '/client_session'
+
+		Taps::ClientSession.quickstart do |session|
+			session.send(method)
+		end
+	end
+end
+end

data/lib/taps/client_session.rb

@@ -0,0 +1,237 @@
+require 'rest_client'
+require 'sequel'
+require 'zlib'
+
+require File.dirname(__FILE__) + '/progress_bar'
+require File.dirname(__FILE__) + '/config'
+require File.dirname(__FILE__) + '/utils'
+
+module Taps
+class ClientSession
+	attr_reader :database_url, :remote_url, :default_chunksize
+
+	def initialize(database_url, remote_url, default_chunksize)
+		@database_url = database_url
+		@remote_url = remote_url
+		@default_chunksize = default_chunksize
+	end
+
+	def self.start(database_url, remote_url, default_chunksize, &block)
+		s = new(database_url, remote_url, default_chunksize)
+		yield s
+		s.close_session
+	end
+
+	def self.quickstart(&block)
+		start(Taps::Config.database_url, Taps::Config.remote_url, Taps::Config.chunksize) do |s|
+			yield s
+		end
+	end
+
+	def db
+		@db ||= Sequel.connect(database_url)
+	end
+
+	def server
+		@server ||= RestClient::Resource.new(remote_url)
+	end
+
+	def session_resource
+		@session_resource ||= open_session
+	end
+
+	def open_session
+		uri = server['sessions'].post('', :taps_version => Taps::VERSION)
+		server[uri]
+	end
+
+	def close_session
+		@session_resource.delete(:taps_version => Taps::VERSION) if @session_resource
+	end
+
+	def cmd_send
+		verify_server
+		cmd_send_schema
+		cmd_send_data
+		cmd_send_indexes
+		cmd_send_reset_sequences
+	end
+
+	def cmd_send_indexes
+		puts "Sending schema indexes to remote taps server #{remote_url} from local database #{database_url}"
+
+		index_data = `#{File.dirname(__FILE__)}/../../bin/schema indexes #{database_url}`
+		session_resource['indexes'].post(index_data, :taps_version => Taps::VERSION)
+	end
+
+	def cmd_send_schema
+		puts "Sending schema to remote taps server #{remote_url} from local database #{database_url}"
+
+		schema_data = `#{File.dirname(__FILE__)}/../../bin/schema dump #{database_url}`
+		session_resource['schema'].post(schema_data, :taps_version => Taps::VERSION)
+	end
+
+	def cmd_send_reset_sequences
+		puts "Resetting db sequences in remote taps server at #{remote_url}"
+
+		session_resource["reset_sequences"].post('', :taps_version => Taps::VERSION)
+	end
+
+	def cmd_send_data
+		puts "Sending schema and data from local database #{database_url} to remote taps server at #{remote_url}"
+
+		db.tables.each do |table_name|
+			table = db[table_name]
+			count = table.count
+			columns = table.columns
+			order = columns.include?(:id) ? :id : columns.first
+			chunksize = self.default_chunksize
+
+			progress = ProgressBar.new(table_name.to_s, count)
+
+			offset = 0
+			loop do
+				rows = Taps::Utils.format_data(table.order(order).limit(chunksize, offset).all)
+				break if rows == { }
+
+				gzip_data = Taps::Utils.gzip(Marshal.dump(rows))
+
+				chunksize = Taps::Utils.calculate_chunksize(chunksize) do
+					begin
+						session_resource["tables/#{table_name}"].post(gzip_data,
+							:taps_version => Taps::VERSION,
+							:content_type => 'application/octet-stream',
+							:taps_checksum => Taps::Utils.checksum(gzip_data).to_s)
+					rescue RestClient::RequestFailed => e
+						# retry the same data, it got corrupted somehow.
+						if e.http_code == 412
+							next
+						end
+						raise
+					end
+				end
+
+				progress.inc(rows[:data].size)
+				offset += rows[:data].size
+			end
+
+			progress.finish
+		end
+	end
+
+	def cmd_receive
+		verify_server
+		cmd_receive_schema
+		cmd_receive_data
+		cmd_receive_indexes
+		cmd_reset_sequences
+	end
+
+	def cmd_receive_data
+		puts "Receiving data from remote taps server #{remote_url} into local database #{database_url}"
+
+		tables_with_counts, record_count = fetch_tables_info
+
+		puts "#{tables_with_counts.size} tables, #{format_number(record_count)} records"
+
+		tables_with_counts.each do |table_name, count|
+			table = db[table_name.to_sym]
+			chunksize = default_chunksize
+
+			progress = ProgressBar.new(table_name.to_s, count)
+
+			offset = 0
+			loop do
+				begin
+					chunksize, rows = fetch_table_rows(table_name, chunksize, offset)
+				rescue CorruptedData
+					next
+				end
+				break if rows == { }
+
+				table.multi_insert(rows[:header], rows[:data])
+
+				progress.inc(rows[:data].size)
+				offset += rows[:data].size
+			end
+
+			progress.finish
+		end
+	end
+
+	class CorruptedData < Exception; end
+
+	def fetch_table_rows(table_name, chunksize, offset)
+		response = nil
+		chunksize = Taps::Utils.calculate_chunksize(chunksize) do
+			response = session_resource["tables/#{table_name}/#{chunksize}?offset=#{offset}"].get(:taps_version => Taps::VERSION)
+		end
+		raise CorruptedData unless Taps::Utils.valid_data?(response.to_s, response.headers[:taps_checksum])
+
+		rows = Marshal.load(Taps::Utils.gunzip(response.to_s))
+		[chunksize, rows]
+	end
+
+	def fetch_tables_info
+		retries = 0
+		max_retries = 1
+		begin
+			tables_with_counts = Marshal.load(session_resource['tables'].get(:taps_version => Taps::VERSION))
+			record_count = tables_with_counts.values.inject(0) { |a,c| a += c }
+		rescue RestClient::Exception
+			retries += 1
+			retry if retries <= max_retries
+			puts "Unable to fetch tables information from #{remote_url}. Please check the server log."
+			exit(1)
+		end
+
+		[ tables_with_counts, record_count ]
+	end
+
+	def cmd_receive_schema
+		puts "Receiving schema from remote taps server #{remote_url} into local database #{database_url}"
+
+		schema_data = session_resource['schema'].get(:taps_version => Taps::VERSION)
+		output = Taps::Utils.load_schema(database_url, schema_data)
+		puts output if output
+	end
+
+	def cmd_receive_indexes
+		puts "Receiving schema indexes from remote taps server #{remote_url} into local database #{database_url}"
+
+		index_data = session_resource['indexes'].get(:taps_version => Taps::VERSION)
+
+		puts Taps::Utils.load_indexes(database_url, index_data)
+	end
+
+	def cmd_reset_sequences
+		puts "Resetting db sequences in #{database_url}"
+
+		puts `#{File.dirname(__FILE__)}/../../bin/schema reset_db_sequences #{database_url}`
+	end
+
+	def format_number(num)
+		num.to_s.gsub(/(\d)(?=(\d\d\d)+(?!\d))/, "\\1,")
+	end
+
+	def verify_server
+		begin
+			server['/'].get(:taps_version => Taps::VERSION)
+		rescue RestClient::RequestFailed => e
+			if e.http_code == 417
+				puts "#{remote_url} is running a different version of taps."
+				puts "#{e.response.body}"
+				exit(1)
+			else
+				raise
+			end
+		rescue RestClient::Unauthorized
+			puts "Bad credentials given for #{remote_url}"
+			exit(1)
+		rescue Errno::ECONNREFUSED
+			puts "Can't connect to #{remote_url}. Please check that it's running"
+			exit(1)
+		end
+	end
+end
+end