matthewtodd-taps 0.2.19

data/LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 2008 Ricardo Chimal, Jr
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.rdoc

@@ -0,0 +1,41 @@
+= Taps -- simple database import/export app
+
+A simple database agnostic import/export app to transfer data to/from a remote database.
+
+== Usage: Server
+
+Here's how you start a taps server
+
+  $ taps server postgres://localdbuser:localdbpass@localhost/dbname httpuser httppassword
+
+You can also specify an encoding in the database url
+
+  $ taps server mysql://localdbuser:localdbpass@localhost/dbname?encoding=latin1 httpuser httppassword
+
+== Usage: Client
+
+When you want to pull down a database from a taps server
+
+  $ taps pull postgres://dbuser:dbpassword@localhost/dbname http://httpuser:httppassword@example.com:5000
+
+or when you want to push a local database to a taps server
+
+  $ taps push postgres://dbuser:dbpassword@localhost/dbname http://httpuser:httppassword@example.com:5000
+
+== Known Issues
+
+* Blob data may not transfer properly, I suspect that SQLite3 is modifying some native ruby objects.
+* Foreign Keys get lost in the schema transfer
+* Large tables (>1 million rows with a large number of columns) get slower as the offset gets larger. This is due to it being inefficient having large offset values in queries.
+
+== Meta
+
+Maintained by Ricardo Chimal, Jr. (ricardo at heroku dot com)
+
+Written by Ricardo Chimal, Jr. (ricardo at heroku dot com) and Adam Wiggins (adam at heroku dot com)
+
+Early research and inspiration by Blake Mizerany
+
+Released under the MIT License: http://www.opensource.org/licenses/mit-license.php
+
+http://github.com/ricardochimal/taps

data/Rakefile

@@ -0,0 +1,61 @@
+begin
+  require 'jeweler'
+  Jeweler::Tasks.new do |s|
+    s.name = "matthewtodd-taps"
+    s.summary = %Q{simple database import/export app}
+    s.email = "ricardo@heroku.com"
+    s.homepage = "http://github.com/matthewtodd/taps"
+    s.description = "A simple database agnostic import/export app to transfer data to/from a remote database."
+    s.authors = ["Ricardo Chimal, Jr.", "Adam Wiggins"]
+
+    s.add_dependency 'sinatra', '= 0.9.2'
+    s.add_dependency 'activerecord', '~> 2.3.4'
+    s.add_dependency 'thor', '= 0.9.9'
+    s.add_dependency 'rest-client', '>= 1.0.1', '< 1.1.0'
+    s.add_dependency 'sequel', '>= 3.0.0', '< 3.1.0'
+    s.add_dependency 'sqlite3-ruby', '~> 1.2.0'
+
+    s.rubyforge_project = "taps"
+    s.rubygems_version = '1.3.1'
+
+    s.files = FileList['spec/*.rb'] + FileList['lib/**/*.rb'] + ['README.rdoc', 'LICENSE', 'VERSION.yml', 'Rakefile'] + FileList['bin/*']
+    s.executables = ['taps', 'schema']
+  end
+rescue LoadError => e
+  if e.message =~ /jeweler/
+    puts "Jeweler not available. Install it with: sudo gem install technicalpickles-jeweler -s http://gems.github.com"
+  else
+    puts e.message + ' -- while loading jeweler.'
+  end
+end
+
+require 'rake/rdoctask'
+Rake::RDocTask.new do |rdoc|
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title = 'taps'
+  rdoc.options << '--line-numbers' << '--inline-source'
+  rdoc.rdoc_files.include('README*')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+end
+
+begin
+  require 'rcov/rcovtask'
+  Rcov::RcovTask.new do |t|
+    t.libs << 'spec'
+    t.test_files = FileList['spec/*_spec.rb']
+    t.verbose = true
+  end
+rescue LoadError
+  puts "RCov is not available. In order to run rcov, you must: sudo gem install spicycode-rcov"
+end
+
+desc "Run all specs; requires the bacon gem"
+task :spec do
+  if `which bacon`.empty?
+    puts "bacon is not available. In order to run the specs, you must: sudo gem install bacon."
+  else
+    system "bacon #{File.dirname(__FILE__)}/spec/*_spec.rb"
+  end
+end
+
+task :default => :spec

data/VERSION.yml

@@ -0,0 +1,4 @@
+--- 
+:patch: 19
+:major: 0
+:minor: 2

data/bin/schema

@@ -0,0 +1,42 @@
+#!/usr/bin/env ruby
+
+require 'taps/schema'
+
+cmd = ARGV.shift.strip rescue ''
+database_url = ARGV.shift.strip rescue ''
+
+def show_usage_and_exit
+	puts <<EOTXT
+	schema console <database_url>
+	schema dump <database_url>
+	schema indexes <database_url>
+	schema reset_db_sequences <database_url>
+	schema load <database_url> <schema_file>
+	schema load_indexes <database_url> <indexes_file>
+EOTXT
+	exit(1)
+end
+
+if cmd == 'dump'
+	puts Taps::Schema.dump_without_indexes(database_url)
+elsif cmd == 'indexes'
+	puts Taps::Schema.indexes(database_url)
+elsif cmd == 'load_indexes'
+	filename = ARGV.shift.strip rescue ''
+	indexes = File.read(filename) rescue show_usage_and_exit
+	Taps::Schema.load_indexes(database_url, indexes)
+elsif cmd == 'load'
+	filename = ARGV.shift.strip rescue ''
+	schema = File.read(filename) rescue show_usage_and_exit
+	Taps::Schema.load(database_url, schema)
+elsif cmd == 'reset_db_sequences'
+	Taps::Schema.reset_db_sequences(database_url)
+elsif cmd == 'console'
+	Taps::Schema.connection(database_url)
+	$db = ActiveRecord::Base.connection
+	require 'irb'
+	require 'irb/completion'
+	IRB.start
+else
+	show_usage_and_exit
+end

data/bin/schema.cmd

@@ -0,0 +1,6 @@
+@ECHO OFF
+IF NOT "%~f0" == "~f0" GOTO :WinNT
+@"ruby.exe" "./schema" %1 %2 %3 %4 %5 %6 %7 %8 %9
+GOTO :EOF
+:WinNT
+@"ruby.exe" "%~dpn0" %*

data/bin/taps

@@ -0,0 +1,5 @@
+#!/usr/bin/env ruby
+
+require 'taps/cli'
+
+Taps::Cli.start

data/lib/taps/adapter_hacks.rb

@@ -0,0 +1,21 @@
+module Taps
+	module AdapterHacks
+		extend self
+
+		LIST = {
+			:all => ['non_rails_schema_dump'],
+			:mysql => ['invalid_text_limit', 'mysql_invalid_primary_key'],
+			:postgresql => ['invalid_text_limit', 'invalid_binary_limit']
+		}
+
+		def load(adapter)
+			LIST[:all].each do |r|
+				require "taps/adapter_hacks/#{r}"
+			end
+
+			(LIST[adapter.to_sym] || []).each do |r|
+				require "taps/adapter_hacks/#{r}"
+			end
+		end
+	end
+end

data/lib/taps/adapter_hacks/invalid_binary_limit.rb

@@ -0,0 +1,13 @@
+module ActiveRecord
+	module ConnectionAdapters
+		class TableDefinition
+			alias_method :original_binary, :binary
+			def binary(*args)
+				options = args.extract_options!
+				options.delete(:limit)
+				column_names = args
+				column_names.each { |name| column(name, 'binary', options) }
+			end
+		end
+	end
+end

data/lib/taps/adapter_hacks/invalid_text_limit.rb

@@ -0,0 +1,13 @@
+module ActiveRecord
+	module ConnectionAdapters
+		class TableDefinition
+			alias_method :original_text, :text
+			def text(*args)
+				options = args.extract_options!
+				options.delete(:limit)
+				column_names = args
+				column_names.each { |name| column(name, 'text', options) }
+			end
+		end
+	end
+end

data/lib/taps/adapter_hacks/mysql_invalid_primary_key.rb

@@ -0,0 +1,17 @@
+module ActiveRecord
+	module ConnectionAdapters
+		class MysqlAdapter < AbstractAdapter
+			alias_method :orig_pk_and_sequence_for, :pk_and_sequence_for
+			# mysql accepts varchar as a primary key but most others do not.
+			# only say that a field is a primary key if mysql says so
+			# and the field is a kind of integer
+			def pk_and_sequence_for(table)
+				keys = []
+				execute("describe #{quote_table_name(table)}").each_hash do |h|
+					keys << h["Field"] if h["Key"] == "PRI" and !(h["Type"] =~ /int/).nil?
+				end
+				keys.length == 1 ? [keys.first, nil] : nil
+			end
+		end
+	end
+end

data/lib/taps/adapter_hacks/non_rails_schema_dump.rb

@@ -0,0 +1,15 @@
+module ActiveRecord
+	class SchemaDumper
+		private
+
+		def header(stream)
+			stream.puts "ActiveRecord::Schema.define do"
+		end
+
+		def tables(stream)
+			@connection.tables.sort.each do |tbl|
+				table(tbl, stream)
+			end
+		end
+	end
+end

data/lib/taps/cli.rb

@@ -0,0 +1,62 @@
+require 'thor'
+require 'taps/config'
+
+Taps::Config.taps_database_url = ENV['TAPS_DATABASE_URL'] || 'sqlite://taps.db'
+
+module Taps
+class Cli < Thor
+	desc "server <local_database_url> <login> <password>", "Start a taps database import/export server"
+	method_options(:port => :numeric)
+	def server(database_url, login, password)
+		Taps::Config.database_url = database_url
+		Taps::Config.login = login
+		Taps::Config.password = password
+
+		port = options[:port] || 5000
+
+		Taps::Config.verify_database_url
+
+		require 'taps/server'
+		Taps::Server.run!({
+			:port => port,
+			:environment => :production,
+			:logging => true
+		})
+	end
+
+	desc "pull <local_database_url> <remote_url>", "Pull a database from a taps server"
+	method_options(:chunksize => :numeric)
+	def pull(database_url, remote_url)
+		clientxfer(:cmd_receive, database_url, remote_url)
+	end
+
+	desc "push <local_database_url> <remote_url>", "Push a database to a taps server"
+	method_options(:chunksize => :numeric)
+	def push(database_url, remote_url)
+		clientxfer(:cmd_send, database_url, remote_url)
+	end
+
+	desc "version", "Taps version"
+	def version
+		puts Taps.version
+	end
+
+	def clientxfer(method, database_url, remote_url)
+		if options[:chunksize]
+			Taps::Config.chunksize = options[:chunksize] < 100 ? 100 : options[:chunksize]
+		else
+			Taps::Config.chunksize = 1000
+		end
+		Taps::Config.database_url = database_url
+		Taps::Config.remote_url = remote_url
+
+		Taps::Config.verify_database_url
+
+		require 'taps/client_session'
+
+		Taps::ClientSession.quickstart do |session|
+			session.send(method)
+		end
+	end
+end
+end

data/lib/taps/client_session.rb

@@ -0,0 +1,306 @@
+require 'rest_client'
+require 'sequel'
+require 'zlib'
+
+require 'taps/progress_bar'
+require 'taps/config'
+require 'taps/utils'
+require 'taps/schema'
+
+module Taps
+class ClientSession
+	attr_reader :database_url, :remote_url, :default_chunksize
+
+	def initialize(database_url, remote_url, default_chunksize)
+		@database_url = database_url
+		@remote_url = remote_url
+		@default_chunksize = default_chunksize
+	end
+
+	def self.start(database_url, remote_url, default_chunksize, &block)
+		s = new(database_url, remote_url, default_chunksize)
+		yield s
+		s.close_session
+	end
+
+	def self.quickstart(&block)
+		start(Taps::Config.database_url, Taps::Config.remote_url, Taps::Config.chunksize) do |s|
+			yield s
+		end
+	end
+
+	def db
+		@db ||= Sequel.connect(database_url)
+	end
+
+	def server
+		@server ||= RestClient::Resource.new(remote_url)
+	end
+
+	def session_resource
+		@session_resource ||= open_session
+	end
+
+	def open_session
+		uri = server['sessions'].post('', http_headers)
+		server[uri]
+	end
+
+	def set_session(uri)
+		@session_resource = server[uri]
+	end
+
+	def close_session
+		@session_resource.delete(http_headers) if @session_resource
+	end
+
+	def safe_url(url)
+		url.sub(/\/\/(.+?)?:(.*?)@/, '//\1:[hidden]@')
+	end
+
+	def safe_remote_url
+		safe_url(remote_url)
+	end
+
+	def safe_database_url
+		safe_url(database_url)
+	end
+
+	def http_headers(extra = {})
+		{ :taps_version => Taps.compatible_version }.merge(extra)
+	end
+
+	def cmd_send
+		begin
+			verify_server
+			cmd_send_schema
+			cmd_send_data
+			cmd_send_indexes
+			cmd_send_reset_sequences
+		rescue RestClient::Exception => e
+			if e.respond_to?(:response)
+				puts "!!! Caught Server Exception"
+				puts "#{e.response.body}"
+				exit(1)
+			else
+				raise
+			end
+		end
+	end
+
+	def cmd_send_indexes
+		puts "Sending indexes"
+
+		index_data = Taps::Schema.indexes(database_url)
+		session_resource['indexes'].post(index_data, http_headers)
+	end
+
+	def cmd_send_schema
+		puts "Sending schema"
+
+		schema_data = Taps::Schema.dump_without_indexes(database_url)
+		session_resource['schema'].post(schema_data, http_headers)
+	end
+
+	def cmd_send_reset_sequences
+		puts "Resetting sequences"
+
+		session_resource["reset_sequences"].post('', http_headers)
+	end
+
+	def cmd_send_data
+		puts "Sending data"
+
+		tables_with_counts, record_count = fetch_tables_info
+
+		puts "#{tables_with_counts.size} tables, #{format_number(record_count)} records"
+
+
+		db.tables.each do |table_name|
+			table = db[table_name]
+			count = table.count
+			order = Taps::Utils.order_by(db, table_name)
+			chunksize = self.default_chunksize
+			string_columns = Taps::Utils.incorrect_blobs(db, table_name)
+
+			progress = ProgressBar.new(table_name.to_s, count)
+
+			offset = 0
+			loop do
+				row_size = 0
+				chunksize = Taps::Utils.calculate_chunksize(chunksize) do |c|
+					rows = Taps::Utils.format_data(table.order(*order).limit(c, offset).all, string_columns)
+					break if rows == { }
+
+					row_size = rows[:data].size
+					gzip_data = Taps::Utils.gzip(Marshal.dump(rows))
+
+					begin
+						session_resource["tables/#{table_name}"].post(gzip_data, http_headers({
+							:content_type => 'application/octet-stream',
+							:taps_checksum => Taps::Utils.checksum(gzip_data).to_s}))
+					rescue RestClient::RequestFailed => e
+						# retry the same data, it got corrupted somehow.
+						if e.http_code == 412
+							next
+						end
+						raise
+					end
+				end
+
+				progress.inc(row_size)
+				offset += row_size
+
+				break if row_size == 0
+			end
+
+			progress.finish
+		end
+	end
+
+	def fetch_tables_info
+		record_count = 0
+		tables = db.tables
+		tables_with_counts = tables.inject({}) do |accum, table|
+			accum[table] = db[table].count
+			record_count += accum[table]
+			accum
+		end
+
+		[ tables_with_counts, record_count ]
+	end
+
+	def cmd_receive
+		begin
+			verify_server
+			cmd_receive_schema
+			cmd_receive_data
+			cmd_receive_indexes
+			cmd_reset_sequences
+		rescue RestClient::Exception => e
+			if e.respond_to?(:response)
+				puts "!!! Caught Server Exception"
+				puts "#{e.response.body}"
+				exit(1)
+			else
+				raise
+			end
+		end
+	end
+
+	def cmd_receive_data
+		puts "Receiving data"
+
+		tables_with_counts, record_count = fetch_remote_tables_info
+
+		puts "#{tables_with_counts.size} tables, #{format_number(record_count)} records"
+
+		tables_with_counts.each do |table_name, count|
+			table = db[table_name.to_sym]
+			chunksize = default_chunksize
+
+			progress = ProgressBar.new(table_name.to_s, count)
+
+			offset = 0
+			loop do
+				begin
+					chunksize, rows = fetch_table_rows(table_name, chunksize, offset)
+				rescue CorruptedData
+					next
+				end
+				break if rows == { }
+
+				table.import(rows[:header], rows[:data])
+
+				progress.inc(rows[:data].size)
+				offset += rows[:data].size
+			end
+
+			progress.finish
+		end
+	end
+
+	class CorruptedData < Exception; end
+
+	def fetch_table_rows(table_name, chunksize, offset)
+		response = nil
+		chunksize = Taps::Utils.calculate_chunksize(chunksize) do |c|
+			response = session_resource["tables/#{table_name}/#{c}?offset=#{offset}"].get(http_headers)
+		end
+		raise CorruptedData unless Taps::Utils.valid_data?(response.to_s, response.headers[:taps_checksum])
+
+		begin
+			rows = Marshal.load(Taps::Utils.gunzip(response.to_s))
+		rescue Object => e
+			puts "Error encountered loading data, wrote the data chunk to dump.#{Process.pid}.gz"
+			File.open("dump.#{Process.pid}.gz", "w") { |f| f.write(response.to_s) }
+			raise
+		end
+		[chunksize, rows]
+	end
+
+	def fetch_remote_tables_info
+		retries = 0
+		max_retries = 1
+		begin
+			tables_with_counts = Marshal.load(session_resource['tables'].get(http_headers))
+			record_count = tables_with_counts.values.inject(0) { |a,c| a += c }
+		rescue RestClient::Exception
+			retries += 1
+			retry if retries <= max_retries
+			puts "Unable to fetch tables information from #{remote_url}. Please check the server log."
+			exit(1)
+		end
+
+		[ tables_with_counts, record_count ]
+	end
+
+	def cmd_receive_schema
+		puts "Receiving schema"
+
+		schema_data = session_resource['schema'].get(http_headers)
+		output = Taps::Schema.load(database_url, schema_data)
+		puts output if output
+	end
+
+	def cmd_receive_indexes
+		puts "Receiving indexes"
+
+		index_data = session_resource['indexes'].get(http_headers)
+
+		output = Taps::Schema.load_indexes(database_url, index_data)
+		puts output if output
+	end
+
+	def cmd_reset_sequences
+		puts "Resetting sequences"
+
+		output = Taps::Schema.reset_db_sequences(database_url)
+		puts output if output
+	end
+
+	def format_number(num)
+		num.to_s.gsub(/(\d)(?=(\d\d\d)+(?!\d))/, "\\1,")
+	end
+
+	def verify_server
+		begin
+			server['/'].get(http_headers)
+		rescue RestClient::RequestFailed => e
+			if e.http_code == 417
+				puts "#{safe_remote_url} is running a different minor version of taps."
+				puts "#{e.response.body}"
+				exit(1)
+			else
+				raise
+			end
+		rescue RestClient::Unauthorized
+			puts "Bad credentials given for #{safe_remote_url}"
+			exit(1)
+		rescue Errno::ECONNREFUSED
+			puts "Can't connect to #{safe_remote_url}. Please check that it's running"
+			exit(1)
+		end
+	end
+end
+end