ricardochimal-taps 0.2.0

data/LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 2008 Ricardo Chimal, Jr
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.rdoc

@@ -0,0 +1,29 @@
+= Taps -- simple database import/export app
+
+A simple database agnostic import/export app to transfer data to/from a remote database.
+
+== Usage: Server
+
+  $ taps server postgres://localdbuser:localdbpass@localhost/dbname user password
+
+== Usage: Client
+
+When you want to pull down a database from a taps server
+
+  $ taps pull postgres://dbuser:dbpassword@localhost/dbname http://user:password@example.com:5000
+
+or when you want to push a local database to a taps server
+
+  $ taps push postgres://dbuser:dbpassword@localhost/dbname http://user:password@example.com:5000
+
+== Meta
+
+Maintained by Ricardo Chimal, Jr. (ricardo at heroku dot com)
+
+Written by Ricardo Chimal, Jr. (ricardo at heroku dot com) and Adam Wiggins (adam at heroku dot com)
+
+Early research and inspiration by Blake Mizerany
+
+Released under the MIT License: http://www.opensource.org/licenses/mit-license.php
+
+http://github.com/ricardochimal/taps

data/Rakefile

@@ -0,0 +1,52 @@
+begin
+  require 'jeweler'
+  Jeweler::Tasks.new do |s|
+    s.name = "taps"
+    s.summary = %Q{simple database import/export app}
+    s.email = "ricardo@heroku.com"
+    s.homepage = "http://github.com/ricardochimal/taps"
+    s.description = "A simple database agnostic import/export app to transfer data to/from a remote database."
+    s.authors = ["Ricardo Chimal, Jr.", "Adam Wiggins"]
+
+    s.add_dependency 'sinatra', '~> 0.9.0'
+    s.add_dependency 'activerecord', '= 2.2.2'
+    s.add_dependency 'thor', '= 0.9.9'
+    s.add_dependency 'rest-client', '~> 0.9.0'
+    s.add_dependency 'sequel', '~> 2.10.0'
+
+    s.rubyforge_project = "taps"
+    s.rubygems_version = '1.3.1'
+
+    s.files = FileList['spec/*.rb'] + FileList['lib/**/*.rb'] + ['README.rdoc', 'LICENSE', 'VERSION.yml', 'Rakefile']
+    s.executables = ['taps', 'schema']
+  end
+rescue LoadError
+  puts "Jeweler not available. Install it with: sudo gem install technicalpickles-jeweler -s http://gems.github.com"
+end
+
+require 'rake/rdoctask'
+Rake::RDocTask.new do |rdoc|
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title = 'taps'
+  rdoc.options << '--line-numbers' << '--inline-source'
+  rdoc.rdoc_files.include('README*')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+end
+
+begin
+  require 'rcov/rcovtask'
+  Rcov::RcovTask.new do |t|
+    t.libs << 'spec'
+    t.test_files = FileList['spec/*_spec.rb']
+    t.verbose = true
+  end
+rescue LoadError
+  puts "RCov is not available. In order to run rcov, you must: sudo gem install spicycode-rcov"
+end
+
+desc "Run all specs"
+task :spec do
+  system "bacon #{File.dirname(__FILE__)}/spec/*_spec.rb"
+end
+
+task :default => :spec

data/VERSION.yml

@@ -0,0 +1,4 @@
+--- 
+:major: 0
+:minor: 2
+:patch: 0

data/bin/schema

@@ -0,0 +1,38 @@
+#!/usr/bin/env ruby
+
+require 'rubygems'
+gem 'activerecord', '= 2.2.2'
+
+require File.dirname(__FILE__) + '/../lib/taps/schema'
+
+cmd = ARGV.shift.strip rescue ''
+database_url = ARGV.shift.strip rescue ''
+
+def show_usage_and_exit
+	puts <<EOTXT
+	schema dump <database_url>
+	schema indexes <database_url>
+	schema reset_db_sequences <database_url>
+	schema load <database_url> <schema_file>
+	schema load_indexes <database_url> <indexes_file>
+EOTXT
+	exit(1)
+end
+
+if cmd == 'dump'
+	puts Taps::Schema.dump_without_indexes(database_url)
+elsif cmd == 'indexes'
+	puts Taps::Schema.indexes(database_url)
+elsif cmd == 'load_indexes'
+	filename = ARGV.shift.strip rescue ''
+	indexes = File.read(filename) rescue show_usage_and_exit
+	Taps::Schema.load_indexes(database_url, indexes)
+elsif cmd == 'load'
+	filename = ARGV.shift.strip rescue ''
+	schema = File.read(filename) rescue show_usage_and_exit
+	Taps::Schema.load(database_url, schema)
+elsif cmd == 'reset_db_sequences'
+	Taps::Schema.reset_db_sequences(database_url)
+else
+	show_usage_and_exit
+end

data/bin/taps

@@ -0,0 +1,12 @@
+#!/usr/bin/env ruby
+
+require 'rubygems'
+gem 'activerecord', '= 2.2.2'
+gem 'thor', '= 0.9.9'
+gem 'rest-client', '~> 0.9.0'
+gem 'sinatra', '~> 0.9.0'
+gem 'sequel', '~> 2.10.0'
+
+require File.dirname(__FILE__) + '/../lib/taps/cli'
+
+Taps::Cli.start

data/lib/taps/cli.rb

@@ -0,0 +1,57 @@
+require 'thor'
+require File.dirname(__FILE__) + '/config'
+
+Taps::Config.taps_database_url = 'sqlite://taps.db'
+
+module Taps
+class Cli < Thor
+	desc "server <local_database_url> <login> <password>", "Start a taps database import/export server"
+	method_options(:port => :numeric)
+	def server(database_url, login, password)
+		Taps::Config.database_url = database_url
+		Taps::Config.login = login
+		Taps::Config.password = password
+
+		port = options[:port] || 5000
+
+		Taps::Config.verify_database_url
+
+		require File.dirname(__FILE__) + '/server'
+		Taps::Server.run!({
+			:port => port,
+			:environment => :production,
+			:logging => true
+		})
+	end
+
+	desc "pull <local_database_url> <remote_url>", "Pull a database from a taps server"
+	method_options(:chunksize => :numeric)
+	def pull(database_url, remote_url)
+		clientxfer(:cmd_receive, database_url, remote_url)
+	end
+
+	desc "push <local_database_url> <remote_url>", "Push a database to a taps server"
+	method_options(:chunksize => :numeric)
+	def push(database_url, remote_url)
+		clientxfer(:cmd_send, database_url, remote_url)
+	end
+
+	def clientxfer(method, database_url, remote_url)
+		if options[:chunksize]
+			Taps::Config.chunksize = options[:chunksize] < 100 ? 100 : options[:chunksize]
+		else
+			Taps::Config.chunksize = 1000
+		end
+		Taps::Config.database_url = database_url
+		Taps::Config.remote_url = remote_url
+
+		Taps::Config.verify_database_url
+
+		require File.dirname(__FILE__) + '/client_session'
+
+		Taps::ClientSession.quickstart do |session|
+			session.send(method)
+		end
+	end
+end
+end

data/lib/taps/client_session.rb

@@ -0,0 +1,236 @@
+require 'rest_client'
+require 'sequel'
+require 'zlib'
+
+require File.dirname(__FILE__) + '/progress_bar'
+require File.dirname(__FILE__) + '/config'
+require File.dirname(__FILE__) + '/utils'
+
+module Taps
+class ClientSession
+	attr_reader :database_url, :remote_url, :default_chunksize
+
+	def initialize(database_url, remote_url, default_chunksize)
+		@database_url = database_url
+		@remote_url = remote_url
+		@default_chunksize = default_chunksize
+	end
+
+	def self.start(database_url, remote_url, default_chunksize, &block)
+		s = new(database_url, remote_url, default_chunksize)
+		yield s
+		s.close_session
+	end
+
+	def self.quickstart(&block)
+		start(Taps::Config.database_url, Taps::Config.remote_url, Taps::Config.chunksize) do |s|
+			yield s
+		end
+	end
+
+	def db
+		@db ||= Sequel.connect(database_url)
+	end
+
+	def server
+		@server ||= RestClient::Resource.new(remote_url)
+	end
+
+	def session_resource
+		@session_resource ||= open_session
+	end
+
+	def open_session
+		uri = server['sessions'].post('', :taps_version => Taps::VERSION)
+		server[uri]
+	end
+
+	def close_session
+		@session_resource.delete(:taps_version => Taps::VERSION) if @session_resource
+	end
+
+	def cmd_send
+		verify_server
+		cmd_send_schema
+		cmd_send_data
+		cmd_send_indexes
+		cmd_send_reset_sequences
+	end
+
+	def cmd_send_indexes
+		puts "Sending schema indexes to remote taps server #{remote_url} from local database #{database_url}"
+
+		index_data = `#{File.dirname(__FILE__)}/../../bin/schema indexes #{database_url}`
+		session_resource['indexes'].post(index_data, :taps_version => Taps::VERSION)
+	end
+
+	def cmd_send_schema
+		puts "Sending schema to remote taps server #{remote_url} from local database #{database_url}"
+
+		schema_data = `#{File.dirname(__FILE__)}/../../bin/schema dump #{database_url}`
+		session_resource['schema'].post(schema_data, :taps_version => Taps::VERSION)
+	end
+
+	def cmd_send_reset_sequences
+		puts "Resetting db sequences in remote taps server at #{remote_url}"
+
+		session_resource["reset_sequences"].post('', :taps_version => Taps::VERSION)
+	end
+
+	def cmd_send_data
+		puts "Sending schema and data from local database #{database_url} to remote taps server at #{remote_url}"
+
+		db.tables.each do |table_name|
+			table = db[table_name]
+			count = table.count
+			columns = table.columns
+			order = columns.include?(:id) ? :id : columns.first
+			chunksize = self.default_chunksize
+
+			progress = ProgressBar.new(table_name.to_s, count)
+
+			offset = 0
+			loop do
+				rows = Taps::Utils.format_data(table.order(order).limit(chunksize, offset).all)
+				break if rows == { }
+
+				gzip_data = Taps::Utils.gzip(Marshal.dump(rows))
+
+				chunksize = Taps::Utils.calculate_chunksize(chunksize) do
+					begin
+						session_resource["tables/#{table_name}"].post(gzip_data,
+							:taps_version => Taps::VERSION,
+							:content_type => 'application/octet-stream',
+							:taps_checksum => Taps::Utils.checksum(gzip_data).to_s)
+					rescue RestClient::RequestFailed => e
+						# retry the same data, it got corrupted somehow.
+						if e.http_code == 412
+							next
+						end
+						raise
+					end
+				end
+
+				progress.inc(rows[:data].size)
+				offset += rows[:data].size
+			end
+
+			progress.finish
+		end
+	end
+
+	def cmd_receive
+		verify_server
+		cmd_receive_schema
+		cmd_receive_data
+		cmd_receive_indexes
+		cmd_reset_sequences
+	end
+
+	def cmd_receive_data
+		puts "Receiving data from remote taps server #{remote_url} into local database #{database_url}"
+
+		tables_with_counts, record_count = fetch_tables_info
+
+		puts "#{tables_with_counts.size} tables, #{format_number(record_count)} records"
+
+		tables_with_counts.each do |table_name, count|
+			table = db[table_name.to_sym]
+			chunksize = default_chunksize
+
+			progress = ProgressBar.new(table_name.to_s, count)
+
+			offset = 0
+			loop do
+				begin
+					chunksize, rows = fetch_table_rows(table_name, chunksize, offset)
+				rescue CorruptedData
+					next
+				end
+				break if rows == { }
+
+				table.multi_insert(rows[:header], rows[:data])
+
+				progress.inc(rows[:data].size)
+				offset += rows[:data].size
+			end
+
+			progress.finish
+		end
+	end
+
+	class CorruptedData < Exception; end
+
+	def fetch_table_rows(table_name, chunksize, offset)
+		response = nil
+		chunksize = Taps::Utils.calculate_chunksize(chunksize) do
+			response = session_resource["tables/#{table_name}/#{chunksize}?offset=#{offset}"].get(:taps_version => Taps::VERSION)
+		end
+		raise CorruptedData unless Taps::Utils.valid_data?(response.to_s, response.headers[:taps_checksum])
+
+		rows = Marshal.load(Taps::Utils.gunzip(response.to_s))
+		[chunksize, rows]
+	end
+
+	def fetch_tables_info
+		retries = 0
+		max_retries = 1
+		begin
+			tables_with_counts = Marshal.load(session_resource['tables'].get(:taps_version => Taps::VERSION))
+			record_count = tables_with_counts.values.inject(0) { |a,c| a += c }
+		rescue RestClient::Exception
+			retries += 1
+			retry if retries <= max_retries
+			puts "Unable to fetch tables information from #{remote_url}. Please check the server log."
+			exit(1)
+		end
+
+		[ tables_with_counts, record_count ]
+	end
+
+	def cmd_receive_schema
+		puts "Receiving schema from remote taps server #{remote_url} into local database #{database_url}"
+
+		schema_data = session_resource['schema'].get(:taps_version => Taps::VERSION)
+		puts Taps::Utils.load_schema(database_url, schema_data)
+	end
+
+	def cmd_receive_indexes
+		puts "Receiving schema indexes from remote taps server #{remote_url} into local database #{database_url}"
+
+		index_data = session_resource['indexes'].get(:taps_version => Taps::VERSION)
+
+		puts Taps::Utils.load_indexes(database_url, index_data)
+	end
+
+	def cmd_reset_sequences
+		puts "Resetting db sequences in #{database_url}"
+
+		puts `#{File.dirname(__FILE__)}/../../bin/schema reset_db_sequences #{database_url}`
+	end
+
+	def format_number(num)
+		num.to_s.gsub(/(\d)(?=(\d\d\d)+(?!\d))/, "\\1,")
+	end
+
+	def verify_server
+		begin
+			server['/'].get(:taps_version => Taps::VERSION)
+		rescue RestClient::RequestFailed => e
+			if e.http_code == 417
+				puts "#{remote_url} is running a different version of taps."
+				puts "#{e.response.body}"
+				exit(1)
+			else
+				raise
+			end
+		rescue RestClient::Unauthorized
+			puts "Bad credentials given for #{remote_url}"
+			exit(1)
+		rescue Errno::ECONNREFUSED
+			puts "Can't connect to #{remote_url}. Please check that it's running"
+			exit(1)
+		end
+	end
+end
+end

data/lib/taps/config.rb

@@ -0,0 +1,23 @@
+require 'sequel'
+
+module Taps
+
+VERSION = '0.2.0'
+
+class Config
+	class << self
+		attr_accessor :taps_database_url
+		attr_accessor :login, :password, :database_url, :remote_url
+		attr_accessor :chunksize
+
+		def verify_database_url
+			db = Sequel.connect(self.database_url)
+			db.tables
+			db.disconnect
+		rescue Object => e
+			puts "Failed to connect to database:\n  #{e.class} -> #{e}"
+			exit 1
+		end
+	end
+end
+end

data/lib/taps/db_session.rb

@@ -0,0 +1,25 @@
+Sequel::Model.db = Sequel.connect(Taps::Config.taps_database_url)
+
+class DbSession < Sequel::Model
+	set_schema do
+		primary_key :id
+		text :key
+		text :database_url
+		timestamp :started_at
+		timestamp :last_access
+	end
+
+	def connection
+		@@connections ||= {}
+		@@connections[key] ||= Sequel.connect(database_url)
+	end
+
+	def disconnect
+		if defined? @@connections and @@connections[key]
+			@@connections[key].disconnect
+			@@connections.delete key
+		end
+	end
+end
+
+DbSession.create_table! unless DbSession.table_exists?

data/lib/taps/progress_bar.rb

@@ -0,0 +1,236 @@
+#
+# Ruby/ProgressBar - a text progress bar library
+#
+# Copyright (C) 2001-2005 Satoru Takabayashi <satoru@namazu.org>
+#     All rights reserved.
+#     This is free software with ABSOLUTELY NO WARRANTY.
+#
+# You can redistribute it and/or modify it under the terms
+# of Ruby's license.
+#
+
+class ProgressBar
+  VERSION = "0.9"
+
+  def initialize (title, total, out = STDERR)
+    @title = title
+    @total = total
+    @out = out
+    @terminal_width = 80
+    @bar_mark = "o"
+    @current = 0
+    @previous = 0
+    @finished_p = false
+    @start_time = Time.now
+    @previous_time = @start_time
+    @title_width = 14
+    @format = "%-#{@title_width}s %3d%% %s %s"
+    @format_arguments = [:title, :percentage, :bar, :stat]
+    clear
+    show
+  end
+  attr_reader   :title
+  attr_reader   :current
+  attr_reader   :total
+  attr_accessor :start_time
+
+  private
+  def fmt_bar
+    bar_width = do_percentage * @terminal_width / 100
+    sprintf("|%s%s|", 
+            @bar_mark * bar_width, 
+            " " *  (@terminal_width - bar_width))
+  end
+
+  def fmt_percentage
+    do_percentage
+  end
+
+  def fmt_stat
+    if @finished_p then elapsed else eta end
+  end
+
+  def fmt_stat_for_file_transfer
+    if @finished_p then 
+      sprintf("%s %s %s", bytes, transfer_rate, elapsed)
+    else 
+      sprintf("%s %s %s", bytes, transfer_rate, eta)
+    end
+  end
+
+  def fmt_title
+    @title[0,(@title_width - 1)] + ":"
+  end
+
+  def convert_bytes (bytes)
+    if bytes < 1024
+      sprintf("%6dB", bytes)
+    elsif bytes < 1024 * 1000 # 1000kb
+      sprintf("%5.1fKB", bytes.to_f / 1024)
+    elsif bytes < 1024 * 1024 * 1000  # 1000mb
+      sprintf("%5.1fMB", bytes.to_f / 1024 / 1024)
+    else
+      sprintf("%5.1fGB", bytes.to_f / 1024 / 1024 / 1024)
+    end
+  end
+
+  def transfer_rate
+    bytes_per_second = @current.to_f / (Time.now - @start_time)
+    sprintf("%s/s", convert_bytes(bytes_per_second))
+  end
+
+  def bytes
+    convert_bytes(@current)
+  end
+
+  def format_time (t)
+    t = t.to_i
+    sec = t % 60
+    min  = (t / 60) % 60
+    hour = t / 3600
+    sprintf("%02d:%02d:%02d", hour, min, sec);
+  end
+
+  # ETA stands for Estimated Time of Arrival.
+  def eta
+    if @current == 0
+      "ETA:  --:--:--"
+    else
+      elapsed = Time.now - @start_time
+      eta = elapsed * @total / @current - elapsed;
+      sprintf("ETA:  %s", format_time(eta))
+    end
+  end
+
+  def elapsed
+    elapsed = Time.now - @start_time
+    sprintf("Time: %s", format_time(elapsed))
+  end
+  
+  def eol
+    if @finished_p then "\n" else "\r" end
+  end
+
+  def do_percentage
+    if @total.zero?
+      100
+    else
+      @current  * 100 / @total
+    end
+  end
+
+  def get_width
+    # FIXME: I don't know how portable it is.
+    default_width = 80
+    begin
+      tiocgwinsz = 0x5413
+      data = [0, 0, 0, 0].pack("SSSS")
+      if @out.ioctl(tiocgwinsz, data) >= 0 then
+        rows, cols, xpixels, ypixels = data.unpack("SSSS")
+        if cols >= 0 then cols else default_width end
+      else
+        default_width
+      end
+    rescue Exception
+      default_width
+    end
+  end
+
+  def show
+    arguments = @format_arguments.map {|method| 
+      method = sprintf("fmt_%s", method)
+      send(method)
+    }
+    line = sprintf(@format, *arguments)
+
+    width = get_width
+    if line.length == width - 1 
+      @out.print(line + eol)
+      @out.flush
+    elsif line.length >= width
+      @terminal_width = [@terminal_width - (line.length - width + 1), 0].max
+      if @terminal_width == 0 then @out.print(line + eol) else show end
+    else # line.length < width - 1
+      @terminal_width += width - line.length + 1
+      show
+    end
+    @previous_time = Time.now
+  end
+
+  def show_if_needed
+    if @total.zero?
+      cur_percentage = 100
+      prev_percentage = 0
+    else
+      cur_percentage  = (@current  * 100 / @total).to_i
+      prev_percentage = (@previous * 100 / @total).to_i
+    end
+
+    # Use "!=" instead of ">" to support negative changes
+    if cur_percentage != prev_percentage || 
+        Time.now - @previous_time >= 1 || @finished_p
+      show
+    end
+  end
+
+  public
+  def clear
+    @out.print "\r"
+    @out.print(" " * (get_width - 1))
+    @out.print "\r"
+  end
+
+  def finish
+    @current = @total
+    @finished_p = true
+    show
+  end
+
+  def finished?
+    @finished_p
+  end
+
+  def file_transfer_mode
+    @format_arguments = [:title, :percentage, :bar, :stat_for_file_transfer]
+  end
+
+  def format= (format)
+    @format = format
+  end
+
+  def format_arguments= (arguments)
+    @format_arguments = arguments
+  end
+
+  def halt
+    @finished_p = true
+    show
+  end
+
+  def inc (step = 1)
+    @current += step
+    @current = @total if @current > @total
+    show_if_needed
+    @previous = @current
+  end
+
+  def set (count)
+    if count < 0 || count > @total
+      raise "invalid count: #{count} (total: #{@total})"
+    end
+    @current = count
+    show_if_needed
+    @previous = @current
+  end
+
+  def inspect
+    "#<ProgressBar:#{@current}/#{@total}>"
+  end
+end
+
+class ReversedProgressBar < ProgressBar
+  def do_percentage
+    100 - super
+  end
+end
+

data/lib/taps/schema.rb

@@ -0,0 +1,82 @@
+require 'active_record'
+require 'active_support'
+require 'stringio'
+require 'uri'
+
+module Taps
+module Schema
+	extend self
+
+	def create_config(url)
+		uri = URI.parse(url)
+		adapter = uri.scheme
+		adapter = 'postgresql' if adapter == 'postgres'
+		adapter = 'sqlite3' if adapter == 'sqlite'
+		config = {
+			'adapter' => adapter,
+			'database' => uri.path.blank? ? uri.host : uri.path.split('/')[1],
+			'username' => uri.user,
+			'password' => uri.password,
+			'host' => uri.host,
+		}
+	end
+
+	def connection(database_url)
+		config = create_config(database_url)
+		ActiveRecord::Base.establish_connection(config)
+	end
+
+	def dump(database_url)
+		connection(database_url)
+
+		stream = StringIO.new
+		ActiveRecord::SchemaDumper.ignore_tables = []
+		ActiveRecord::SchemaDumper.dump(ActiveRecord::Base.connection, stream)
+		stream.string
+	end
+
+	def dump_without_indexes(database_url)
+		schema = dump(database_url)
+		schema.split("\n").collect do |line|
+			if line =~ /^\s+add_index/
+				line = "##{line}"
+			end
+			line
+		end.join("\n")
+	end
+
+	def indexes(database_url)
+		schema = dump(database_url)
+		schema.split("\n").collect do |line|
+			line if line =~ /^\s+add_index/
+		end.uniq.join("\n")
+	end
+
+	def load(database_url, schema)
+		connection(database_url)
+		eval(schema)
+		ActiveRecord::Base.connection.execute("DELETE FROM schema_migrations") rescue nil
+	end
+
+	def load_indexes(database_url, indexes)
+		connection(database_url)
+
+		schema =<<EORUBY
+ActiveRecord::Schema.define do
+	#{indexes}
+end
+EORUBY
+		eval(schema)
+	end
+
+	def reset_db_sequences(database_url)
+		connection(database_url)
+
+		if ActiveRecord::Base.connection.respond_to?(:reset_pk_sequence!)
+			ActiveRecord::Base.connection.tables.each do |table|
+				ActiveRecord::Base.connection.reset_pk_sequence!(table)
+			end
+		end
+	end
+end
+end

data/lib/taps/server.rb

@@ -0,0 +1,141 @@
+require 'sinatra/base'
+require File.dirname(__FILE__) + '/config'
+require File.dirname(__FILE__) + '/utils'
+require File.dirname(__FILE__) + '/db_session'
+
+module Taps
+class Server < Sinatra::Base
+	use Rack::Auth::Basic do |login, password|
+		login == Taps::Config.login && password == Taps::Config.password
+	end
+
+	error do
+		e = request.env['sinatra.error']
+		puts e.to_s
+		puts e.backtrace.join("\n")
+		"Application error"
+	end
+
+	before do
+		unless request.env['HTTP_TAPS_VERSION'] == Taps::VERSION
+			halt 417, "Taps version #{Taps::VERSION} is required for this server"
+		end
+	end
+
+	get '/' do
+		"hello"
+	end
+
+	post '/sessions' do
+		key = rand(9999999999).to_s
+		database_url = Taps::Config.database_url || request.body.string
+
+		DbSession.create(:key => key, :database_url => database_url, :started_at => Time.now, :last_access => Time.now)
+
+		"/sessions/#{key}"
+	end
+
+	post '/sessions/:key/tables/:table' do
+		session = DbSession.filter(:key => params[:key]).first
+		halt 404 unless session
+
+		gzip_data = request.body.read
+		halt 412 unless Taps::Utils.valid_data?(gzip_data, request.env['HTTP_TAPS_CHECKSUM'])
+
+		rows = Marshal.load(Taps::Utils.gunzip(gzip_data))
+
+		db = session.connection
+		table = db[params[:table].to_sym]
+		table.multi_insert(rows[:header], rows[:data])
+
+		"#{rows[:data].size}"
+	end
+
+	post '/sessions/:key/reset_sequences' do
+		session = DbSession.filter(:key => params[:key]).first
+		halt 404 unless session
+
+		schema_app = File.dirname(__FILE__) + '/../../bin/schema'
+		`#{schema_app} reset_db_sequences #{session.database_url}`
+	end
+
+	post '/sessions/:key/schema' do
+		session = DbSession.filter(:key => params[:key]).first
+		halt 404 unless session
+
+		schema_data = request.body.read
+		Taps::Utils.load_schema(session.database_url, schema_data)
+	end
+
+	post '/sessions/:key/indexes' do
+		session = DbSession.filter(:key => params[:key]).first
+		halt 404 unless session
+
+		index_data = request.body.read
+		Taps::Utils.load_indexes(session.database_url, index_data)
+	end
+
+	get '/sessions/:key/schema' do
+		session = DbSession.filter(:key => params[:key]).first
+		halt 404 unless session
+
+		schema_app = File.dirname(__FILE__) + '/../../bin/schema'
+		`#{schema_app} dump #{session.database_url}`
+	end
+
+	get '/sessions/:key/indexes' do
+		session = DbSession.filter(:key => params[:key]).first
+		halt 404 unless session
+
+		schema_app = File.dirname(__FILE__) + '/../../bin/schema'
+		`#{schema_app} indexes #{session.database_url}`
+	end
+
+	get '/sessions/:key/tables' do
+		session = DbSession.filter(:key => params[:key]).first
+		halt 404 unless session
+
+		db = session.connection
+		tables = db.tables
+
+		tables_with_counts = tables.inject({}) do |accum, table|
+			accum[table] = db[table].count
+			accum
+		end
+
+		Marshal.dump(tables_with_counts)
+	end
+
+	get '/sessions/:key/tables/:table/:chunk' do
+		session = DbSession.filter(:key => params[:key]).first
+		halt 404 unless session
+
+		chunk = params[:chunk].to_i
+		chunk = 500 if chunk < 1
+
+		offset = params[:offset].to_i
+		offset = 0 if offset < 0
+
+		db = session.connection
+		table = db[params[:table].to_sym]
+		columns = table.columns
+		order = columns.include?(:id) ? :id : columns.first
+		raw_data = Marshal.dump(Taps::Utils.format_data(table.order(order).limit(chunk, offset).all))
+		gzip_data = Taps::Utils.gzip(raw_data)
+		response['Taps-Checksum'] = Taps::Utils.checksum(gzip_data).to_s
+		response['Content-Type'] = "application/octet-stream"
+		gzip_data
+	end
+
+	delete '/sessions/:key' do
+		session = DbSession.filter(:key => params[:key]).first
+		halt 404 unless session
+
+		session.disconnect
+		session.destroy
+
+		"ok"
+	end
+
+end
+end

data/lib/taps/utils.rb

@@ -0,0 +1,76 @@
+require 'zlib'
+require 'stringio'
+require 'time'
+require 'tempfile'
+
+module Taps
+module Utils
+	extend self
+
+	def checksum(data)
+		Zlib.crc32(data)
+	end
+
+	def valid_data?(data, crc32)
+		Zlib.crc32(data) == crc32.to_i
+	end
+
+	def gzip(data)
+		io = StringIO.new
+		gz = Zlib::GzipWriter.new(io)
+		gz.write data
+		gz.close
+		io.string
+	end
+
+	def gunzip(gzip_data)
+		io = StringIO.new(gzip_data)
+		gz = Zlib::GzipReader.new(io)
+		data = gz.read
+		gz.close
+		data
+	end
+
+	def format_data(data)
+		return {} if data.size == 0
+		header = data[0].keys
+		only_data = data.collect do |row|
+			header.collect { |h| row[h] }
+		end
+		{ :header => header, :data => only_data }
+	end
+
+	def calculate_chunksize(old_chunksize)
+		t1 = Time.now
+		yield
+		t2 = Time.now
+
+		diff = t2 - t1
+		new_chunksize = if diff > 3.0
+			(old_chunksize / 3).ceil
+		elsif diff > 1.1
+			old_chunksize - 100
+		elsif diff < 0.8
+			old_chunksize * 2
+		else
+			old_chunksize + 100
+		end
+		new_chunksize = 100 if new_chunksize < 100
+		new_chunksize
+	end
+
+	def load_schema(database_url, schema_data)
+		Tempfile.open('taps') do |tmp|
+			File.open(tmp.path, 'w') { |f| f.write(schema_data) }
+			`#{File.dirname(__FILE__)}/../../bin/schema load #{database_url} #{tmp.path}`
+		end
+	end
+
+	def load_indexes(database_url, index_data)
+		Tempfile.open('taps') do |tmp|
+			File.open(tmp.path, 'w') { |f| f.write(index_data) }
+			`#{File.dirname(__FILE__)}/../../bin/schema load_indexes #{database_url} #{tmp.path}`
+		end
+	end
+end
+end

data/spec/base.rb

@@ -0,0 +1,21 @@
+require 'rubygems'
+require 'bacon'
+require 'mocha'
+
+class Bacon::Context
+	include Mocha::Standalone
+
+	alias_method :old_it, :it
+	def it(description)
+		old_it(description) do
+			mocha_setup
+			yield
+			mocha_verify
+			mocha_teardown
+		end
+	end
+end
+
+require File.dirname(__FILE__) + '/../lib/taps/config'
+Taps::Config.taps_database_url = 'sqlite://test.db'
+Sequel.connect(Taps::Config.taps_database_url)

data/spec/client_session_spec.rb

@@ -0,0 +1,82 @@
+require File.dirname(__FILE__) + '/base'
+require File.dirname(__FILE__) + '/../lib/taps/client_session'
+
+describe Taps::ClientSession do
+	before do
+		@client = Taps::ClientSession.new('sqlite://my.db', 'http://example.com:3000', 1000)
+		@client.stubs(:session_resource).returns(mock('session resource'))
+	end
+
+	it "starts a session and yields the session object to the block" do
+		Taps::ClientSession.start('x', 'y', 1000) do |session|
+			session.database_url.should == 'x'
+			session.remote_url.should == 'y'
+			session.default_chunksize.should == 1000
+		end
+	end
+
+	it "opens the local db connection via sequel and the database url" do
+		Sequel.expects(:connect).with('sqlite://my.db').returns(:con)
+		@client.db.should == :con
+	end
+
+	it "creates a restclient resource to the remote server" do
+		@client.server.url.should == 'http://example.com:3000'
+	end
+
+	it "verifies the db version, receive the schema, data, indexes, then reset the sequences" do
+		@client.expects(:verify_server)
+		@client.expects(:cmd_receive_schema)
+		@client.expects(:cmd_receive_data)
+		@client.expects(:cmd_receive_indexes)
+		@client.expects(:cmd_reset_sequences)
+		@client.cmd_receive.should.be.nil
+	end
+
+	it "checks the version of the server by seeing if it has access" do
+		@client.stubs(:server).returns(mock('server'))
+		@request = mock('request')
+		@client.server.expects(:[]).with('/').returns(@request)
+		@request.expects(:get).with({:taps_version => Taps::VERSION})
+
+		lambda { @client.verify_server }.should.not.raise
+	end
+
+	it "receives data from a remote taps server" do
+		@client.stubs(:puts)
+		@progressbar = mock('progressbar')
+		ProgressBar.stubs(:new).with('mytable', 2).returns(@progressbar)
+		@progressbar.stubs(:inc)
+		@progressbar.stubs(:finish)
+		@mytable = mock('mytable')
+		@client.expects(:fetch_tables_info).returns([ { :mytable => 2 }, 2 ])
+		@client.stubs(:db).returns(mock('db'))
+		@client.db.stubs(:[]).with(:mytable).returns(@mytable)
+		@client.expects(:fetch_table_rows).with(:mytable, 1000, 0).returns([ 1000, { :header => [:x, :y], :data => [[1, 2], [3, 4]] } ])
+		@client.expects(:fetch_table_rows).with(:mytable, 1000, 2).returns([ 1000, { }])
+		@mytable.expects(:multi_insert).with([:x, :y], [[1, 2], [3, 4]])
+
+		lambda { @client.cmd_receive_data }.should.not.raise
+	end
+
+	it "fetches tables info from taps server" do
+		@marshal_data = Marshal.dump({ :mytable => 2 })
+		@client.session_resource.stubs(:[]).with('tables').returns(mock('tables'))
+		@client.session_resource['tables'].stubs(:get).with(:taps_version => Taps::VERSION).returns(@marshal_data)
+		@client.fetch_tables_info.should == [ { :mytable => 2 }, 2 ]
+	end
+
+	it "fetches table rows given a chunksize and offset from taps server" do
+		@data = { :header => [ :x, :y ], :data => [ [1, 2], [3, 4] ] }
+		@gzip_data = Taps::Utils.gzip(Marshal.dump(@data))
+		Taps::Utils.stubs(:calculate_chunksize).with(1000).yields.returns(1000)
+
+		@response = mock('response')
+		@client.session_resource.stubs(:[]).with('tables/mytable/1000?offset=0').returns(mock('table resource'))
+		@client.session_resource['tables/mytable/1000?offset=0'].expects(:get).with(:taps_version => Taps::VERSION).returns(@response)
+		@response.stubs(:to_s).returns(@gzip_data)
+		@response.stubs(:headers).returns({ :taps_checksum => Taps::Utils.checksum(@gzip_data) })
+		@client.fetch_table_rows('mytable', 1000, 0).should == [ 1000, { :header => [:x, :y], :data => [[1, 2], [3, 4]] } ]
+	end
+end
+

data/spec/schema_spec.rb

@@ -0,0 +1,38 @@
+require File.dirname(__FILE__) + '/base'
+require File.dirname(__FILE__) + '/../lib/taps/schema'
+
+describe Taps::Schema do
+	before do
+		@connection = mock("AR connection")
+		ActiveRecord::Base.stubs(:connection).returns(@connection)
+	end
+
+	it "parses a database url and returns a config hash for activerecord" do
+		Taps::Schema.create_config("postgres://myuser:mypass@localhost/mydb").should == {
+			'adapter' => 'postgresql',
+			'database' => 'mydb',
+			'username' => 'myuser',
+			'password' => 'mypass',
+			'host' => 'localhost'
+		}
+	end
+
+	it "translates sqlite in the database url to sqlite3" do
+		Taps::Schema.create_config("sqlite://myuser:mypass@localhost/mydb")['adapter'].should == 'sqlite3'
+	end
+
+	it "connects activerecord to the database" do
+		Taps::Schema.expects(:create_config).with("postgres://myuser:mypass@localhost/mydb").returns("db config")
+		ActiveRecord::Base.expects(:establish_connection).with("db config").returns(true)
+		Taps::Schema.connection("postgres://myuser:mypass@localhost/mydb").should == true
+	end
+
+	it "resets the db tables' primary keys" do
+		Taps::Schema.stubs(:connection)
+		ActiveRecord::Base.connection.expects(:respond_to?).with(:reset_pk_sequence!).returns(true)
+		ActiveRecord::Base.connection.stubs(:tables).returns(['table1'])
+		ActiveRecord::Base.connection.expects(:reset_pk_sequence!).with('table1')
+		should.not.raise { Taps::Schema.reset_db_sequences("postgres://myuser:mypass@localhost/mydb") }
+	end
+end
+

data/spec/server_spec.rb

@@ -0,0 +1,33 @@
+require File.dirname(__FILE__) + '/base'
+require 'sinatra'
+require 'sinatra/test/bacon'
+
+require File.dirname(__FILE__) + '/../lib/taps/server'
+
+require 'pp'
+
+describe Taps::Server do
+	before do
+		Taps::Config.login = 'taps'
+		Taps::Config.password = 'tpass'
+
+		@app = Taps::Server
+		@auth_header = "Basic " + ["taps:tpass"].pack("m*")
+	end
+
+	it "asks for http basic authentication" do
+		get '/'
+		status.should == 401
+	end
+
+	it "verifies the client taps version" do
+		get('/', { }, { 'HTTP_AUTHORIZATION' => @auth_header, 'HTTP_TAPS_VERSION' => Taps::VERSION })
+		status.should == 200
+	end
+
+	it "yells loudly if the client taps version doesn't match" do
+		get('/', { }, { 'HTTP_AUTHORIZATION' => @auth_header, 'HTTP_TAPS_VERSION' => '0.0.1' })
+		status.should == 417
+	end
+end
+

data/spec/utils_spec.rb

@@ -0,0 +1,45 @@
+require File.dirname(__FILE__) + '/base'
+require File.dirname(__FILE__) + '/../lib/taps/utils'
+
+describe Taps::Utils do
+	it "gunzips a string" do
+		@hello_world = "\037\213\b\000R\261\207I\000\003\313H\315\311\311W(\317/\312I\001\000\205\021J\r\v\000\000\000"
+		Taps::Utils.gunzip(@hello_world).should == "hello world"
+	end
+
+	it "gzips and gunzips a string and returns the same string" do
+		Taps::Utils.gunzip(Taps::Utils.gzip("hello world")).should == "hello world"
+	end
+
+	it "generates a checksum using crc32" do
+		Taps::Utils.checksum("hello world").should == Zlib.crc32("hello world")
+	end
+
+	it "formats a data hash into one hash that contains an array of headers and an array of array of data" do
+		Taps::Utils.format_data([ { :x => 1, :y => 1 }, { :x => 2, :y => 2 } ]).should == { :header => [ :x, :y ], :data => [ [1, 1], [2, 2] ] }
+	end
+
+	it "scales chunksize down slowly when the time delta of the block is just over a second" do
+		Time.stubs(:now).returns(10.0).returns(11.5)
+		Taps::Utils.calculate_chunksize(1000) { }.should == 900
+	end
+
+	it "scales chunksize down fast when the time delta of the block is over 3 seconds" do
+		Time.stubs(:now).returns(10.0).returns(15.0)
+		Taps::Utils.calculate_chunksize(3000) { }.should == 1000
+	end
+
+	it "scales up chunksize fast when the time delta of the block is under 0.8 seconds" do
+		Time.stubs(:now).returns(10.0).returns(10.7)
+		Taps::Utils.calculate_chunksize(1000) { }.should == 2000
+	end
+
+	it "scales up chunksize slow when the time delta of the block is between 0.8 and 1.1 seconds" do
+		Time.stubs(:now).returns(10.0).returns(10.8)
+		Taps::Utils.calculate_chunksize(1000) { }.should == 1100
+
+		Time.stubs(:now).returns(10.0).returns(11.1)
+		Taps::Utils.calculate_chunksize(1000) { }.should == 1100
+	end
+end
+

metadata

@@ -0,0 +1,118 @@
+--- !ruby/object:Gem::Specification 
+name: ricardochimal-taps
+version: !ruby/object:Gem::Version 
+  version: 0.2.0
+platform: ruby
+authors: 
+- Ricardo Chimal, Jr.
+- Adam Wiggins
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2009-02-06 00:00:00 -08:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: sinatra
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        version: 0.9.0
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: activerecord
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - "="
+      - !ruby/object:Gem::Version 
+        version: 2.2.2
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: thor
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - "="
+      - !ruby/object:Gem::Version 
+        version: 0.9.9
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: rest-client
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        version: 0.9.0
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: sequel
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        version: 2.10.0
+    version: 
+description: A simple database agnostic import/export app to transfer data to/from a remote database.
+email: ricardo@heroku.com
+executables: 
+- taps
+- schema
+extensions: []
+
+extra_rdoc_files: []
+
+files: 
+- spec/base.rb
+- spec/client_session_spec.rb
+- spec/schema_spec.rb
+- spec/server_spec.rb
+- spec/utils_spec.rb
+- lib/taps/config.rb
+- lib/taps/db_session.rb
+- lib/taps/progress_bar.rb
+- lib/taps/server.rb
+- lib/taps/utils.rb
+- lib/taps/schema.rb
+- lib/taps/client_session.rb
+- lib/taps/cli.rb
+- README.rdoc
+- LICENSE
+- VERSION.yml
+- Rakefile
+- bin/taps
+- bin/schema
+has_rdoc: true
+homepage: http://github.com/ricardochimal/taps
+post_install_message: 
+rdoc_options: 
+- --inline-source
+- --charset=UTF-8
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+requirements: []
+
+rubyforge_project: taps
+rubygems_version: 1.2.0
+signing_key: 
+specification_version: 2
+summary: simple database import/export app
+test_files: []
+