dkastner-taps 0.3.11

data/LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 2008 Ricardo Chimal, Jr
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.rdoc

@@ -0,0 +1,51 @@
+= Taps -- simple database import/export app
+
+A simple database agnostic import/export app to transfer data to/from a remote database.
+
+== Usage: Server
+
+Here's how you start a taps server
+
+  $ taps server postgres://localdbuser:localdbpass@localhost/dbname httpuser httppassword
+
+You can also specify an encoding in the database url
+
+  $ taps server mysql://localdbuser:localdbpass@localhost/dbname?encoding=latin1 httpuser httppassword
+
+== Usage: Client
+
+When you want to pull down a database from a taps server
+
+  $ taps pull postgres://dbuser:dbpassword@localhost/dbname http://httpuser:httppassword@example.com:5000
+
+or when you want to push a local database to a taps server
+
+  $ taps push postgres://dbuser:dbpassword@localhost/dbname http://httpuser:httppassword@example.com:5000
+
+or when you want to transfer a list of tables
+
+  $ taps push postgres://dbuser:dbpassword@localhost/dbname http://httpuser:httppassword@example.com:5000 --tables logs,tags
+
+or when you want to transfer tables that start with a word
+
+  $ taps push postgres://dbuser:dbpassword@localhost/dbname http://httpuser:httppassword@example.com:5000 --filter '^log_'
+
+== Known Issues
+
+* Foreign Keys get lost in the schema transfer
+* Tables without primary keys will be incredibly slow to transfer. This is due to it being inefficient having large offset values in queries.
+* Multiple schemas are currently not supported
+
+== Meta
+
+Maintained by Ricardo Chimal, Jr. (ricardo at heroku dot com)
+
+Written by Ricardo Chimal, Jr. (ricardo at heroku dot com) and Adam Wiggins (adam at heroku dot com)
+
+Early research and inspiration by Blake Mizerany
+
+Released under the MIT License: http://www.opensource.org/licenses/mit-license.php
+
+http://github.com/ricardochimal/taps
+
+Special Thanks to Sequel for making this tool possible http://sequel.rubyforge.org/

data/Rakefile

@@ -0,0 +1,75 @@
+begin
+  require 'jeweler'
+  Jeweler::Tasks.new do |s|
+    s.name = "dkastner-taps"
+    s.summary = %Q{simple database import/export app}
+    s.email = 'derek@brighterplanet.com'
+    s.homepage = "http://github.com/dkastner/taps"
+    s.description = "A simple database agnostic import/export app to transfer data to/from a remote database."
+    s.authors = ["Ricardo Chimal, Jr.", 'Derek Kastner']
+
+    s.rubygems_version = %q{1.3.5}
+
+    s.add_dependency 'json_pure', '>= 1.2.0', '< 1.5.0'
+    s.add_dependency 'sinatra', '~> 1.0.0'
+    s.add_dependency 'rest-client', '~> 1.4.0'
+    s.add_dependency 'sequel', '~> 3.13.0'
+    s.add_dependency 'sqlite3-ruby', '~> 1.2'
+    s.add_dependency 'rack', '>= 1.0.1'
+    s.add_development_dependency 'jeweler'
+    s.add_development_dependency 'bacon'
+
+    s.rubyforge_project = "taps"
+
+    s.files = FileList['spec/*.rb'] + FileList['lib/**/*.rb'] + ['README.rdoc', 'LICENSE', 'VERSION.yml', 'Rakefile'] + FileList['bin/*']
+    s.executables = ['taps', 'schema']
+  end
+  Jeweler::GemcutterTasks.new
+rescue LoadError => e
+  if e.message =~ /jeweler/
+    puts "Jeweler not available. Install it with: sudo gem install technicalpickles-jeweler -s http://gems.github.com"
+  else
+    puts e.message + ' -- while loading jeweler.'
+  end
+end
+
+begin
+  require 'rake/rdoctask'
+  Rake::RDocTask.new do |rdoc|
+    rdoc.rdoc_dir = 'rdoc'
+    rdoc.title = 'taps'
+    rdoc.options << '--line-numbers' << '--inline-source'
+    rdoc.rdoc_files.include('README*')
+    rdoc.rdoc_files.include('lib/**/*.rb')
+  end
+rescue LoadError
+   puts "Rdoc is not available"
+end
+
+begin
+  require 'rcov/rcovtask'
+  Rcov::RcovTask.new do |t|
+    t.libs << 'spec'
+    t.test_files = FileList['spec/*_spec.rb']
+    t.verbose = true
+  end
+rescue LoadError
+  puts "RCov is not available. In order to run rcov, you must: sudo gem install rcov"
+end
+
+desc "Run all specs; requires the bacon gem"
+task :spec do
+  if `which bacon`.empty?
+    puts "bacon is not available. In order to run the specs, you must: sudo gem install bacon."
+  else
+    system "bacon #{File.dirname(__FILE__)}/spec/*_spec.rb"
+  end
+end
+
+desc "copy/paste env vars for dev testing"
+task :env do
+	puts "export RUBYLIB='#{File.dirname(__FILE__) + '/lib'}'"
+	puts "export RUBYOPT='-rrubygems'"
+end
+
+task :default => :spec

data/TODO

@@ -0,0 +1 @@
+* Detect incompatible Marshal versions

data/VERSION.yml

@@ -0,0 +1,5 @@
+--- 
+:patch: 11
+:build: 
+:major: 0
+:minor: 3

data/bin/schema

@@ -0,0 +1,54 @@
+#!/usr/bin/env ruby
+
+require 'rubygems'
+gem 'sequel', '~> 3.13.0'
+
+$:.unshift File.dirname(__FILE__) + '/../lib'
+
+require 'taps/schema'
+
+cmd = ARGV.shift.strip rescue ''
+database_url = ARGV.shift.strip rescue ''
+
+def show_usage_and_exit
+	puts <<EOTXT
+	schema console <database_url>
+	schema dump <database_url>
+	schema dump_table <database_url> <table>
+	schema indexes <database_url>
+	schema indexes_individual <database_url>
+	schema reset_db_sequences <database_url>
+	schema load <database_url> <schema_file>
+	schema load_indexes <database_url> <indexes_file>
+EOTXT
+	exit(1)
+end
+
+case cmd
+when 'dump'
+	puts Taps::Schema.dump(database_url)
+when 'dump_table'
+	table = ARGV.shift.strip
+	puts Taps::Schema.dump_table(database_url, table)
+when 'indexes'
+	puts Taps::Schema.indexes(database_url)
+when 'indexes_individual'
+	puts Taps::Schema.indexes_individual(database_url)
+when 'load_indexes'
+	filename = ARGV.shift.strip rescue ''
+	indexes = File.read(filename) rescue show_usage_and_exit
+	Taps::Schema.load_indexes(database_url, indexes)
+when 'load'
+	filename = ARGV.shift.strip rescue ''
+	schema = File.read(filename) rescue show_usage_and_exit
+	Taps::Schema.load(database_url, schema)
+when 'reset_db_sequences'
+	Taps::Schema.reset_db_sequences(database_url)
+when 'console'
+	$db = Sequel.connect(database_url)
+	require 'irb'
+	require 'irb/completion'
+	IRB.start
+else
+	show_usage_and_exit
+end

data/bin/schema.cmd

@@ -0,0 +1,6 @@
+@ECHO OFF
+IF NOT "%~f0" == "~f0" GOTO :WinNT
+@"ruby.exe" "./schema" %1 %2 %3 %4 %5 %6 %7 %8 %9
+GOTO :EOF
+:WinNT
+@"ruby.exe" "%~dpn0" %*

data/bin/taps

@@ -0,0 +1,6 @@
+#!/usr/bin/env ruby
+
+$:.unshift File.dirname(__FILE__) + '/../lib'
+require 'taps/cli'
+
+Taps::Cli.new(ARGV.dup).run

data/lib/taps/cli.rb

@@ -0,0 +1,188 @@
+require 'optparse'
+require 'tempfile'
+require 'json/pure'
+require 'taps/monkey'
+require 'taps/config'
+require 'taps/log'
+
+Taps::Config.taps_database_url = ENV['TAPS_DATABASE_URL'] || "sqlite://#{Tempfile.new('taps.db').path}"
+
+module Taps
+class Cli
+	attr_accessor :argv
+
+	def initialize(argv)
+		@argv = argv
+	end
+
+	def run
+		method = (argv.shift || 'help').to_sym
+		if [:pull, :push, :server, :version].include? method
+			send(method)
+		else
+			help
+		end
+	end
+
+	def pull
+		opts = clientoptparse(:pull)
+		Taps.log.level = Logger::DEBUG if opts[:debug]
+		if opts[:resume_filename]
+			clientresumexfer(:pull, opts)
+		else
+			clientxfer(:pull, opts)
+		end
+	end
+
+	def push
+		opts = clientoptparse(:push)
+		Taps.log.level = Logger::DEBUG if opts[:debug]
+		if opts[:resume_filename]
+			clientresumexfer(:push, opts)
+		else
+			clientxfer(:push, opts)
+		end
+	end
+
+	def server
+		opts = serveroptparse
+		Taps.log.level = Logger::DEBUG if opts[:debug]
+		Taps::Config.database_url = opts[:database_url]
+		Taps::Config.login = opts[:login]
+		Taps::Config.password = opts[:password]
+
+		Taps::Config.verify_database_url
+		require 'taps/server'
+		Taps::Server.run!({
+			:port => opts[:port],
+			:environment => :production,
+			:logging => true,
+			:dump_errors => true,
+		})
+	end
+
+	def version
+		puts Taps.version
+	end
+
+	def help
+		puts <<EOHELP
+Options
+=======
+server    Start a taps database import/export server
+pull      Pull a database from a taps server
+push      Push a database to a taps server
+version   Taps version
+
+Add '-h' to any command to see their usage
+EOHELP
+	end
+
+	def serveroptparse
+		opts={:port => 5000, :database_url => nil, :login => nil, :password => nil, :debug => false}
+		OptionParser.new do |o|
+			o.banner = "Usage: #{File.basename($0)} server [OPTIONS] <local_database_url> <login> <password>"
+			o.define_head "Start a taps database import/export server"
+
+			o.on("-p", "--port=N", "Server Port") { |v| opts[:port] = v.to_i if v.to_i > 0 }
+			o.on("-d", "--debug", "Enable Debug Messages") { |v| opts[:debug] = true }
+			o.parse!(argv)
+
+			opts[:database_url] = argv.shift
+			opts[:login] = argv.shift
+			opts[:password] = argv.shift
+
+			if opts[:database_url].nil?
+				$stderr.puts "Missing Database URL"
+				puts o
+				exit 1
+			end
+			if opts[:login].nil?
+				$stderr.puts "Missing Login"
+				puts o
+				exit 1
+			end
+			if opts[:password].nil?
+				$stderr.puts "Missing Password"
+				puts o
+				exit 1
+			end
+		end
+		opts
+	end
+
+	def clientoptparse(cmd)
+		opts={:default_chunksize => 1000, :database_url => nil, :remote_url => nil, :debug => false, :resume_filename => nil, :disable_compresion => false, :indexes_first => false}
+		OptionParser.new do |o|
+			o.banner = "Usage: #{File.basename($0)} #{cmd} [OPTIONS] <local_database_url> <remote_url>"
+
+			case cmd
+			when :pull
+				o.define_head "Pull a database from a taps server"
+			when :push
+				o.define_head "Push a database to a taps server"
+			end
+
+			o.on("-i", "--indexes-first", "Transfer indexes first before data") { |v| opts[:indexes_first] = true }
+			o.on("-r", "--resume=file", "Resume a Taps Session from a stored file") { |v| opts[:resume_filename] = v }
+			o.on("-c", "--chunksize=N", "Initial Chunksize") { |v| opts[:default_chunksize] = (v.to_i < 10 ? 10 : v.to_i) }
+			o.on("-g", "--disable-compression", "Disable Compression") { |v| opts[:disable_compression] = true }
+			o.on("-f", "--filter=regex", "Regex Filter for tables") { |v| opts[:table_filter] = v }
+			o.on("-t", "--tables=A,B,C", Array, "Shortcut to filter on a list of tables") do |v|
+				r_tables = v.collect { |t| "^#{t}$" }.join("|")
+				opts[:table_filter] = "(#{r_tables})"
+			end
+			o.on("-d", "--debug", "Enable Debug Messages") { |v| opts[:debug] = true }
+			o.parse!(argv)
+
+			opts[:database_url] = argv.shift
+			opts[:remote_url] = argv.shift
+
+			if opts[:database_url].nil?
+				$stderr.puts "Missing Database URL"
+				puts o
+				exit 1
+			end
+			if opts[:remote_url].nil?
+				$stderr.puts "Missing Remote Taps URL"
+				puts o
+				exit 1
+			end
+		end
+
+		opts
+	end
+
+	def clientxfer(method, opts)
+		database_url = opts.delete(:database_url)
+		remote_url = opts.delete(:remote_url)
+
+		Taps::Config.verify_database_url(database_url)
+
+		require 'taps/operation'
+
+		Taps::Operation.factory(method, database_url, remote_url, opts).run
+	end
+
+	def clientresumexfer(method, opts)
+		session = JSON.parse(File.read(opts.delete(:resume_filename)))
+		session.symbolize_recursively!
+
+		database_url = opts.delete(:database_url)
+		remote_url = opts.delete(:remote_url) || session.delete(:remote_url)
+
+		Taps::Config.verify_database_url(database_url)
+
+		require 'taps/operation'
+
+		newsession = session.merge({
+			:default_chunksize => opts[:default_chunksize],
+			:disable_compression => opts[:disable_compression],
+			:resume => true,
+		})
+
+		Taps::Operation.factory(method, database_url, remote_url, newsession).run
+	end
+
+end
+end

data/lib/taps/config.rb

@@ -0,0 +1,47 @@
+require 'sequel'
+require 'sqlite3'
+require 'yaml'
+
+Sequel.datetime_class = DateTime
+
+module Taps
+	def self.version_yml
+		@@version_yml ||= YAML.load(File.read(File.dirname(__FILE__) + '/../../VERSION.yml'))
+	end
+
+	def self.version
+		version = "#{version_yml[:major]}.#{version_yml[:minor]}.#{version_yml[:patch]}"
+		version += ".#{version_yml[:build]}" if version_yml[:build]
+		version
+	end
+
+	def self.compatible_version
+		"#{version_yml[:major]}.#{version_yml[:minor]}"
+	end
+
+	def self.exiting=(val)
+		@@exiting = val
+	end
+
+	def exiting?
+		(@@exiting ||= false) == true
+	end
+
+	class Config
+		class << self
+			attr_accessor :taps_database_url
+			attr_accessor :login, :password, :database_url, :remote_url
+			attr_accessor :chunksize
+
+			def verify_database_url(db_url=nil)
+				db_url ||= self.database_url
+				db = Sequel.connect(db_url)
+				db.tables
+				db.disconnect
+			rescue Object => e
+				puts "Failed to connect to database:\n  #{e.class} -> #{e}"
+				exit 1
+			end
+		end
+	end
+end