taps-jruby 0.3.14

data/LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 2008 Ricardo Chimal, Jr
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.rdoc

@@ -0,0 +1,51 @@
+= Taps -- simple database import/export app
+
+A simple database agnostic import/export app to transfer data to/from a remote database.
+
+== Usage: Server
+
+Here's how you start a taps server
+
+  $ taps server postgres://localdbuser:localdbpass@localhost/dbname httpuser httppassword
+
+You can also specify an encoding in the database url
+
+  $ taps server mysql://localdbuser:localdbpass@localhost/dbname?encoding=latin1 httpuser httppassword
+
+== Usage: Client
+
+When you want to pull down a database from a taps server
+
+  $ taps pull postgres://dbuser:dbpassword@localhost/dbname http://httpuser:httppassword@example.com:5000
+
+or when you want to push a local database to a taps server
+
+  $ taps push postgres://dbuser:dbpassword@localhost/dbname http://httpuser:httppassword@example.com:5000
+
+or when you want to transfer a list of tables
+
+  $ taps push postgres://dbuser:dbpassword@localhost/dbname http://httpuser:httppassword@example.com:5000 --tables logs,tags
+
+or when you want to transfer tables that start with a word
+
+  $ taps push postgres://dbuser:dbpassword@localhost/dbname http://httpuser:httppassword@example.com:5000 --filter '^log_'
+
+== Known Issues
+
+* Foreign Keys get lost in the schema transfer
+* Tables without primary keys will be incredibly slow to transfer. This is due to it being inefficient having large offset values in queries.
+* Multiple schemas are currently not supported
+
+== Meta
+
+Maintained by Ricardo Chimal, Jr. (ricardo at heroku dot com)
+
+Written by Ricardo Chimal, Jr. (ricardo at heroku dot com) and Adam Wiggins (adam at heroku dot com)
+
+Early research and inspiration by Blake Mizerany
+
+Released under the MIT License: http://www.opensource.org/licenses/mit-license.php
+
+http://github.com/ricardochimal/taps
+
+Special Thanks to Sequel for making this tool possible http://sequel.rubyforge.org/

data/Rakefile

@@ -0,0 +1,70 @@
+begin
+  require 'jeweler'
+  Jeweler::Tasks.new do |s|
+    s.name = "taps-jruby"
+    s.summary = %Q{simple database import/export app - jruby version}
+    s.email = "heittman.rob@gmail.com"
+    s.homepage = "http://github.com/rfc2616/taps-jruby"
+    s.description = "A simple database agnostic import/export app to transfer data to/from a remote database. (JRuby version)"
+    s.authors = ["Ricardo Chimal, Jr.", "Rob Heittman"]
+
+    s.rubygems_version = %q{1.3.5}
+
+    s.add_dependency 'json_pure', '>= 1.2.0', '< 1.5.0'
+    s.add_dependency 'sinatra', '~> 1.0.0'
+    s.add_dependency 'rest-client', '>= 1.4.0', '< 1.7.0'
+    s.add_dependency 'sequel', '~> 3.17.0'
+    s.add_dependency 'activerecord-jdbcsqlite3-adapter', '>= 1.0.2'
+    s.add_dependency 'rack', '>= 1.0.1'
+
+    s.files = FileList['spec/*.rb'] + FileList['lib/**/*.rb'] + ['README.rdoc', 'LICENSE', 'VERSION.yml', 'Rakefile'] + FileList['bin/*']
+    s.executables = ['taps', 'schema']
+  end
+rescue LoadError => e
+  if e.message =~ /jeweler/
+    puts "Jeweler not available. Install it with: sudo gem install technicalpickles-jeweler -s http://gems.github.com"
+  else
+    puts e.message + ' -- while loading jeweler.'
+  end
+end
+
+begin
+  require 'rake/rdoctask'
+  Rake::RDocTask.new do |rdoc|
+    rdoc.rdoc_dir = 'rdoc'
+    rdoc.title = 'taps'
+    rdoc.options << '--line-numbers' << '--inline-source'
+    rdoc.rdoc_files.include('README*')
+    rdoc.rdoc_files.include('lib/**/*.rb')
+  end
+rescue LoadError
+   puts "Rdoc is not available"
+end
+
+begin
+  require 'rcov/rcovtask'
+  Rcov::RcovTask.new do |t|
+    t.libs << 'spec'
+    t.test_files = FileList['spec/*_spec.rb']
+    t.verbose = true
+  end
+rescue LoadError
+  puts "RCov is not available. In order to run rcov, you must: sudo gem install rcov"
+end
+
+desc "Run all specs; requires the bacon gem"
+task :spec do
+  if `which bacon`.empty?
+    puts "bacon is not available. In order to run the specs, you must: sudo gem install bacon."
+  else
+    system "bacon #{File.dirname(__FILE__)}/spec/*_spec.rb"
+  end
+end
+
+desc "copy/paste env vars for dev testing"
+task :env do
+  puts "export RUBYLIB='#{File.dirname(__FILE__) + '/lib'}'"
+  puts "export RUBYOPT='-rrubygems'"
+end
+
+task :default => :spec

data/TODO

@@ -0,0 +1 @@
+* Detect incompatible Marshal versions

data/VERSION.yml

@@ -0,0 +1,5 @@
+--- 
+:build: 
+:minor: 3
+:patch: 14
+:major: 0

data/bin/schema

@@ -0,0 +1,54 @@
+#!/usr/bin/env ruby
+
+require 'rubygems'
+gem 'sequel', '~> 3.17.0'
+
+$:.unshift File.dirname(__FILE__) + '/../lib'
+
+require 'taps/schema'
+
+cmd = ARGV.shift.strip rescue ''
+database_url = ARGV.shift.strip rescue ''
+
+def show_usage_and_exit
+  puts <<EOTXT
+  schema console <database_url>
+  schema dump <database_url>
+  schema dump_table <database_url> <table>
+  schema indexes <database_url>
+  schema indexes_individual <database_url>
+  schema reset_db_sequences <database_url>
+  schema load <database_url> <schema_file>
+  schema load_indexes <database_url> <indexes_file>
+EOTXT
+  exit(1)
+end
+
+case cmd
+when 'dump'
+  puts Taps::Schema.dump(database_url)
+when 'dump_table'
+  table = ARGV.shift.strip
+  puts Taps::Schema.dump_table(database_url, table)
+when 'indexes'
+  puts Taps::Schema.indexes(database_url)
+when 'indexes_individual'
+  puts Taps::Schema.indexes_individual(database_url)
+when 'load_indexes'
+  filename = ARGV.shift.strip rescue ''
+  indexes = File.read(filename) rescue show_usage_and_exit
+  Taps::Schema.load_indexes(database_url, indexes)
+when 'load'
+  filename = ARGV.shift.strip rescue ''
+  schema = File.read(filename) rescue show_usage_and_exit
+  Taps::Schema.load(database_url, schema)
+when 'reset_db_sequences'
+  Taps::Schema.reset_db_sequences(database_url)
+when 'console'
+  $db = Sequel.connect(database_url)
+  require 'irb'
+  require 'irb/completion'
+  IRB.start
+else
+  show_usage_and_exit
+end

data/bin/schema.cmd

@@ -0,0 +1,6 @@
+@ECHO OFF
+IF NOT "%~f0" == "~f0" GOTO :WinNT
+@"ruby.exe" "./schema" %1 %2 %3 %4 %5 %6 %7 %8 %9
+GOTO :EOF
+:WinNT
+@"ruby.exe" "%~dpn0" %*

data/bin/taps

@@ -0,0 +1,6 @@
+#!/usr/bin/env ruby
+
+$:.unshift File.dirname(__FILE__) + '/../lib'
+require 'taps/cli'
+
+Taps::Cli.new(ARGV.dup).run

data/lib/taps/cli.rb

@@ -0,0 +1,193 @@
+require 'optparse'
+require 'tempfile'
+require 'json/pure'
+require 'taps/monkey'
+require 'taps/config'
+require 'taps/log'
+
+Taps::Config.taps_database_url = ENV['TAPS_DATABASE_URL'] || begin
+  # this is dirty but it solves a weird problem where the tempfile disappears mid-process
+  $__taps_database = Tempfile.new('taps.db')
+  $__taps_database.open()
+  "jdbc:sqlite://#{$__taps_database.path}"
+end
+
+module Taps
+class Cli
+  attr_accessor :argv
+
+  def initialize(argv)
+    @argv = argv
+  end
+
+  def run
+    method = (argv.shift || 'help').to_sym
+    if [:pull, :push, :server, :version].include? method
+      send(method)
+    else
+      help
+    end
+  end
+
+  def pull
+    opts = clientoptparse(:pull)
+    Taps.log.level = Logger::DEBUG if opts[:debug]
+    if opts[:resume_filename]
+      clientresumexfer(:pull, opts)
+    else
+      clientxfer(:pull, opts)
+    end
+  end
+
+  def push
+    opts = clientoptparse(:push)
+    Taps.log.level = Logger::DEBUG if opts[:debug]
+    if opts[:resume_filename]
+      clientresumexfer(:push, opts)
+    else
+      clientxfer(:push, opts)
+    end
+  end
+
+  def server
+    opts = serveroptparse
+    Taps.log.level = Logger::DEBUG if opts[:debug]
+    Taps::Config.database_url = opts[:database_url]
+    Taps::Config.login = opts[:login]
+    Taps::Config.password = opts[:password]
+
+    Taps::Config.verify_database_url
+    require 'taps/server'
+    Taps::Server.run!({
+      :port => opts[:port],
+      :environment => :production,
+      :logging => true,
+      :dump_errors => true,
+    })
+  end
+
+  def version
+    puts Taps.version
+  end
+
+  def help
+    puts <<EOHELP
+Options
+=======
+server    Start a taps database import/export server
+pull      Pull a database from a taps server
+push      Push a database to a taps server
+version   Taps version
+
+Add '-h' to any command to see their usage
+EOHELP
+  end
+
+  def serveroptparse
+    opts={:port => 5000, :database_url => nil, :login => nil, :password => nil, :debug => false}
+    OptionParser.new do |o|
+      o.banner = "Usage: #{File.basename($0)} server [OPTIONS] <local_database_url> <login> <password>"
+      o.define_head "Start a taps database import/export server"
+
+      o.on("-p", "--port=N", "Server Port") { |v| opts[:port] = v.to_i if v.to_i > 0 }
+      o.on("-d", "--debug", "Enable Debug Messages") { |v| opts[:debug] = true }
+      o.parse!(argv)
+
+      opts[:database_url] = argv.shift
+      opts[:login] = argv.shift
+      opts[:password] = argv.shift
+
+      if opts[:database_url].nil?
+        $stderr.puts "Missing Database URL"
+        puts o
+        exit 1
+      end
+      if opts[:login].nil?
+        $stderr.puts "Missing Login"
+        puts o
+        exit 1
+      end
+      if opts[:password].nil?
+        $stderr.puts "Missing Password"
+        puts o
+        exit 1
+      end
+    end
+    opts
+  end
+
+  def clientoptparse(cmd)
+    opts={:default_chunksize => 1000, :database_url => nil, :remote_url => nil, :debug => false, :resume_filename => nil, :disable_compresion => false, :indexes_first => false}
+    OptionParser.new do |o|
+      o.banner = "Usage: #{File.basename($0)} #{cmd} [OPTIONS] <local_database_url> <remote_url>"
+
+      case cmd
+      when :pull
+        o.define_head "Pull a database from a taps server"
+      when :push
+        o.define_head "Push a database to a taps server"
+      end
+
+      o.on("-i", "--indexes-first", "Transfer indexes first before data") { |v| opts[:indexes_first] = true }
+      o.on("-r", "--resume=file", "Resume a Taps Session from a stored file") { |v| opts[:resume_filename] = v }
+      o.on("-c", "--chunksize=N", "Initial Chunksize") { |v| opts[:default_chunksize] = (v.to_i < 10 ? 10 : v.to_i) }
+      o.on("-g", "--disable-compression", "Disable Compression") { |v| opts[:disable_compression] = true }
+      o.on("-f", "--filter=regex", "Regex Filter for tables") { |v| opts[:table_filter] = v }
+      o.on("-t", "--tables=A,B,C", Array, "Shortcut to filter on a list of tables") do |v|
+        r_tables = v.collect { |t| "^#{t}$" }.join("|")
+        opts[:table_filter] = "(#{r_tables})"
+      end
+      o.on("-d", "--debug", "Enable Debug Messages") { |v| opts[:debug] = true }
+      o.parse!(argv)
+
+      opts[:database_url] = argv.shift
+      opts[:remote_url] = argv.shift
+
+      if opts[:database_url].nil?
+        $stderr.puts "Missing Database URL"
+        puts o
+        exit 1
+      end
+      if opts[:remote_url].nil?
+        $stderr.puts "Missing Remote Taps URL"
+        puts o
+        exit 1
+      end
+    end
+
+    opts
+  end
+
+  def clientxfer(method, opts)
+    database_url = opts.delete(:database_url)
+    remote_url = opts.delete(:remote_url)
+
+    Taps::Config.verify_database_url(database_url)
+
+    require 'taps/operation'
+
+    Taps::Operation.factory(method, database_url, remote_url, opts).run
+  end
+
+  def clientresumexfer(method, opts)
+    session = JSON.parse(File.read(opts.delete(:resume_filename)))
+    session.symbolize_recursively!
+
+    database_url = opts.delete(:database_url)
+    remote_url = opts.delete(:remote_url) || session.delete(:remote_url)
+
+    Taps::Config.verify_database_url(database_url)
+
+    require 'taps/operation'
+
+    newsession = session.merge({
+      :default_chunksize => opts[:default_chunksize],
+      :disable_compression => opts[:disable_compression],
+      :resume => true,
+    })
+
+    Taps::Operation.factory(method, database_url, remote_url, newsession).run
+  end
+
+end
+end

data/lib/taps/config.rb

@@ -0,0 +1,47 @@
+require 'sequel'
+#require 'sqlite3'
+require 'yaml'
+
+Sequel.datetime_class = DateTime
+
+module Taps
+  def self.version_yml
+    @@version_yml ||= YAML.load(File.read(File.dirname(__FILE__) + '/../../VERSION.yml'))
+  end
+
+  def self.version
+    version = "#{version_yml[:major]}.#{version_yml[:minor]}.#{version_yml[:patch]}"
+    version += ".#{version_yml[:build]}" if version_yml[:build]
+    version
+  end
+
+  def self.compatible_version
+    "#{version_yml[:major]}.#{version_yml[:minor]}"
+  end
+
+  def self.exiting=(val)
+    @@exiting = val
+  end
+
+  def exiting?
+    (@@exiting ||= false) == true
+  end
+
+  class Config
+    class << self
+      attr_accessor :taps_database_url
+      attr_accessor :login, :password, :database_url, :remote_url
+      attr_accessor :chunksize
+
+      def verify_database_url(db_url=nil)
+        db_url ||= self.database_url
+        db = Sequel.connect(db_url)
+        db.tables
+        db.disconnect
+      rescue Object => e
+        puts "Failed to connect to database:\n  #{e.class} -> #{e}"
+        exit 1
+      end
+    end
+  end
+end