taps-taps 0.3.24

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 1bc580dc61808bdb17b0de45c3fa27ce6edc445d
+  data.tar.gz: d3b1faefee6bf8828d20f71c73e809f54ab5ef8d
+SHA512:
+  metadata.gz: 11057870f0e66186550ec1aceae822a7f3f2907417b134215e44cad8b7defd4fb82bf4c5f7d2721670600e525246cbf5770d24da515e513f3e308f7f39c4d6b8
+  data.tar.gz: bf6c27c267e777ae54f97d51106dfd7d8ee0fcdf8e6d1fc402f9ef50513e418f0d6a6775d24337f18def51b6471a0b61f66da7fa8243b0e31a86bf8e5b3b4729

data/README.rdoc

@@ -0,0 +1,51 @@
+= Taps -- simple database import/export app
+
+A simple database agnostic import/export app to transfer data to/from a remote database.
+
+== Usage: Server
+
+Here's how you start a taps server
+
+  $ taps server postgres://localdbuser:localdbpass@localhost/dbname httpuser httppassword
+
+You can also specify an encoding in the database url
+
+  $ taps server mysql://localdbuser:localdbpass@localhost/dbname?encoding=latin1 httpuser httppassword
+
+== Usage: Client
+
+When you want to pull down a database from a taps server
+
+  $ taps pull postgres://dbuser:dbpassword@localhost/dbname http://httpuser:httppassword@example.com:5000
+
+or when you want to push a local database to a taps server
+
+  $ taps push postgres://dbuser:dbpassword@localhost/dbname http://httpuser:httppassword@example.com:5000
+
+or when you want to transfer a list of tables
+
+  $ taps push postgres://dbuser:dbpassword@localhost/dbname http://httpuser:httppassword@example.com:5000 --tables logs,tags
+
+or when you want to transfer tables that start with a word
+
+  $ taps push postgres://dbuser:dbpassword@localhost/dbname http://httpuser:httppassword@example.com:5000 --filter '^log_'
+
+== Known Issues
+
+* Foreign key constraints get lost in the schema transfer
+* Tables without primary keys will be incredibly slow to transfer. This is due to it being inefficient having large offset values in queries.
+* Multiple schemas are currently not supported
+
+== Meta
+
+Maintained by Ricardo Chimal, Jr. (ricardo at heroku dot com)
+
+Written by Ricardo Chimal, Jr. (ricardo at heroku dot com) and Adam Wiggins (adam at heroku dot com)
+
+Early research and inspiration by Blake Mizerany
+
+Released under the MIT License: http://www.opensource.org/licenses/mit-license.php
+
+http://github.com/ricardochimal/taps
+
+Special Thanks to Sequel for making this tool possible http://sequel.rubyforge.org/

data/VERSION.yml

@@ -0,0 +1,5 @@
+---
+:build:
+:major: 0
+:minor: 3
+:patch: 24

data/bin/schema

@@ -0,0 +1,54 @@
+#!/usr/bin/env ruby
+
+require 'rubygems'
+gem 'sequel', '~> 3.20.0'
+
+$:.unshift File.dirname(__FILE__) + '/../lib'
+
+require 'taps/schema'
+
+cmd = ARGV.shift.strip rescue ''
+database_url = ARGV.shift.strip rescue ''
+
+def show_usage_and_exit
+  puts <<EOTXT
+  schema console <database_url>
+  schema dump <database_url>
+  schema dump_table <database_url> <table>
+  schema indexes <database_url>
+  schema indexes_individual <database_url>
+  schema reset_db_sequences <database_url>
+  schema load <database_url> <schema_file>
+  schema load_indexes <database_url> <indexes_file>
+EOTXT
+  exit(1)
+end
+
+case cmd
+when 'dump'
+  puts Taps::Schema.dump(database_url)
+when 'dump_table'
+  table = ARGV.shift.strip
+  puts Taps::Schema.dump_table(database_url, table)
+when 'indexes'
+  puts Taps::Schema.indexes(database_url)
+when 'indexes_individual'
+  puts Taps::Schema.indexes_individual(database_url)
+when 'load_indexes'
+  filename = ARGV.shift.strip rescue ''
+  indexes = File.read(filename) rescue show_usage_and_exit
+  Taps::Schema.load_indexes(database_url, indexes)
+when 'load'
+  filename = ARGV.shift.strip rescue ''
+  schema = File.read(filename) rescue show_usage_and_exit
+  Taps::Schema.load(database_url, schema)
+when 'reset_db_sequences'
+  Taps::Schema.reset_db_sequences(database_url)
+when 'console'
+  $db = Sequel.connect(database_url)
+  require 'irb'
+  require 'irb/completion'
+  IRB.start
+else
+  show_usage_and_exit
+end

data/bin/schema.cmd

@@ -0,0 +1,6 @@
+@ECHO OFF
+IF NOT "%~f0" == "~f0" GOTO :WinNT
+@"ruby.exe" "./schema" %1 %2 %3 %4 %5 %6 %7 %8 %9
+GOTO :EOF
+:WinNT
+@"ruby.exe" "%~dpn0" %*

data/bin/taps

@@ -0,0 +1,6 @@
+#!/usr/bin/env ruby
+
+$:.unshift File.dirname(__FILE__) + '/../lib'
+require 'taps/cli'
+
+Taps::Cli.new(ARGV.dup).run

data/lib/taps/chunksize.rb

@@ -0,0 +1,52 @@
+require 'taps/errors'
+
+class Taps::Chunksize
+  attr_accessor :idle_secs, :time_in_db, :start_time, :end_time, :retries
+  attr_reader :chunksize
+
+  def initialize(chunksize)
+    @chunksize = chunksize
+    @idle_secs = 0.0
+    @retries = 0
+  end
+
+  def to_i
+    chunksize
+  end
+
+  def reset_chunksize
+    @chunksize = (retries <= 1) ? 10 : 1
+  end
+
+  def diff
+    end_time - start_time - time_in_db - idle_secs
+  end
+
+  def time_in_db=(t)
+    @time_in_db = t
+    @time_in_db = @time_in_db.to_f rescue 0.0
+  end
+
+  def time_delta
+    t1 = Time.now
+    yield if block_given?
+    t2 = Time.now
+    t2 - t1
+  end
+
+  def calc_new_chunksize
+    new_chunksize = if retries > 0
+      chunksize
+    elsif diff > 3.0
+      (chunksize / 3).ceil
+    elsif diff > 1.1
+      chunksize - 100
+    elsif diff < 0.8
+      chunksize * 2
+    else
+      chunksize + 100
+    end
+    new_chunksize = 1 if new_chunksize < 1
+    new_chunksize
+  end
+end

data/lib/taps/cli.rb

@@ -0,0 +1,196 @@
+require 'optparse'
+require 'tempfile'
+require 'taps/monkey'
+require 'taps/config'
+require 'taps/log'
+require 'vendor/okjson'
+
+Taps::Config.taps_database_url = ENV['TAPS_DATABASE_URL'] || begin
+  # this is dirty but it solves a weird problem where the tempfile disappears mid-process
+  require 'sqlite3'
+  $__taps_database = Tempfile.new('taps.db')
+  $__taps_database.open()
+  "sqlite://#{$__taps_database.path}"
+end
+
+module Taps
+class Cli
+  attr_accessor :argv
+
+  def initialize(argv)
+    @argv = argv
+  end
+
+  def run
+    method = (argv.shift || 'help').to_sym
+    if [:pull, :push, :server, :version].include? method
+      send(method)
+    else
+      help
+    end
+  end
+
+  def pull
+    opts = clientoptparse(:pull)
+    Taps.log.level = Logger::DEBUG if opts[:debug]
+    if opts[:resume_filename]
+      clientresumexfer(:pull, opts)
+    else
+      clientxfer(:pull, opts)
+    end
+  end
+
+  def push
+    opts = clientoptparse(:push)
+    Taps.log.level = Logger::DEBUG if opts[:debug]
+    if opts[:resume_filename]
+      clientresumexfer(:push, opts)
+    else
+      clientxfer(:push, opts)
+    end
+  end
+
+  def server
+    opts = serveroptparse
+    Taps.log.level = Logger::DEBUG if opts[:debug]
+    Taps::Config.database_url = opts[:database_url]
+    Taps::Config.login = opts[:login]
+    Taps::Config.password = opts[:password]
+
+    Taps::Config.verify_database_url
+    require 'taps/server'
+    Taps::Server.run!({
+      :port => opts[:port],
+      :environment => :production,
+      :logging => true,
+      :dump_errors => true,
+    })
+  end
+
+  def version
+    puts Taps.version
+  end
+
+  def help
+    puts <<EOHELP
+Options
+=======
+server    Start a taps database import/export server
+pull      Pull a database from a taps server
+push      Push a database to a taps server
+version   Taps version
+
+Add '-h' to any command to see their usage
+EOHELP
+  end
+
+  def serveroptparse
+    opts={:port => 5000, :database_url => nil, :login => nil, :password => nil, :debug => false}
+    OptionParser.new do |o|
+      o.banner = "Usage: #{File.basename($0)} server [OPTIONS] <local_database_url> <login> <password>"
+      o.define_head "Start a taps database import/export server"
+
+      o.on("-p", "--port=N", "Server Port") { |v| opts[:port] = v.to_i if v.to_i > 0 }
+      o.on("-d", "--debug", "Enable Debug Messages") { |v| opts[:debug] = true }
+      o.parse!(argv)
+
+      opts[:database_url] = argv.shift
+      opts[:login] = argv.shift
+      opts[:password] = argv.shift
+
+      if opts[:database_url].nil?
+        $stderr.puts "Missing Database URL"
+        puts o
+        exit 1
+      end
+      if opts[:login].nil?
+        $stderr.puts "Missing Login"
+        puts o
+        exit 1
+      end
+      if opts[:password].nil?
+        $stderr.puts "Missing Password"
+        puts o
+        exit 1
+      end
+    end
+    opts
+  end
+
+  def clientoptparse(cmd)
+    opts={:default_chunksize => 1000, :database_url => nil, :remote_url => nil, :debug => false, :resume_filename => nil, :disable_compresion => false, :indexes_first => false}
+    OptionParser.new do |o|
+      o.banner = "Usage: #{File.basename($0)} #{cmd} [OPTIONS] <local_database_url> <remote_url>"
+
+      case cmd
+      when :pull
+        o.define_head "Pull a database from a taps server"
+      when :push
+        o.define_head "Push a database to a taps server"
+      end
+
+      o.on("-s", "--skip-schema", "Don't transfer the schema, just data") { |v| opts[:skip_schema] = true }
+      o.on("-i", "--indexes-first", "Transfer indexes first before data") { |v| opts[:indexes_first] = true }
+      o.on("-r", "--resume=file", "Resume a Taps Session from a stored file") { |v| opts[:resume_filename] = v }
+      o.on("-c", "--chunksize=N", "Initial Chunksize") { |v| opts[:default_chunksize] = (v.to_i < 10 ? 10 : v.to_i) }
+      o.on("-g", "--disable-compression", "Disable Compression") { |v| opts[:disable_compression] = true }
+      o.on("-f", "--filter=regex", "Regex Filter for tables") { |v| opts[:table_filter] = v }
+      o.on("-t", "--tables=A,B,C", Array, "Shortcut to filter on a list of tables") do |v|
+        r_tables = v.collect { |t| "^#{t}$" }.join("|")
+        opts[:table_filter] = "(#{r_tables})"
+      end
+      o.on("-e", "--exclude_tables=A,B,C", Array, "Shortcut to exclude a list of tables") { |v| opts[:exclude_tables] = v }
+      o.on("-d", "--debug", "Enable Debug Messages") { |v| opts[:debug] = true }
+      o.parse!(argv)
+
+      opts[:database_url] = argv.shift
+      opts[:remote_url] = argv.shift
+
+      if opts[:database_url].nil?
+        $stderr.puts "Missing Database URL"
+        puts o
+        exit 1
+      end
+      if opts[:remote_url].nil?
+        $stderr.puts "Missing Remote Taps URL"
+        puts o
+        exit 1
+      end
+    end
+
+    opts
+  end
+
+  def clientxfer(method, opts)
+    database_url = opts.delete(:database_url)
+    remote_url = opts.delete(:remote_url)
+
+    Taps::Config.verify_database_url(database_url)
+
+    require 'taps/operation'
+
+    Taps::Operation.factory(method, database_url, remote_url, opts).run
+  end
+
+  def clientresumexfer(method, opts)
+    session = OkJson.decode(File.read(opts.delete(:resume_filename)))
+    session.symbolize_recursively!
+
+    database_url = opts.delete(:database_url)
+    remote_url = opts.delete(:remote_url) || session.delete(:remote_url)
+
+    Taps::Config.verify_database_url(database_url)
+
+    require 'taps/operation'
+
+    newsession = session.merge({
+      :default_chunksize => opts[:default_chunksize],
+      :disable_compression => opts[:disable_compression],
+      :resume => true,
+    })
+
+    Taps::Operation.factory(method, database_url, remote_url, newsession).run
+  end
+
+end
+end

data/lib/taps/config.rb

@@ -0,0 +1,32 @@
+require 'sequel'
+require 'taps/version'
+
+Sequel.datetime_class = DateTime
+
+module Taps
+  def self.exiting=(val)
+    @@exiting = val
+  end
+
+  def exiting?
+    (@@exiting ||= false) == true
+  end
+
+  class Config
+    class << self
+      attr_accessor :taps_database_url
+      attr_accessor :login, :password, :database_url, :remote_url
+      attr_accessor :chunksize
+
+      def verify_database_url(db_url=nil)
+        db_url ||= self.database_url
+        db = Sequel.connect(db_url)
+        db.tables
+        db.disconnect
+      rescue Object => e
+        puts "Failed to connect to database:\n  #{e.class} -> #{e}"
+        exit 1
+      end
+    end
+  end
+end

data/lib/taps/data_stream.rb

@@ -0,0 +1,343 @@
+require 'taps/monkey'
+require 'taps/multipart'
+require 'taps/utils'
+require 'taps/log'
+require 'taps/errors'
+require 'vendor/okjson'
+
+module Taps
+
+class DataStream
+  DEFAULT_CHUNKSIZE = 1000
+
+  attr_reader :db, :state
+
+  def initialize(db, state)
+    @db = db
+    @state = {
+      :offset => 0,
+      :avg_chunksize => 0,
+      :num_chunksize => 0,
+      :total_chunksize => 0,
+    }.merge(state)
+    @state[:chunksize] ||= DEFAULT_CHUNKSIZE
+    @complete = false
+  end
+
+  def log
+    Taps.log
+  end
+
+  def error=(val)
+    state[:error] = val
+  end
+
+  def error
+    state[:error] || false
+  end
+
+  def table_name
+    state[:table_name].to_sym
+  end
+
+  def table_name_sql
+    table_name.identifier
+  end
+
+  def to_hash
+    state.merge(:klass => self.class.to_s)
+  end
+
+  def to_json
+    OkJson.encode(to_hash)
+  end
+
+  def string_columns
+    @string_columns ||= Taps::Utils.incorrect_blobs(db, table_name)
+  end
+
+  def table
+    @table ||= db[table_name_sql]
+  end
+
+  def order_by(name=nil)
+    @order_by ||= begin
+      name ||= table_name
+      Taps::Utils.order_by(db, name)
+    end
+  end
+
+  def increment(row_count)
+    state[:offset] += row_count
+  end
+
+  # keep a record of the average chunksize within the first few hundred thousand records, after chunksize
+  # goes below 100 or maybe if offset is > 1000
+  def fetch_rows
+    state[:chunksize] = fetch_chunksize
+    ds = table.order(*order_by).limit(state[:chunksize], state[:offset])
+    log.debug "DataStream#fetch_rows SQL -> #{ds.sql}"
+    rows = Taps::Utils.format_data(ds.all,
+      :string_columns => string_columns,
+      :schema => db.schema(table_name),
+      :table  => table_name
+    )
+    update_chunksize_stats
+    rows
+  end
+
+  def max_chunksize_training
+    20
+  end
+
+  def fetch_chunksize
+    chunksize = state[:chunksize]
+    return chunksize if state[:num_chunksize] < max_chunksize_training
+    return chunksize if state[:avg_chunksize] == 0
+    return chunksize if state[:error]
+    state[:avg_chunksize] > chunksize ? state[:avg_chunksize] : chunksize
+  end
+
+  def update_chunksize_stats
+    return if state[:num_chunksize] >= max_chunksize_training
+    state[:total_chunksize] += state[:chunksize]
+    state[:num_chunksize] += 1
+    state[:avg_chunksize] = state[:total_chunksize] / state[:num_chunksize] rescue state[:chunksize]
+  end
+
+  def encode_rows(rows)
+    Taps::Utils.base64encode(Marshal.dump(rows))
+  end
+
+  def fetch
+    log.debug "DataStream#fetch state -> #{state.inspect}"
+
+    t1 = Time.now
+    rows = fetch_rows
+    encoded_data = encode_rows(rows)
+    t2 = Time.now
+    elapsed_time = t2 - t1
+
+    @complete = rows == { }
+
+    [encoded_data, (@complete ? 0 : rows[:data].size), elapsed_time]
+  end
+
+  def complete?
+    @complete
+  end
+
+  def fetch_remote(resource, headers)
+    params = fetch_from_resource(resource, headers)
+    encoded_data = params[:encoded_data]
+    json = params[:json]
+
+    rows = parse_encoded_data(encoded_data, json[:checksum])
+    @complete = rows == { }
+
+    # update local state
+    state.merge!(json[:state].merge(:chunksize => state[:chunksize]))
+
+    unless @complete
+      import_rows(rows)
+      rows[:data].size
+    else
+      0
+    end
+  end
+
+  # this one is used inside the server process
+  def fetch_remote_in_server(params)
+    json = self.class.parse_json(params[:json])
+    encoded_data = params[:encoded_data]
+
+    rows = parse_encoded_data(encoded_data, json[:checksum])
+    @complete = rows == { }
+
+    unless @complete
+      import_rows(rows)
+      rows[:data].size
+    else
+      0
+    end
+  end
+
+  def fetch_from_resource(resource, headers)
+    res = nil
+    log.debug "DataStream#fetch_from_resource state -> #{state.inspect}"
+    state[:chunksize] = Taps::Utils.calculate_chunksize(state[:chunksize]) do |c|
+      state[:chunksize] = c.to_i
+      res = resource.post({:state => OkJson.encode(self.to_hash)}, headers)
+    end
+
+    begin
+      params = Taps::Multipart.parse(res)
+      params[:json] = self.class.parse_json(params[:json]) if params.has_key?(:json)
+      return params
+    rescue OkJson::Parser
+      raise Taps::CorruptedData.new("Invalid OkJson Received")
+    end
+  end
+
+  def self.parse_json(json)
+    hash = OkJson.decode(json).symbolize_keys
+    hash[:state].symbolize_keys! if hash.has_key?(:state)
+    hash
+  end
+
+  def parse_encoded_data(encoded_data, checksum)
+    raise Taps::CorruptedData.new("Checksum Failed") unless Taps::Utils.valid_data?(encoded_data, checksum)
+
+    begin
+      return Marshal.load(Taps::Utils.base64decode(encoded_data))
+    rescue Object => e
+      unless ENV['NO_DUMP_MARSHAL_ERRORS']
+        puts "Error encountered loading data, wrote the data chunk to dump.#{Process.pid}.dat"
+        File.open("dump.#{Process.pid}.dat", "w") { |f| f.write(encoded_data) }
+      end
+      raise
+    end
+  end
+
+  def import_rows(rows)
+    table.import(rows[:header], rows[:data])
+    state[:offset] += rows[:data].size
+  rescue Exception => ex
+    case ex.message
+    when /integer out of range/ then
+      raise Taps::InvalidData, <<-ERROR, []
+\nDetected integer data that exceeds the maximum allowable size for an integer type.
+This generally occurs when importing from SQLite due to the fact that SQLite does
+not enforce maximum values on integer types.
+      ERROR
+    else raise ex
+    end
+  end
+
+  def verify_stream
+    state[:offset] = table.count
+  end
+
+  def verify_remote_stream(resource, headers)
+    json_raw = resource.post({:state => OkJson.encode(self)}, headers).to_s
+    json = self.class.parse_json(json_raw)
+
+    self.class.new(db, json[:state])
+  end
+
+  def self.factory(db, state)
+    if defined?(Sequel::MySQL) && Sequel::MySQL.respond_to?(:convert_invalid_date_time=)
+      Sequel::MySQL.convert_invalid_date_time = :nil
+    end
+
+    if state.has_key?(:klass)
+      return eval(state[:klass]).new(db, state)
+    end
+
+    if Taps::Utils.single_integer_primary_key(db, state[:table_name].to_sym)
+      DataStreamKeyed.new(db, state)
+    else
+      DataStream.new(db, state)
+    end
+  end
+end
+
+
+class DataStreamKeyed < DataStream
+  attr_accessor :buffer
+
+  def initialize(db, state)
+    super(db, state)
+    @state = { :primary_key => order_by(state[:table_name]).first, :filter => 0 }.merge(state)
+    @state[:chunksize] ||= DEFAULT_CHUNKSIZE
+    @buffer = []
+  end
+
+  def primary_key
+    state[:primary_key].to_sym
+  end
+
+  def buffer_limit
+    if state[:last_fetched] and state[:last_fetched] < state[:filter] and self.buffer.size == 0
+      state[:last_fetched]
+    else
+      state[:filter]
+    end
+  end
+
+  def calc_limit(chunksize)
+    # we want to not fetch more than is needed while we're
+    # inside sinatra but locally we can select more than
+    # is strictly needed
+    if defined?(Sinatra)
+      (chunksize * 1.1).ceil
+    else
+      (chunksize * 3).ceil
+    end
+  end
+
+  def load_buffer(chunksize)
+    # make sure BasicObject is not polluted by subsequent requires
+    Sequel::BasicObject.remove_methods!
+
+    num = 0
+    loop do
+      limit = calc_limit(chunksize)
+      # we have to use local variables in order for the virtual row filter to work correctly
+      key = primary_key
+      buf_limit = buffer_limit
+      ds = table.order(*order_by).filter { key.sql_number > buf_limit }.limit(limit)
+      log.debug "DataStreamKeyed#load_buffer SQL -> #{ds.sql}"
+      data = ds.all
+      self.buffer += data
+      num += data.size
+      if data.size > 0
+        # keep a record of the last primary key value in the buffer
+        state[:filter] = self.buffer.last[ primary_key ]
+      end
+
+      break if num >= chunksize or data.size == 0
+    end
+  end
+
+  def fetch_buffered(chunksize)
+    load_buffer(chunksize) if self.buffer.size < chunksize
+    rows = buffer.slice(0, chunksize)
+    state[:last_fetched] = if rows.size > 0
+      rows.last[ primary_key ]
+    else
+      nil
+    end
+    rows
+  end
+
+  def import_rows(rows)
+    table.import(rows[:header], rows[:data])
+  end
+
+  def fetch_rows
+    chunksize = state[:chunksize]
+    Taps::Utils.format_data(fetch_buffered(chunksize) || [],
+      :string_columns => string_columns)
+  end
+
+  def increment(row_count)
+    # pop the rows we just successfully sent off the buffer
+    @buffer.slice!(0, row_count)
+  end
+
+  def verify_stream
+    key = primary_key
+    ds = table.order(*order_by)
+    current_filter = ds.max(key.sql_number)
+
+    # set the current filter to the max of the primary key
+    state[:filter] = current_filter
+    # clear out the last_fetched value so it can restart from scratch
+    state[:last_fetched] = nil
+
+    log.debug "DataStreamKeyed#verify_stream -> state: #{state.inspect}"
+  end
+end
+
+end