taps-jruby 0.3.14

data/lib/taps/utils.rb

@@ -0,0 +1,199 @@
+require 'zlib'
+require 'stringio'
+require 'time'
+require 'tempfile'
+
+require 'taps/errors'
+
+module Taps
+module Utils
+  extend self
+
+  def windows?
+    return @windows if defined?(@windows)
+    require 'rbconfig'
+    @windows = !!(::Config::CONFIG['host_os'] =~ /mswin|mingw/)
+  end
+
+  def bin(cmd)
+    cmd = "#{cmd}.cmd" if windows?
+    cmd
+  end
+
+  def checksum(data)
+    Zlib.crc32(data)
+  end
+
+  def valid_data?(data, crc32)
+    Zlib.crc32(data) == crc32.to_i
+  end
+
+  def base64encode(data)
+    [data].pack("m")
+  end
+
+  def base64decode(data)
+    data.unpack("m").first
+  end
+
+  def format_data(data, opts={})
+    return {} if data.size == 0
+    string_columns = opts[:string_columns] || []
+    schema = opts[:schema] || []
+    table  = opts[:table]
+
+    max_lengths = schema.inject({}) do |hash, (column, meta)|
+      if meta[:db_type] =~ /^\w+\((\d+)\)/
+        hash.update(column => $1.to_i)
+      end
+      hash
+    end
+
+    header = data[0].keys
+    only_data = data.collect do |row|
+      row = blobs_to_string(row, string_columns)
+      row.each do |column, data|
+        if data.to_s.length > (max_lengths[column] || data.to_s.length)
+          raise Taps::InvalidData.new(<<-ERROR)
+Detected data that exceeds the length limitation of its column. This is
+generally due to the fact that SQLite does not enforce length restrictions.
+
+Table  : #{table}
+Column : #{column}
+Type   : #{schema.detect{|s| s.first == column}.last[:db_type]}
+Data   : #{data}
+          ERROR
+        end
+      end
+      header.collect { |h| row[h] }
+    end
+    { :header => header, :data => only_data }
+  end
+
+  # mysql text and blobs fields are handled the same way internally
+  # this is not true for other databases so we must check if the field is
+  # actually text and manually convert it back to a string
+  def incorrect_blobs(db, table)
+    return [] if (db.url =~ /mysql:\/\//).nil?
+
+    columns = []
+    db.schema(table).each do |data|
+      column, cdata = data
+      columns << column if cdata[:db_type] =~ /text/
+    end
+    columns
+  end
+
+  def blobs_to_string(row, columns)
+    return row if columns.size == 0
+    columns.each do |c|
+      row[c] = row[c].to_s if row[c].kind_of?(Sequel::SQL::Blob)
+    end
+    row
+  end
+
+  def calculate_chunksize(old_chunksize)
+    chunksize = old_chunksize
+
+    retries = 0
+    time_in_db = 0
+    begin
+      t1 = Time.now
+      time_in_db = yield chunksize
+      time_in_db = time_in_db.to_f rescue 0
+    rescue Errno::EPIPE, RestClient::RequestFailed, RestClient::RequestTimeout
+      retries += 1
+      raise if retries > 2
+
+      # we got disconnected, the chunksize could be too large
+      # on first retry change to 10, on successive retries go down to 1
+      chunksize = (retries == 1) ? 10 : 1
+
+      retry
+    end
+
+    t2 = Time.now
+
+    diff = t2 - t1 - time_in_db
+
+    new_chunksize = if retries > 0
+      chunksize
+    elsif diff > 3.0
+      (chunksize / 3).ceil
+    elsif diff > 1.1
+      chunksize - 100
+    elsif diff < 0.8
+      chunksize * 2
+    else
+      chunksize + 100
+    end
+    new_chunksize = 1 if new_chunksize < 1
+    new_chunksize
+  end
+
+  def load_schema(database_url, schema_data)
+    Tempfile.open('taps') do |tmp|
+      File.open(tmp.path, 'w') { |f| f.write(schema_data) }
+      schema_bin(:load, database_url, tmp.path)
+    end
+  end
+
+  def load_indexes(database_url, index_data)
+    Tempfile.open('taps') do |tmp|
+      File.open(tmp.path, 'w') { |f| f.write(index_data) }
+      schema_bin(:load_indexes, database_url, tmp.path)
+    end
+  end
+
+  def schema_bin(*args)
+    bin_path = File.expand_path("#{File.dirname(__FILE__)}/../../bin/#{bin('schema')}")
+    `"#{bin_path}" #{args.map { |a| "'#{a}'" }.join(' ')}`
+  end
+
+  def primary_key(db, table)
+    db.schema(table).select { |c| c[1][:primary_key] }.map { |c| c[0] }
+  end
+
+  def single_integer_primary_key(db, table)
+    table = table.to_sym.identifier unless table.kind_of?(Sequel::SQL::Identifier)
+    keys = db.schema(table).select { |c| c[1][:primary_key] and c[1][:type] == :integer }
+    not keys.nil? and keys.size == 1
+  end
+
+  def order_by(db, table)
+    pkey = primary_key(db, table)
+    if pkey
+      pkey.kind_of?(Array) ? pkey : [pkey.to_sym]
+    else
+      table = table.to_sym.identifier unless table.kind_of?(Sequel::SQL::Identifier)
+      db[table].columns
+    end
+  end
+
+
+  # try to detect server side errors to
+  # give the client a more useful error message
+  def server_error_handling(&blk)
+    begin
+      blk.call
+    rescue Sequel::DatabaseError => e
+      if e.message =~ /duplicate key value/i
+        raise Taps::DuplicatePrimaryKeyError, e.message
+      else
+        raise
+      end
+    end
+  end
+
+  def reraise_server_exception(e)
+    if e.kind_of?(RestClient::Exception)
+      if e.respond_to?(:response) && e.response.headers[:content_type] == 'application/json'
+        json = JSON.parse(e.response.to_s)
+        klass = eval(json['error_class']) rescue nil
+        raise klass.new(json['error_message'], :backtrace => json['error_backtrace']) if klass
+      end
+    end
+    raise e
+  end
+end
+end

data/spec/base.rb

@@ -0,0 +1,26 @@
+require 'rubygems'
+require 'bacon'
+require 'mocha'
+require 'rack/test'
+require 'tempfile'
+
+$:.unshift File.dirname(__FILE__) + "/../lib"
+
+class Bacon::Context
+  include Mocha::Standalone
+  include Rack::Test::Methods
+
+  alias_method :old_it, :it
+  def it(description)
+    old_it(description) do
+      mocha_setup
+      yield
+      mocha_verify
+      mocha_teardown
+    end
+  end
+end
+
+require 'taps/config'
+Taps::Config.taps_database_url = "jdbc:sqlite://#{Tempfile.new('test.db').path}"
+Sequel.connect(Taps::Config.taps_database_url)

data/spec/cli_spec.rb

@@ -0,0 +1,10 @@
+require File.dirname(__FILE__) + '/base'
+require 'taps/cli'
+
+describe Taps::Cli do
+  it "translates a list of tables into a regex that can be used in table_filter" do
+    @cli = Taps::Cli.new(["-t", "mytable1,logs", "jdbc:sqlite://tmp.db", "http://x:y@localhost:5000"])
+    opts = @cli.clientoptparse(:pull)
+    opts[:table_filter].should == "(^mytable1$|^logs$)"
+  end
+end

data/spec/data_stream_spec.rb

@@ -0,0 +1,23 @@
+require File.dirname(__FILE__) + '/base'
+require 'taps/data_stream'
+
+describe Taps::DataStream do
+  before do
+    @db = mock('db')
+  end
+
+  it "increments the offset" do
+    stream = Taps::DataStream.new(@db, :table_name => 'test_table', :chunksize => 100)
+    stream.state[:offset].should == 0
+    stream.increment(100)
+    stream.state[:offset].should == 100
+  end
+
+  it "marks the stream complete if no rows are fetched" do
+    stream = Taps::DataStream.new(@db, :table_name => 'test_table', :chunksize => 100)
+    stream.stubs(:fetch_rows).returns({})
+    stream.complete?.should.be.false
+    stream.fetch
+    stream.complete?.should.be.true
+  end
+end

data/spec/operation_spec.rb

@@ -0,0 +1,32 @@
+require File.dirname(__FILE__) + '/base'
+require 'taps/operation'
+
+describe Taps::Operation do
+  before do
+    @op = Taps::Operation.new('dummy://localhost', 'http://x:y@localhost:5000')
+  end
+
+  it "returns an array of tables that match the regex table_filter" do
+    @op = Taps::Operation.new('dummy://localhost', 'http://x:y@localhost:5000', :table_filter => 'abc')
+    @op.apply_table_filter(['abc', 'def']).should == ['abc']
+  end
+
+  it "returns a hash of tables that match the regex table_filter" do
+    @op = Taps::Operation.new('dummy://localhost', 'http://x:y@localhost:5000', :table_filter => 'abc')
+    @op.apply_table_filter({ 'abc' => 1, 'def' => 2 }).should == { 'abc' => 1 }
+  end
+
+  it "masks a url's password" do
+    @op.safe_url("mysql://root:password@localhost/mydb").should == "mysql://root:[hidden]@localhost/mydb"
+  end
+
+  it "returns http headers with compression enabled" do
+    @op.http_headers.should == { :taps_version => Taps.version, :accept_encoding => "gzip, deflate" }
+  end
+
+  it "returns http headers with compression disabled" do
+    @op.stubs(:compression_disabled?).returns(true)
+    @op.http_headers.should == { :taps_version => Taps.version, :accept_encoding => "" }
+  end
+
+end

data/spec/server_spec.rb

@@ -0,0 +1,35 @@
+require File.dirname(__FILE__) + '/base'
+
+require 'taps/server'
+
+require 'pp'
+
+describe Taps::Server do
+  def app
+    Taps::Server.new
+  end
+
+  before do
+    Taps::Config.login = 'taps'
+    Taps::Config.password = 'tpass'
+
+    @app = Taps::Server
+    @auth_header = "Basic " + ["taps:tpass"].pack("m*")
+  end
+
+  it "asks for http basic authentication" do
+    get '/'
+    last_response.status.should == 401
+  end
+
+  it "verifies the client taps version" do
+    get('/', { }, { 'HTTP_AUTHORIZATION' => @auth_header, 'HTTP_TAPS_VERSION' => Taps.version })
+    last_response.status.should == 200
+  end
+
+  it "yells loudly if the client taps version doesn't match" do
+    get('/', { }, { 'HTTP_AUTHORIZATION' => @auth_header, 'HTTP_TAPS_VERSION' => '0.0.1' })
+    last_response.status.should == 417
+  end
+end
+

data/spec/utils_spec.rb

@@ -0,0 +1,61 @@
+require File.dirname(__FILE__) + '/base'
+require 'taps/utils'
+
+describe Taps::Utils do
+  it "generates a checksum using crc32" do
+    Taps::Utils.checksum("hello world").should == Zlib.crc32("hello world")
+  end
+
+  it "formats a data hash into one hash that contains an array of headers and an array of array of data" do
+    first_row = { :x => 1, :y => 1 }
+    first_row.stubs(:keys).returns([:x, :y])
+    Taps::Utils.format_data([ first_row, { :x => 2, :y => 2 } ]).should == { :header => [ :x, :y ], :data => [ [1, 1], [2, 2] ] }
+  end
+
+  it "enforces length limitations on columns" do
+    data = [ { :a => "aaabbbccc" } ]
+    schema = [ [ :a, { :db_type => "varchar(3)" }]]
+    lambda { Taps::Utils.format_data(data, :schema => schema) }.should.raise(Taps::InvalidData)
+  end
+
+  it "scales chunksize down slowly when the time delta of the block is just over a second" do
+    Time.stubs(:now).returns(10.0).returns(11.5)
+    Taps::Utils.calculate_chunksize(1000) { }.should == 900
+  end
+
+  it "scales chunksize down fast when the time delta of the block is over 3 seconds" do
+    Time.stubs(:now).returns(10.0).returns(15.0)
+    Taps::Utils.calculate_chunksize(3000) { }.should == 1000
+  end
+
+  it "scales up chunksize fast when the time delta of the block is under 0.8 seconds" do
+    Time.stubs(:now).returns(10.0).returns(10.7)
+    Taps::Utils.calculate_chunksize(1000) { }.should == 2000
+  end
+
+  it "scales up chunksize slow when the time delta of the block is between 0.8 and 1.1 seconds" do
+    Time.stubs(:now).returns(10.0).returns(10.8)
+    Taps::Utils.calculate_chunksize(1000) { }.should == 1100
+
+    Time.stubs(:now).returns(10.0).returns(11.1)
+    Taps::Utils.calculate_chunksize(1000) { }.should == 1100
+  end
+
+  it "will reset the chunksize to a small value if we got a broken pipe exception" do
+    Taps::Utils.calculate_chunksize(1000) { |c| raise Errno::EPIPE if c == 1000; c.should == 10 }.should == 10
+  end
+
+  it "will reset the chunksize to a small value if we got a broken pipe exception a second time" do
+    Taps::Utils.calculate_chunksize(1000) { |c| raise Errno::EPIPE if c == 1000 || c == 10; c.should == 1 }.should == 1
+  end
+
+  it "returns a list of columns that are text fields if the database is mysql" do
+    @db = mock("db", :url => "mysql://localhost/mydb")
+    @db.stubs(:schema).with(:mytable).returns([
+      [:id, { :db_type => "int" }],
+      [:mytext, { :db_type => "text" }]
+    ])
+    Taps::Utils.incorrect_blobs(@db, :mytable).should == [:mytext]
+  end
+end
+

metadata

@@ -0,0 +1,194 @@
+--- !ruby/object:Gem::Specification 
+name: taps-jruby
+version: !ruby/object:Gem::Version 
+  prerelease: false
+  segments: 
+    - 0
+    - 3
+    - 14
+  version: 0.3.14
+platform: ruby
+authors: 
+  - Ricardo Chimal, Jr.
+  - Rob Heittman
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2010-11-19 00:00:00 -05:00
+default_executable: 
+dependencies: 
+  - !ruby/object:Gem::Dependency 
+    name: json_pure
+    prerelease: false
+    requirement: &id001 !ruby/object:Gem::Requirement 
+      requirements: 
+        - - ">="
+          - !ruby/object:Gem::Version 
+            segments: 
+              - 1
+              - 2
+              - 0
+            version: 1.2.0
+        - - <
+          - !ruby/object:Gem::Version 
+            segments: 
+              - 1
+              - 5
+              - 0
+            version: 1.5.0
+    type: :runtime
+    version_requirements: *id001
+  - !ruby/object:Gem::Dependency 
+    name: sinatra
+    prerelease: false
+    requirement: &id002 !ruby/object:Gem::Requirement 
+      requirements: 
+        - - ~>
+          - !ruby/object:Gem::Version 
+            segments: 
+              - 1
+              - 0
+              - 0
+            version: 1.0.0
+    type: :runtime
+    version_requirements: *id002
+  - !ruby/object:Gem::Dependency 
+    name: rest-client
+    prerelease: false
+    requirement: &id003 !ruby/object:Gem::Requirement 
+      requirements: 
+        - - ">="
+          - !ruby/object:Gem::Version 
+            segments: 
+              - 1
+              - 4
+              - 0
+            version: 1.4.0
+        - - <
+          - !ruby/object:Gem::Version 
+            segments: 
+              - 1
+              - 7
+              - 0
+            version: 1.7.0
+    type: :runtime
+    version_requirements: *id003
+  - !ruby/object:Gem::Dependency 
+    name: sequel
+    prerelease: false
+    requirement: &id004 !ruby/object:Gem::Requirement 
+      requirements: 
+        - - ~>
+          - !ruby/object:Gem::Version 
+            segments: 
+              - 3
+              - 17
+              - 0
+            version: 3.17.0
+    type: :runtime
+    version_requirements: *id004
+  - !ruby/object:Gem::Dependency 
+    name: activerecord-jdbcsqlite3-adapter
+    prerelease: false
+    requirement: &id005 !ruby/object:Gem::Requirement 
+      requirements: 
+        - - ">="
+          - !ruby/object:Gem::Version 
+            segments: 
+              - 1
+              - 0
+              - 2
+            version: 1.0.2
+    type: :runtime
+    version_requirements: *id005
+  - !ruby/object:Gem::Dependency 
+    name: rack
+    prerelease: false
+    requirement: &id006 !ruby/object:Gem::Requirement 
+      requirements: 
+        - - ">="
+          - !ruby/object:Gem::Version 
+            segments: 
+              - 1
+              - 0
+              - 1
+            version: 1.0.1
+    type: :runtime
+    version_requirements: *id006
+description: A simple database agnostic import/export app to transfer data to/from a remote database. (JRuby version)
+email: heittman.rob@gmail.com
+executables: 
+  - taps
+  - schema
+extensions: []
+
+extra_rdoc_files: 
+  - LICENSE
+  - README.rdoc
+  - TODO
+files: 
+  - LICENSE
+  - README.rdoc
+  - Rakefile
+  - VERSION.yml
+  - bin/schema
+  - bin/schema.cmd
+  - bin/taps
+  - lib/taps/cli.rb
+  - lib/taps/config.rb
+  - lib/taps/data_stream.rb
+  - lib/taps/db_session.rb
+  - lib/taps/errors.rb
+  - lib/taps/log.rb
+  - lib/taps/monkey.rb
+  - lib/taps/multipart.rb
+  - lib/taps/operation.rb
+  - lib/taps/progress_bar.rb
+  - lib/taps/schema.rb
+  - lib/taps/server.rb
+  - lib/taps/utils.rb
+  - spec/base.rb
+  - spec/cli_spec.rb
+  - spec/data_stream_spec.rb
+  - spec/operation_spec.rb
+  - spec/server_spec.rb
+  - spec/utils_spec.rb
+  - TODO
+has_rdoc: true
+homepage: http://github.com/rfc2616/taps-jruby
+licenses: []
+
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+  - lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        segments: 
+          - 0
+        version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        segments: 
+          - 0
+        version: "0"
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.3.6
+signing_key: 
+specification_version: 3
+summary: simple database import/export app - jruby version
+test_files: 
+  - spec/base.rb
+  - spec/cli_spec.rb
+  - spec/data_stream_spec.rb
+  - spec/operation_spec.rb
+  - spec/server_spec.rb
+  - spec/utils_spec.rb