taps2 0.5.1

data/spec/base.rb

@@ -0,0 +1,27 @@
+require 'rubygems'
+require 'bacon'
+require 'mocha'
+require 'mocha/api'
+require 'rack/test'
+require 'tempfile'
+
+$:.unshift File.dirname(__FILE__) + "/../lib"
+
+class Bacon::Context
+  include Mocha::API
+  include Rack::Test::Methods
+
+  alias_method :old_it, :it
+  def it(description)
+    old_it(description) do
+      mocha_setup
+      yield
+      mocha_verify
+      mocha_teardown
+    end
+  end
+end
+
+require 'taps/config'
+Taps::Config.taps_database_url = "sqlite://#{Tempfile.new('test.db').path}"
+Sequel.connect(Taps::Config.taps_database_url)

data/spec/chunksize_spec.rb

@@ -0,0 +1,41 @@
+require File.dirname(__FILE__) + '/base'
+require 'taps/utils'
+
+describe Taps::Chunksize do
+  it "scales chunksize down slowly when the time delta of the block is just over a second" do
+    Time.stubs(:now).returns(10.0).returns(11.5)
+    Taps::Utils.calculate_chunksize(1000) { |c| }.should == 900
+  end
+
+  it "scales chunksize down fast when the time delta of the block is over 3 seconds" do
+    Time.stubs(:now).returns(10.0).returns(15.0)
+    Taps::Utils.calculate_chunksize(3000) { |c| }.should == 1000
+  end
+
+  it "scales up chunksize fast when the time delta of the block is under 0.8 seconds" do
+    Time.stubs(:now).returns(10.0).returns(10.7)
+    Taps::Utils.calculate_chunksize(1000) { |c| }.should == 2000
+  end
+
+  it "scales up chunksize slow when the time delta of the block is between 0.8 and 1.1 seconds" do
+    Time.stubs(:now).returns(10.0).returns(10.8)
+    Taps::Utils.calculate_chunksize(1000) { |c| }.should == 1100
+
+    Time.stubs(:now).returns(10.0).returns(11.1)
+    Taps::Utils.calculate_chunksize(1000) { |c| }.should == 1100
+  end
+
+  it "will reset the chunksize to a small value if we got a broken pipe exception" do
+    Taps::Utils.calculate_chunksize(1000) do |c|
+      raise Errno::EPIPE if c.chunksize == 1000
+      c.chunksize.should == 10
+    end.should == 10
+  end
+
+  it "will reset the chunksize to a small value if we got a broken pipe exception a second time" do
+    Taps::Utils.calculate_chunksize(1000) do |c|
+      raise Errno::EPIPE if c.chunksize == 1000 || c.chunksize == 10
+      c.chunksize.should == 1
+    end.should == 1
+  end
+end

data/spec/cli_spec.rb

@@ -0,0 +1,16 @@
+require File.dirname(__FILE__) + '/base'
+require 'taps/cli'
+
+describe Taps::Cli do
+  it "translates a list of tables into a regex that can be used in table_filter" do
+    @cli = Taps::Cli.new(["-t", "mytable1,logs", "sqlite://tmp.db", "http://x:y@localhost:5000"])
+    opts = @cli.clientoptparse(:pull)
+    opts[:table_filter].should == "(^mytable1$|^logs$)"
+  end
+
+  it "translates a list of tables to exclude into a regex that can be used in table_filter" do
+    @cli = Taps::Cli.new(["-e", "mytable1,logs", "sqlite://tmp.db", "http://x:y@localhost:5000"])
+    opts = @cli.clientoptparse(:pull)
+    opts[:exclude_tables].should == ['mytable1','logs']
+  end
+end

data/spec/data_stream_spec.rb

@@ -0,0 +1,23 @@
+require File.dirname(__FILE__) + '/base'
+require 'taps/data_stream'
+
+describe Taps::DataStream do
+  before do
+    @db = mock('db')
+  end
+
+  it "increments the offset" do
+    stream = Taps::DataStream.new(@db, :table_name => 'test_table', :chunksize => 100)
+    stream.state[:offset].should == 0
+    stream.increment(100)
+    stream.state[:offset].should == 100
+  end
+
+  it "marks the stream complete if no rows are fetched" do
+    stream = Taps::DataStream.new(@db, :table_name => 'test_table', :chunksize => 100)
+    stream.stubs(:fetch_rows).returns({})
+    stream.complete?.should.be.false
+    stream.fetch
+    stream.complete?.should.be.true
+  end
+end

data/spec/operation_spec.rb

@@ -0,0 +1,42 @@
+require File.dirname(__FILE__) + '/base'
+require 'taps/operation'
+
+describe Taps::Operation do
+  before do
+    @op = Taps::Operation.new('dummy://localhost', 'http://x:y@localhost:5000')
+  end
+
+  it "returns an array of tables that match the regex table_filter" do
+    @op = Taps::Operation.new('dummy://localhost', 'http://x:y@localhost:5000', :table_filter => 'abc')
+    @op.apply_table_filter(['abc', 'def']).should == ['abc']
+  end
+
+  it "returns a hash of tables that match the regex table_filter" do
+    @op = Taps::Operation.new('dummy://localhost', 'http://x:y@localhost:5000', :table_filter => 'abc')
+    @op.apply_table_filter({ 'abc' => 1, 'def' => 2 }).should == { 'abc' => 1 }
+  end
+
+  it "returns an array of tables without the exclude_tables tables" do
+    @op = Taps::Operation.new('dummy://localhost', 'http://x:y@localhost:5000', :exclude_tables => ['abc', 'ghi', 'jkl'])
+    @op.apply_table_filter(['abc', 'def', 'ghi', 'jkl', 'mno']).should == ['def', 'mno']
+  end
+
+  it "returns a hash of tables without the exclude_tables tables" do
+    @op = Taps::Operation.new('dummy://localhost', 'http://x:y@localhost:5000', :exclude_tables => ['abc', 'ghi', 'jkl'])
+    @op.apply_table_filter({ 'abc' => 1, 'def' => 2, 'ghi' => 3, 'jkl' => 4, 'mno' => 5 }).should == { 'def' => 2, 'mno' => 5 }
+  end
+
+  it "masks a url's password" do
+    @op.safe_url("mysql://root:password@localhost/mydb").should == "mysql://root:[hidden]@localhost/mydb"
+  end
+
+  it "returns http headers with compression enabled" do
+    @op.http_headers.should == { :taps_version => Taps.version, :accept_encoding => "gzip, deflate" }
+  end
+
+  it "returns http headers with compression disabled" do
+    @op.stubs(:compression_disabled?).returns(true)
+    @op.http_headers.should == { :taps_version => Taps.version, :accept_encoding => "" }
+  end
+
+end

data/spec/server_spec.rb

@@ -0,0 +1,38 @@
+require File.dirname(__FILE__) + '/base'
+
+require 'taps/server'
+require 'pp'
+
+describe Taps::Server do
+  def app
+    Taps::Server.new
+  end
+
+  before do
+    Taps::Config.login = 'taps'
+    Taps::Config.password = 'tpass'
+
+    @app = Taps::Server
+    @auth_header = "Basic " + ["taps:tpass"].pack("m*")
+  end
+
+  it "asks for http basic authentication" do
+    get '/'
+    last_response.status.should == 401
+  end
+
+  it "verifies the client taps version" do
+    get('/', { }, { 'HTTP_AUTHORIZATION' => @auth_header, 'HTTP_TAPS_VERSION' => Taps.version })
+    last_response.status.should == 200
+  end
+
+  it "yells loudly if the client taps version doesn't match" do
+    get('/', { }, { 'HTTP_AUTHORIZATION' => @auth_header, 'HTTP_TAPS_VERSION' => '0.0.1' })
+    last_response.status.should == 417
+  end
+
+  it "allows healthcheck to be accessed w/o HTTP_TAPS_VERSION" do
+    get('/health', { }, { 'HTTP_AUTHORIZATION' => @auth_header })
+    last_response.status.should == 200
+  end
+end

data/spec/utils_spec.rb

@@ -0,0 +1,38 @@
+require File.dirname(__FILE__) + '/base'
+require 'taps/utils'
+
+describe Taps::Utils do
+  it "generates a checksum using crc32" do
+    Taps::Utils.checksum("hello world").should == Zlib.crc32("hello world")
+  end
+
+  it "formats a data hash into one hash that contains an array of headers and an array of array of data" do
+    first_row = { :x => 1, :y => 1 }
+    first_row.stubs(:keys).returns([:x, :y])
+    Taps::Utils.format_data([ first_row, { :x => 2, :y => 2 } ]).should == { :header => [ :x, :y ], :data => [ [1, 1], [2, 2] ] }
+  end
+
+  it "enforces length limitations on columns" do
+    data = [ { :a => "aaabbbccc" } ]
+    schema = [ [ :a, { :db_type => "varchar(3)" }]]
+    lambda { Taps::Utils.format_data(data, :schema => schema) }.should.raise(Taps::InvalidData)
+  end
+
+  it "returns a list of columns that are text fields if the database is mysql" do
+    @db = mock("db", :url => "mysql://localhost/mydb")
+    @db.stubs(:schema).with(:mytable).returns([
+      [:id, { :db_type => "int" }],
+      [:mytext, { :db_type => "text" }]
+    ])
+    Taps::Utils.incorrect_blobs(@db, :mytable).should == [:mytext]
+  end
+
+  it "rejects a multiple-column primary key as a single integer primary key" do
+    @db = mock("db")
+    @db.stubs(:schema).with(:pktable.identifier).returns([
+      [:id1, { :primary_key => true, :type => :integer }],
+      [:id2, { :primary_key => true, :type => :string }]
+    ])
+    Taps::Utils.single_integer_primary_key(@db, :pktable).should.be.false
+  end
+end

metadata

@@ -0,0 +1,230 @@
+--- !ruby/object:Gem::Specification
+name: taps2
+version: !ruby/object:Gem::Version
+  version: 0.5.1
+platform: ruby
+authors:
+- Ricardo Chimal, Jr.
+- Joel Van Horn
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2017-04-07 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rack
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.0.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.0.1
+- !ruby/object:Gem::Dependency
+  name: rest-client
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.4.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.4.0
+- !ruby/object:Gem::Dependency
+  name: sequel
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 4.0.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 4.0.0
+- !ruby/object:Gem::Dependency
+  name: sinatra
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.4.4
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.4.4
+- !ruby/object:Gem::Dependency
+  name: sqlite3
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.3.8
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.3.8
+- !ruby/object:Gem::Dependency
+  name: extlib
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bacon
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: mocha
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.2.1
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.2.1
+- !ruby/object:Gem::Dependency
+  name: rack-test
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: simplecov
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: A simple database agnostic import/export app to transfer data to/from
+  a remote database.
+email:
+- ricardo@heroku.com
+- joel@joelvanhorn.com
+executables:
+- taps
+- schema
+extensions: []
+extra_rdoc_files: []
+files:
+- README.rdoc
+- VERSION.yml
+- bin/schema
+- bin/schema.cmd
+- bin/taps
+- lib/taps/chunksize.rb
+- lib/taps/cli.rb
+- lib/taps/config.rb
+- lib/taps/data_stream.rb
+- lib/taps/db_session.rb
+- lib/taps/errors.rb
+- lib/taps/log.rb
+- lib/taps/monkey.rb
+- lib/taps/multipart.rb
+- lib/taps/operation.rb
+- lib/taps/progress_bar.rb
+- lib/taps/schema.rb
+- lib/taps/server.rb
+- lib/taps/utils.rb
+- lib/taps/version.rb
+- lib/vendor/okjson.rb
+- spec/base.rb
+- spec/chunksize_spec.rb
+- spec/cli_spec.rb
+- spec/data_stream_spec.rb
+- spec/operation_spec.rb
+- spec/server_spec.rb
+- spec/utils_spec.rb
+homepage: http://github.com/joelvh/taps2
+licenses: []
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.6.11
+signing_key: 
+specification_version: 4
+summary: Simple database import/export app
+test_files: []