taps 0.3.15 → 0.3.17

data/VERSION.yml

@@ -1,5 +1,5 @@
 ---
 :build:
 :minor: 3
-:patch: 15
+:patch: 17
 :major: 0

data/bin/schema

@@ -1,7 +1,7 @@
 #!/usr/bin/env ruby
 
 require 'rubygems'
-gem 'sequel', '~> 3.17.0'
+gem 'sequel', '~> 3.20.0'
 
 $:.unshift File.dirname(__FILE__) + '/../lib'
 

data/lib/taps/chunksize.rb

@@ -0,0 +1,52 @@
+require 'taps/errors'
+
+class Taps::Chunksize
+  attr_accessor :idle_secs, :time_in_db, :start_time, :end_time, :retries
+  attr_reader :chunksize
+
+  def initialize(chunksize)
+    @chunksize = chunksize
+    @idle_secs = 0.0
+    @retries = 0
+  end
+
+  def to_i
+    chunksize
+  end
+
+  def reset_chunksize
+    @chunksize = (retries <= 1) ? 10 : 1
+  end
+
+  def diff
+    end_time - start_time - time_in_db - idle_secs
+  end
+
+  def time_in_db=(t)
+    @time_in_db = t
+    @time_in_db = @time_in_db.to_f rescue 0.0
+  end
+
+  def time_delta
+    t1 = Time.now
+    yield if block_given?
+    t2 = Time.now
+    t2 - t1
+  end
+
+  def calc_new_chunksize
+    new_chunksize = if retries > 0
+      chunksize
+    elsif diff > 3.0
+      (chunksize / 3).ceil
+    elsif diff > 1.1
+      chunksize - 100
+    elsif diff < 0.8
+      chunksize * 2
+    else
+      chunksize + 100
+    end
+    new_chunksize = 1 if new_chunksize < 1
+    new_chunksize
+  end
+end

data/lib/taps/cli.rb

@@ -128,6 +128,7 @@ EOHELP
         o.define_head "Push a database to a taps server"
       end
 
+      o.on("-s", "--skip-schema", "Don't transfer the schema, just data") { |v| opts[:skip_schema] = true }
       o.on("-i", "--indexes-first", "Transfer indexes first before data") { |v| opts[:indexes_first] = true }
       o.on("-r", "--resume=file", "Resume a Taps Session from a stored file") { |v| opts[:resume_filename] = v }
       o.on("-c", "--chunksize=N", "Initial Chunksize") { |v| opts[:default_chunksize] = (v.to_i < 10 ? 10 : v.to_i) }
@@ -137,6 +138,7 @@ EOHELP
         r_tables = v.collect { |t| "^#{t}$" }.join("|")
         opts[:table_filter] = "(#{r_tables})"
       end
+      o.on("-e", "--exclude_tables=A,B,C", Array, "Shortcut to exclude a list of tables") { |v| opts[:exclude_tables] = v }
       o.on("-d", "--debug", "Enable Debug Messages") { |v| opts[:debug] = true }
       o.parse!(argv)
 

data/lib/taps/data_stream.rb

@@ -166,7 +166,7 @@ class DataStream
     res = nil
     log.debug "DataStream#fetch_from_resource state -> #{state.inspect}"
     state[:chunksize] = Taps::Utils.calculate_chunksize(state[:chunksize]) do |c|
-      state[:chunksize] = c
+      state[:chunksize] = c.to_i
       res = resource.post({:state => self.to_json}, headers)
     end
 
@@ -202,6 +202,16 @@ class DataStream
   def import_rows(rows)
     table.import(rows[:header], rows[:data])
     state[:offset] += rows[:data].size
+  rescue Exception => ex
+    case ex.message
+    when /integer out of range/ then
+      raise Taps::InvalidData, <<-ERROR, []
+\nDetected integer data that exceeds the maximum allowable size for an integer type.
+This generally occurs when importing from SQLite due to the fact that SQLite does
+not enforce maximum values on integer types.
+      ERROR
+    else raise ex
+    end
   end
 
   def verify_stream

data/lib/taps/operation.rb

@@ -29,6 +29,10 @@ class Operation
     "op"
   end
 
+  def skip_schema?
+    !!opts[:skip_schema]
+  end
+
   def indexes_first?
     !!opts[:indexes_first]
   end
@@ -37,19 +41,24 @@ class Operation
     opts[:table_filter]
   end
 
+  def exclude_tables
+    opts[:exclude_tables] || []
+  end
+
   def apply_table_filter(tables)
-    return tables unless table_filter
-    re = Regexp.new(table_filter)
+    return tables unless table_filter || exclude_tables
+
+    re = table_filter ? Regexp.new(table_filter) : nil
     if tables.kind_of?(Hash)
       ntables = {}
       tables.each do |t, d|
-        unless re.match(t.to_s).nil?
+        if !exclude_tables.include?(t.to_s) && (!re || !re.match(t.to_s).nil?)
           ntables[t] = d
         end
       end
       ntables
     else
-      tables.reject { |t| re.match(t.to_s).nil? }
+      tables.reject { |t| exclude_tables.include?(t.to_s) || (re && re.match(t.to_s).nil?) }
     end
   end
 
@@ -236,13 +245,13 @@ class Pull < Operation
   def run
     catch_errors do
       unless resuming?
-        pull_schema
-        pull_indexes if indexes_first?
+        pull_schema if !skip_schema?
+        pull_indexes if indexes_first? && !skip_schema?
       end
       setup_signal_trap
       pull_partial_data if resuming?
       pull_data
-      pull_indexes unless indexes_first?
+      pull_indexes if !indexes_first? && !skip_schema?
       pull_reset_sequences
     end
   end
@@ -395,13 +404,13 @@ class Push < Operation
   def run
     catch_errors do
       unless resuming?
-        push_schema
-        push_indexes if indexes_first?
+        push_schema if !skip_schema?
+        push_indexes if indexes_first? && !skip_schema?
       end
       setup_signal_trap
       push_partial_data if resuming?
       push_data
-      push_indexes unless indexes_first?
+      push_indexes if !indexes_first? && !skip_schema?
       push_reset_sequences
     end
   end
@@ -480,23 +489,32 @@ class Push < Operation
 
       begin
         chunksize = Taps::Utils.calculate_chunksize(chunksize) do |c|
-          stream.state[:chunksize] = c
-          encoded_data, row_size, elapsed_time = stream.fetch
+          stream.state[:chunksize] = c.to_i
+          encoded_data, row_size, elapsed_time = nil
+          d1 = c.time_delta do
+            encoded_data, row_size, elapsed_time = stream.fetch
+          end
           break if stream.complete?
 
-          data = {
-            :state => stream.to_hash,
-            :checksum => Taps::Utils.checksum(encoded_data).to_s
-          }
+          data = nil
+          d2 = c.time_delta do
+            data = {
+              :state => stream.to_hash,
+              :checksum => Taps::Utils.checksum(encoded_data).to_s
+            }
+          end
 
           begin
-            content, content_type = Taps::Multipart.create do |r|
-              r.attach :name => :encoded_data,
-                :payload => encoded_data,
-                :content_type => 'application/octet-stream'
-              r.attach :name => :json,
-                :payload => data.to_json,
-                :content_type => 'application/json'
+            content, content_type = nil
+            d3 = c.time_delta do
+              content, content_type = Taps::Multipart.create do |r|
+                r.attach :name => :encoded_data,
+                  :payload => encoded_data,
+                  :content_type => 'application/octet-stream'
+                r.attach :name => :json,
+                  :payload => data.to_json,
+                  :content_type => 'application/json'
+              end
             end
             session_resource['push/table'].post(content, http_headers(:content_type => content_type))
             self.stream_state = stream.to_hash
@@ -504,6 +522,8 @@ class Push < Operation
             Taps::Utils.reraise_server_exception(e)
           end
 
+          c.idle_secs = (d1 + d2 + d3)
+
           elapsed_time
         end
       rescue Taps::CorruptedData => e

data/lib/taps/utils.rb

@@ -2,8 +2,10 @@ require 'zlib'
 require 'stringio'
 require 'time'
 require 'tempfile'
+require 'rest_client'
 
 require 'taps/errors'
+require 'taps/chunksize'
 
 module Taps
 module Utils
@@ -93,42 +95,23 @@ Data   : #{data}
   end
 
   def calculate_chunksize(old_chunksize)
-    chunksize = old_chunksize
+    c = Taps::Chunksize.new(old_chunksize)
 
-    retries = 0
-    time_in_db = 0
     begin
-      t1 = Time.now
-      time_in_db = yield chunksize
-      time_in_db = time_in_db.to_f rescue 0
+      c.start_time = Time.now
+      c.time_in_db = yield c
     rescue Errno::EPIPE, RestClient::RequestFailed, RestClient::RequestTimeout
-      retries += 1
-      raise if retries > 2
+      c.retries += 1
+      raise if c.retries > 2
 
       # we got disconnected, the chunksize could be too large
-      # on first retry change to 10, on successive retries go down to 1
-      chunksize = (retries == 1) ? 10 : 1
-
+      # reset the chunksize based on the number of retries
+      c.reset_chunksize
       retry
     end
 
-    t2 = Time.now
-
-    diff = t2 - t1 - time_in_db
-
-    new_chunksize = if retries > 0
-      chunksize
-    elsif diff > 3.0
-      (chunksize / 3).ceil
-    elsif diff > 1.1
-      chunksize - 100
-    elsif diff < 0.8
-      chunksize * 2
-    else
-      chunksize + 100
-    end
-    new_chunksize = 1 if new_chunksize < 1
-    new_chunksize
+    c.end_time = Time.now
+    c.calc_new_chunksize
   end
 
   def load_schema(database_url, schema_data)

data/spec/chunksize_spec.rb

@@ -0,0 +1,41 @@
+require File.dirname(__FILE__) + '/base'
+require 'taps/utils'
+
+describe Taps::Chunksize do
+  it "scales chunksize down slowly when the time delta of the block is just over a second" do
+    Time.stubs(:now).returns(10.0).returns(11.5)
+    Taps::Utils.calculate_chunksize(1000) { |c| }.should == 900
+  end
+
+  it "scales chunksize down fast when the time delta of the block is over 3 seconds" do
+    Time.stubs(:now).returns(10.0).returns(15.0)
+    Taps::Utils.calculate_chunksize(3000) { |c| }.should == 1000
+  end
+
+  it "scales up chunksize fast when the time delta of the block is under 0.8 seconds" do
+    Time.stubs(:now).returns(10.0).returns(10.7)
+    Taps::Utils.calculate_chunksize(1000) { |c| }.should == 2000
+  end
+
+  it "scales up chunksize slow when the time delta of the block is between 0.8 and 1.1 seconds" do
+    Time.stubs(:now).returns(10.0).returns(10.8)
+    Taps::Utils.calculate_chunksize(1000) { |c| }.should == 1100
+
+    Time.stubs(:now).returns(10.0).returns(11.1)
+    Taps::Utils.calculate_chunksize(1000) { |c| }.should == 1100
+  end
+
+  it "will reset the chunksize to a small value if we got a broken pipe exception" do
+    Taps::Utils.calculate_chunksize(1000) do |c|
+      raise Errno::EPIPE if c.chunksize == 1000
+      c.chunksize.should == 10
+    end.should == 10
+  end
+
+  it "will reset the chunksize to a small value if we got a broken pipe exception a second time" do
+    Taps::Utils.calculate_chunksize(1000) do |c|
+      raise Errno::EPIPE if c.chunksize == 1000 || c.chunksize == 10
+      c.chunksize.should == 1
+    end.should == 1
+  end
+end

data/spec/cli_spec.rb

@@ -7,4 +7,10 @@ describe Taps::Cli do
     opts = @cli.clientoptparse(:pull)
     opts[:table_filter].should == "(^mytable1$|^logs$)"
   end
+
+  it "translates a list of tables to exclude into a regex that can be used in table_filter" do
+    @cli = Taps::Cli.new(["-e", "mytable1,logs", "sqlite://tmp.db", "http://x:y@localhost:5000"])
+    opts = @cli.clientoptparse(:pull)
+    opts[:exclude_tables].should == ['mytable1','logs']
+  end
 end

data/spec/operation_spec.rb

@@ -16,6 +16,16 @@ describe Taps::Operation do
     @op.apply_table_filter({ 'abc' => 1, 'def' => 2 }).should == { 'abc' => 1 }
   end
 
+  it "returns an array of tables without the exclude_tables tables" do
+    @op = Taps::Operation.new('dummy://localhost', 'http://x:y@localhost:5000', :exclude_tables => ['abc', 'ghi', 'jkl'])
+    @op.apply_table_filter(['abc', 'def', 'ghi', 'jkl', 'mno']).should == ['def', 'mno']
+  end
+
+  it "returns a hash of tables without the exclude_tables tables" do
+    @op = Taps::Operation.new('dummy://localhost', 'http://x:y@localhost:5000', :exclude_tables => ['abc', 'ghi', 'jkl'])
+    @op.apply_table_filter({ 'abc' => 1, 'def' => 2, 'ghi' => 3, 'jkl' => 4, 'mno' => 5 }).should == { 'def' => 2, 'mno' => 5 }
+  end
+
   it "masks a url's password" do
     @op.safe_url("mysql://root:password@localhost/mydb").should == "mysql://root:[hidden]@localhost/mydb"
   end

data/spec/utils_spec.rb

@@ -18,37 +18,6 @@ describe Taps::Utils do
     lambda { Taps::Utils.format_data(data, :schema => schema) }.should.raise(Taps::InvalidData)
   end
 
-  it "scales chunksize down slowly when the time delta of the block is just over a second" do
-    Time.stubs(:now).returns(10.0).returns(11.5)
-    Taps::Utils.calculate_chunksize(1000) { }.should == 900
-  end
-
-  it "scales chunksize down fast when the time delta of the block is over 3 seconds" do
-    Time.stubs(:now).returns(10.0).returns(15.0)
-    Taps::Utils.calculate_chunksize(3000) { }.should == 1000
-  end
-
-  it "scales up chunksize fast when the time delta of the block is under 0.8 seconds" do
-    Time.stubs(:now).returns(10.0).returns(10.7)
-    Taps::Utils.calculate_chunksize(1000) { }.should == 2000
-  end
-
-  it "scales up chunksize slow when the time delta of the block is between 0.8 and 1.1 seconds" do
-    Time.stubs(:now).returns(10.0).returns(10.8)
-    Taps::Utils.calculate_chunksize(1000) { }.should == 1100
-
-    Time.stubs(:now).returns(10.0).returns(11.1)
-    Taps::Utils.calculate_chunksize(1000) { }.should == 1100
-  end
-
-  it "will reset the chunksize to a small value if we got a broken pipe exception" do
-    Taps::Utils.calculate_chunksize(1000) { |c| raise Errno::EPIPE if c == 1000; c.should == 10 }.should == 10
-  end
-
-  it "will reset the chunksize to a small value if we got a broken pipe exception a second time" do
-    Taps::Utils.calculate_chunksize(1000) { |c| raise Errno::EPIPE if c == 1000 || c == 10; c.should == 1 }.should == 1
-  end
-
   it "returns a list of columns that are text fields if the database is mysql" do
     @db = mock("db", :url => "mysql://localhost/mydb")
     @db.stubs(:schema).with(:mytable).returns([

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: taps
 version: !ruby/object:Gem::Version 
-  hash: 13
+  hash: 49
   prerelease: 
   segments: 
   - 0
   - 3
-  - 15
-  version: 0.3.15
+  - 17
+  version: 0.3.17
 platform: ruby
 authors: 
 - Ricardo Chimal, Jr.
@@ -15,7 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-01-11 00:00:00 -05:00
+date: 2011-02-25 00:00:00 -08:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -26,12 +26,12 @@ dependencies:
     requirements: 
     - - ~>
       - !ruby/object:Gem::Version 
-        hash: 11
+        hash: 1
         segments: 
         - 1
-        - 4
-        - 6
-        version: 1.4.6
+        - 5
+        - 1
+        version: 1.5.1
   type: :runtime
   version_requirements: *id001
 - !ruby/object:Gem::Dependency 
@@ -82,12 +82,12 @@ dependencies:
     requirements: 
     - - ~>
       - !ruby/object:Gem::Version 
-        hash: 67
+        hash: 87
         segments: 
         - 3
-        - 17
+        - 20
         - 0
-        version: 3.17.0
+        version: 3.20.0
   type: :runtime
   version_requirements: *id004
 - !ruby/object:Gem::Dependency 
@@ -204,6 +204,7 @@ files:
 - bin/schema
 - bin/schema.cmd
 - bin/taps
+- lib/taps/chunksize.rb
 - lib/taps/cli.rb
 - lib/taps/config.rb
 - lib/taps/data_stream.rb
@@ -220,6 +221,7 @@ files:
 - lib/taps/version.rb
 - README.rdoc
 - spec/base.rb
+- spec/chunksize_spec.rb
 - spec/cli_spec.rb
 - spec/data_stream_spec.rb
 - spec/operation_spec.rb