taps 0.3.13 → 0.3.14

data/Rakefile

@@ -13,7 +13,7 @@ begin
     s.add_dependency 'json_pure', '>= 1.2.0', '< 1.5.0'
     s.add_dependency 'sinatra', '~> 1.0.0'
     s.add_dependency 'rest-client', '>= 1.4.0', '< 1.7.0'
-    s.add_dependency 'sequel', '~> 3.15.0'
+    s.add_dependency 'sequel', '~> 3.17.0'
     s.add_dependency 'sqlite3-ruby', '~> 1.2'
     s.add_dependency 'rack', '>= 1.0.1'
 

data/VERSION.yml

@@ -1,5 +1,5 @@
 --- 
 :build: 
 :minor: 3
-:patch: 13
+:patch: 14
 :major: 0

data/bin/schema

@@ -1,7 +1,7 @@
 #!/usr/bin/env ruby
 
 require 'rubygems'
-gem 'sequel', '~> 3.15.0'
+gem 'sequel', '~> 3.17.0'
 
 $:.unshift File.dirname(__FILE__) + '/../lib'
 

data/lib/taps/data_stream.rb

@@ -2,13 +2,12 @@ require 'taps/monkey'
 require 'taps/multipart'
 require 'taps/utils'
 require 'taps/log'
+require 'taps/errors'
 require 'json/pure'
 
 module Taps
 
 class DataStream
-  class CorruptedData < Exception; end
-
   DEFAULT_CHUNKSIZE = 1000
 
   attr_reader :db, :state
@@ -79,7 +78,10 @@ class DataStream
     ds = table.order(*order_by).limit(state[:chunksize], state[:offset])
     log.debug "DataStream#fetch_rows SQL -> #{ds.sql}"
     rows = Taps::Utils.format_data(ds.all,
-      :string_columns => string_columns)
+      :string_columns => string_columns,
+      :schema => db.schema(table_name),
+      :table  => table_name
+    )
     update_chunksize_stats
     rows
   end
@@ -173,7 +175,7 @@ class DataStream
       params[:json] = self.class.parse_json(params[:json]) if params.has_key?(:json)
       return params
     rescue JSON::Parser
-      raise DataStream::CorruptedData.new("Invalid JSON Received")
+      raise Taps::CorruptedData.new("Invalid JSON Received")
     end
   end
 
@@ -184,7 +186,7 @@ class DataStream
   end
 
   def parse_encoded_data(encoded_data, checksum)
-    raise DataStream::CorruptedData.new("Checksum Failed") unless Taps::Utils.valid_data?(encoded_data, checksum)
+    raise Taps::CorruptedData.new("Checksum Failed") unless Taps::Utils.valid_data?(encoded_data, checksum)
 
     begin
       return Marshal.load(Taps::Utils.base64decode(encoded_data))
@@ -202,6 +204,17 @@ class DataStream
     state[:offset] += rows[:data].size
   end
 
+  def verify_stream
+    state[:offset] = table.count
+  end
+
+  def verify_remote_stream(resource, headers)
+    json_raw = resource.post({:state => self.to_json}, headers).to_s
+    json = self.class.parse_json(json_raw)
+
+    self.class.new(db, json[:state])
+  end
+
   def self.factory(db, state)
     if defined?(Sequel::MySQL) && Sequel::MySQL.respond_to?(:convert_invalid_date_time=)
       Sequel::MySQL.convert_invalid_date_time = :nil
@@ -302,6 +315,19 @@ class DataStreamKeyed < DataStream
     # pop the rows we just successfully sent off the buffer
     @buffer.slice!(0, row_count)
   end
+
+  def verify_stream
+    key = primary_key
+    ds = table.order(*order_by)
+    current_filter = ds.max(key.sql_number)
+
+    # set the current filter to the max of the primary key
+    state[:filter] = current_filter
+    # clear out the last_fetched value so it can restart from scratch
+    state[:last_fetched] = nil
+
+    log.debug "DataStreamKeyed#verify_stream -> state: #{state.inspect}"
+  end
 end
 
 end

data/lib/taps/errors.rb

@@ -0,0 +1,15 @@
+module Taps
+  class BaseError < StandardError
+    attr_reader :original_backtrace
+
+    def initialize(message, opts={})
+      @original_backtrace = opts.delete(:backtrace)
+      super(message)
+    end
+  end
+
+  class NotImplemented < BaseError; end
+  class DuplicatePrimaryKeyError < BaseError; end
+  class CorruptedData < BaseError; end
+  class InvalidData < BaseError; end
+end

data/lib/taps/operation.rb

@@ -6,6 +6,7 @@ require 'taps/progress_bar'
 require 'taps/config'
 require 'taps/utils'
 require 'taps/data_stream'
+require 'taps/errors'
 
 # disable warnings, rest client makes a lot of noise right now
 $VERBOSE = nil
@@ -186,6 +187,30 @@ class Operation
     end
   end
 
+  def catch_errors(&blk)
+    verify_server
+
+    begin
+      blk.call
+      close_session
+    rescue RestClient::Exception, Taps::BaseError => e
+      store_session
+      if e.kind_of?(Taps::BaseError)
+        puts "!!! Caught Server Exception"
+        puts "#{e.class}: #{e.message}"
+        puts "\n#{e.original_backtrace}" if e.original_backtrace
+        exit(1)
+      elsif e.respond_to?(:response)
+        puts "!!! Caught Server Exception"
+        puts "HTTP CODE: #{e.http_code}"
+        puts "#{e.response.to_s}"
+        exit(1)
+      else
+        raise
+      end
+    end
+  end
+
   def self.factory(type, database_url, remote_url, opts)
     type = :resume if opts[:resume]
     klass = case type
@@ -209,9 +234,7 @@ class Pull < Operation
   end
 
   def run
-    verify_server
-
-    begin
+    catch_errors do
       unless resuming?
         pull_schema
         pull_indexes if indexes_first?
@@ -221,17 +244,6 @@ class Pull < Operation
       pull_data
       pull_indexes unless indexes_first?
       pull_reset_sequences
-      close_session
-    rescue RestClient::Exception => e
-      store_session
-      if e.respond_to?(:response)
-        puts "!!! Caught Server Exception"
-        puts "HTTP CODE: #{e.http_code}"
-        puts "#{e.response.to_s}"
-        exit(1)
-      else
-        raise
-      end
     end
   end
 
@@ -289,7 +301,7 @@ class Pull < Operation
         progress.inc(size) unless exiting?
         stream.error = false
         self.stream_state = stream.to_hash
-      rescue DataStream::CorruptedData => e
+      rescue Taps::CorruptedData => e
         puts "Corrupted Data Received #{e.message}, retrying..."
         stream.error = true
         next
@@ -381,8 +393,7 @@ class Push < Operation
   end
 
   def run
-    verify_server
-    begin
+    catch_errors do
       unless resuming?
         push_schema
         push_indexes if indexes_first?
@@ -392,17 +403,6 @@ class Push < Operation
       push_data
       push_indexes unless indexes_first?
       push_reset_sequences
-      close_session
-    rescue RestClient::Exception => e
-      store_session
-      if e.respond_to?(:response)
-        puts "!!! Caught Server Exception"
-        puts "HTTP CODE: #{e.http_code}"
-        puts "#{e.response.to_s}"
-        exit(1)
-      else
-        raise
-      end
     end
   end
 
@@ -477,35 +477,42 @@ class Push < Operation
 
       row_size = 0
       chunksize = stream.state[:chunksize]
-      chunksize = Taps::Utils.calculate_chunksize(chunksize) do |c|
-        stream.state[:chunksize] = c
-        encoded_data, row_size, elapsed_time = stream.fetch
-        break if stream.complete?
 
-        data = {
-          :state => stream.to_hash,
-          :checksum => Taps::Utils.checksum(encoded_data).to_s
-        }
-
-        begin
-          content, content_type = Taps::Multipart.create do |r|
-            r.attach :name => :encoded_data,
-              :payload => encoded_data,
-              :content_type => 'application/octet-stream'
-            r.attach :name => :json,
-              :payload => data.to_json,
-              :content_type => 'application/json'
-          end
-          session_resource['push/table'].post(content, http_headers(:content_type => content_type))
-          self.stream_state = stream.to_hash
-        rescue RestClient::RequestFailed => e
-          # retry the same data, it got corrupted somehow.
-          if e.http_code == 412
-            next
+      begin
+        chunksize = Taps::Utils.calculate_chunksize(chunksize) do |c|
+          stream.state[:chunksize] = c
+          encoded_data, row_size, elapsed_time = stream.fetch
+          break if stream.complete?
+
+          data = {
+            :state => stream.to_hash,
+            :checksum => Taps::Utils.checksum(encoded_data).to_s
+          }
+
+          begin
+            content, content_type = Taps::Multipart.create do |r|
+              r.attach :name => :encoded_data,
+                :payload => encoded_data,
+                :content_type => 'application/octet-stream'
+              r.attach :name => :json,
+                :payload => data.to_json,
+                :content_type => 'application/json'
+            end
+            session_resource['push/table'].post(content, http_headers(:content_type => content_type))
+            self.stream_state = stream.to_hash
+          rescue => e
+            Taps::Utils.reraise_server_exception(e)
           end
-          raise
+
+          elapsed_time
         end
-        elapsed_time
+      rescue Taps::CorruptedData => e
+        # retry the same data, it got corrupted somehow.
+        next
+      rescue Taps::DuplicatePrimaryKeyError => e
+        # verify the stream and retry it
+        stream = stream.verify_remote_stream(session_resource['push/verify_stream'], http_headers)
+        next
       end
       stream.state[:chunksize] = chunksize
 

data/lib/taps/server.rb

@@ -14,13 +14,18 @@ class Server < Sinatra::Base
 
   error do
     e = request.env['sinatra.error']
-    "Taps Server Error: #{e}\n#{e.backtrace}"
+    if e.kind_of?(Taps::BaseError)
+      content_type "application/json"
+      halt 412, { 'error_class' => e.class.to_s, 'error_message' => e.message, 'error_backtrace' => e.backtrace.join("\n") }.to_json
+    else
+      "Taps Server Error: #{e}\n#{e.backtrace}"
+    end
   end
 
   before do
     major, minor, patch = request.env['HTTP_TAPS_VERSION'].split('.') rescue []
-    unless "#{major}.#{minor}" == Taps.compatible_version
-      halt 417, "Taps v#{Taps.compatible_version}.x is required for this server"
+    unless "#{major}.#{minor}" == Taps.compatible_version && patch.to_i >= 14
+      halt 417, "Taps >= v#{Taps.compatible_version}.14 is required for this server"
     end
   end
 
@@ -42,6 +47,25 @@ class Server < Sinatra::Base
     "/sessions/#{key}"
   end
 
+  post '/sessions/:key/push/verify_stream' do
+    session = DbSession.filter(:key => params[:key]).first
+    halt 404 unless session
+
+    state = DataStream.parse_json(params[:state])
+    stream = nil
+
+    size = 0
+    session.conn do |db|
+      Taps::Utils.server_error_handling do
+        stream = DataStream.factory(db, state)
+        stream.verify_stream
+      end
+    end
+
+    content_type 'application/json'
+    { :state => stream.to_hash }.to_json
+  end
+
   post '/sessions/:key/push/table' do
     session = DbSession.filter(:key => params[:key]).first
     halt 404 unless session
@@ -50,11 +74,9 @@ class Server < Sinatra::Base
 
     size = 0
     session.conn do |db|
-      begin
+      Taps::Utils.server_error_handling do
         stream = DataStream.factory(db, json[:state])
         size = stream.fetch_remote_in_server(params)
-      rescue Taps::DataStream::CorruptedData
-        halt 412
       end
     end
 

data/lib/taps/utils.rb

@@ -3,6 +3,8 @@ require 'stringio'
 require 'time'
 require 'tempfile'
 
+require 'taps/errors'
+
 module Taps
 module Utils
   extend self
@@ -37,10 +39,32 @@ module Utils
   def format_data(data, opts={})
     return {} if data.size == 0
     string_columns = opts[:string_columns] || []
+    schema = opts[:schema] || []
+    table  = opts[:table]
+
+    max_lengths = schema.inject({}) do |hash, (column, meta)|
+      if meta[:db_type] =~ /^\w+\((\d+)\)/
+        hash.update(column => $1.to_i)
+      end
+      hash
+    end
 
     header = data[0].keys
     only_data = data.collect do |row|
       row = blobs_to_string(row, string_columns)
+      row.each do |column, data|
+        if data.to_s.length > (max_lengths[column] || data.to_s.length)
+          raise Taps::InvalidData.new(<<-ERROR)
+Detected data that exceeds the length limitation of its column. This is
+generally due to the fact that SQLite does not enforce length restrictions.
+
+Table  : #{table}
+Column : #{column}
+Type   : #{schema.detect{|s| s.first == column}.last[:db_type]}
+Data   : #{data}
+          ERROR
+        end
+      end
       header.collect { |h| row[h] }
     end
     { :header => header, :data => only_data }
@@ -127,12 +151,7 @@ module Utils
   end
 
   def primary_key(db, table)
-    table = table.to_sym.identifier unless table.kind_of?(Sequel::SQL::Identifier)
-    if db.respond_to?(:primary_key)
-      db.primary_key(table)
-    else
-      db.schema(table).select { |c| c[1][:primary_key] }.map { |c| c.first.to_sym }
-    end
+    db.schema(table).select { |c| c[1][:primary_key] }.map { |c| c[0] }
   end
 
   def single_integer_primary_key(db, table)
@@ -150,5 +169,31 @@ module Utils
       db[table].columns
     end
   end
+
+
+  # try to detect server side errors to
+  # give the client a more useful error message
+  def server_error_handling(&blk)
+    begin
+      blk.call
+    rescue Sequel::DatabaseError => e
+      if e.message =~ /duplicate key value/i
+        raise Taps::DuplicatePrimaryKeyError, e.message
+      else
+        raise
+      end
+    end
+  end
+
+  def reraise_server_exception(e)
+    if e.kind_of?(RestClient::Exception)
+      if e.respond_to?(:response) && e.response.headers[:content_type] == 'application/json'
+        json = JSON.parse(e.response.to_s)
+        klass = eval(json['error_class']) rescue nil
+        raise klass.new(json['error_message'], :backtrace => json['error_backtrace']) if klass
+      end
+    end
+    raise e
+  end
 end
 end

data/spec/utils_spec.rb

@@ -12,6 +12,12 @@ describe Taps::Utils do
     Taps::Utils.format_data([ first_row, { :x => 2, :y => 2 } ]).should == { :header => [ :x, :y ], :data => [ [1, 1], [2, 2] ] }
   end
 
+  it "enforces length limitations on columns" do
+    data = [ { :a => "aaabbbccc" } ]
+    schema = [ [ :a, { :db_type => "varchar(3)" }]]
+    lambda { Taps::Utils.format_data(data, :schema => schema) }.should.raise(Taps::InvalidData)
+  end
+
   it "scales chunksize down slowly when the time delta of the block is just over a second" do
     Time.stubs(:now).returns(10.0).returns(11.5)
     Taps::Utils.calculate_chunksize(1000) { }.should == 900

metadata

@@ -5,8 +5,8 @@ version: !ruby/object:Gem::Version
   segments: 
   - 0
   - 3
-  - 13
-  version: 0.3.13
+  - 14
+  version: 0.3.14
 platform: ruby
 authors: 
 - Ricardo Chimal, Jr.
@@ -14,7 +14,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-09-27 00:00:00 -07:00
+date: 2010-11-12 00:00:00 -08:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -82,9 +82,9 @@ dependencies:
       - !ruby/object:Gem::Version 
         segments: 
         - 3
-        - 15
+        - 17
         - 0
-        version: 3.15.0
+        version: 3.17.0
   type: :runtime
   version_requirements: *id004
 - !ruby/object:Gem::Dependency 
@@ -137,6 +137,7 @@ files:
 - lib/taps/config.rb
 - lib/taps/data_stream.rb
 - lib/taps/db_session.rb
+- lib/taps/errors.rb
 - lib/taps/log.rb
 - lib/taps/monkey.rb
 - lib/taps/multipart.rb