taps 0.2.26 → 0.3.0

data/spec/data_stream_spec.rb

@@ -0,0 +1,23 @@
+require File.dirname(__FILE__) + '/base'
+require 'taps/data_stream'
+
+describe Taps::DataStream do
+	before do
+		@db = mock('db')
+	end
+
+	it "increments the offset" do
+		stream = Taps::DataStream.new(@db, :table_name => 'test_table', :chunksize => 100)
+		stream.state[:offset].should == 0
+		stream.increment(100)
+		stream.state[:offset].should == 100
+	end
+
+	it "marks the stream complete if no rows are fetched" do
+		stream = Taps::DataStream.new(@db, :table_name => 'test_table', :chunksize => 100)
+		stream.stubs(:fetch_rows).returns({})
+		stream.complete?.should.be.false
+		stream.fetch
+		stream.complete?.should.be.true
+	end
+end

data/spec/operation_spec.rb

@@ -0,0 +1,32 @@
+require File.dirname(__FILE__) + '/base'
+require 'taps/operation'
+
+describe Taps::Operation do
+	before do
+		@op = Taps::Operation.new('dummy://localhost', 'http://x:y@localhost:5000')
+	end
+
+	it "returns an array of tables that match the regex table_filter" do
+		@op = Taps::Operation.new('dummy://localhost', 'http://x:y@localhost:5000', :table_filter => 'abc')
+		@op.apply_table_filter(['abc', 'def']).should == ['abc']
+	end
+
+	it "returns a hash of tables that match the regex table_filter" do
+		@op = Taps::Operation.new('dummy://localhost', 'http://x:y@localhost:5000', :table_filter => 'abc')
+		@op.apply_table_filter({ 'abc' => 1, 'def' => 2 }).should == { 'abc' => 1 }
+	end
+
+	it "masks a url's password" do
+		@op.safe_url("mysql://root:password@localhost/mydb").should == "mysql://root:[hidden]@localhost/mydb"
+	end
+
+	it "returns http headers with compression enabled" do
+		@op.http_headers.should == { :taps_version => Taps.compatible_version, :accept_encoding => "gzip, deflate" }
+	end
+
+	it "returns http headers with compression disabled" do
+		@op.stubs(:compression_disabled?).returns(true)
+		@op.http_headers.should == { :taps_version => Taps.compatible_version, :accept_encoding => "" }
+	end
+
+end

data/spec/server_spec.rb

@@ -1,6 +1,6 @@
 require File.dirname(__FILE__) + '/base'
 
-require File.dirname(__FILE__) + '/../lib/taps/server'
+require 'taps/server'
 
 require 'pp'
 

data/spec/utils_spec.rb

@@ -1,16 +1,7 @@
 require File.dirname(__FILE__) + '/base'
-require File.dirname(__FILE__) + '/../lib/taps/utils'
+require 'taps/utils'
 
 describe Taps::Utils do
-	it "gunzips a string" do
-		@hello_world = "\037\213\b\000R\261\207I\000\003\313H\315\311\311W(\317/\312I\001\000\205\021J\r\v\000\000\000"
-		Taps::Utils.gunzip(@hello_world).should == "hello world"
-	end
-
-	it "gzips and gunzips a string and returns the same string" do
-		Taps::Utils.gunzip(Taps::Utils.gzip("hello world")).should == "hello world"
-	end
-
 	it "generates a checksum using crc32" do
 		Taps::Utils.checksum("hello world").should == Zlib.crc32("hello world")
 	end
@@ -18,7 +9,7 @@ describe Taps::Utils do
 	it "formats a data hash into one hash that contains an array of headers and an array of array of data" do
 		first_row = { :x => 1, :y => 1 }
 		first_row.stubs(:keys).returns([:x, :y])
-		Taps::Utils.format_data([ first_row, { :x => 2, :y => 2 } ], []).should == { :header => [ :x, :y ], :data => [ [1, 1], [2, 2] ] }
+		Taps::Utils.format_data([ first_row, { :x => 2, :y => 2 } ]).should == { :header => [ :x, :y ], :data => [ [1, 1], [2, 2] ] }
 	end
 
 	it "scales chunksize down slowly when the time delta of the block is just over a second" do
@@ -53,8 +44,7 @@ describe Taps::Utils do
 	end
 
 	it "returns a list of columns that are text fields if the database is mysql" do
-		@db = mock("db")
-		@db.class.stubs(:to_s).returns("Sequel::MySQL::Database")
+		@db = mock("db", :url => "mysql://localhost/mydb")
 		@db.stubs(:schema).with(:mytable).returns([
 			[:id, { :db_type => "int" }],
 			[:mytext, { :db_type => "text" }]

metadata

@@ -4,109 +4,101 @@ version: !ruby/object:Gem::Version
   prerelease: false
   segments: 
   - 0
-  - 2
-  - 26
-  version: 0.2.26
+  - 3
+  - 0
+  version: 0.3.0
 platform: ruby
 authors: 
 - Ricardo Chimal, Jr.
-- Adam Wiggins
 autorequire: 
 bindir: bin
 cert_chain: []
 
-date: 2010-02-25 00:00:00 -08:00
+date: 2010-04-14 00:00:00 -07:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
-  name: sinatra
+  name: json_pure
   prerelease: false
   requirement: &id001 !ruby/object:Gem::Requirement 
     requirements: 
-    - - "="
+    - - ~>
       - !ruby/object:Gem::Version 
         segments: 
-        - 0
-        - 9
+        - 1
         - 2
-        version: 0.9.2
+        - 0
+        version: 1.2.0
   type: :runtime
   version_requirements: *id001
 - !ruby/object:Gem::Dependency 
-  name: activerecord
+  name: sinatra
   prerelease: false
   requirement: &id002 !ruby/object:Gem::Requirement 
     requirements: 
-    - - "="
+    - - ~>
       - !ruby/object:Gem::Version 
         segments: 
-        - 2
-        - 2
-        - 2
-        version: 2.2.2
+        - 1
+        - 0
+        - 0
+        version: 1.0.0
   type: :runtime
   version_requirements: *id002
 - !ruby/object:Gem::Dependency 
-  name: thor
+  name: rest-client
   prerelease: false
   requirement: &id003 !ruby/object:Gem::Requirement 
     requirements: 
-    - - "="
+    - - ~>
       - !ruby/object:Gem::Version 
         segments: 
+        - 1
+        - 4
         - 0
-        - 9
-        - 9
-        version: 0.9.9
+        version: 1.4.0
   type: :runtime
   version_requirements: *id003
 - !ruby/object:Gem::Dependency 
-  name: rest-client
+  name: sequel
   prerelease: false
   requirement: &id004 !ruby/object:Gem::Requirement 
     requirements: 
     - - ~>
       - !ruby/object:Gem::Version 
         segments: 
-        - 1
         - 3
+        - 10
         - 0
-        version: 1.3.0
+        version: 3.10.0
   type: :runtime
   version_requirements: *id004
 - !ruby/object:Gem::Dependency 
-  name: sequel
+  name: sqlite3-ruby
   prerelease: false
   requirement: &id005 !ruby/object:Gem::Requirement 
     requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        segments: 
-        - 3
-        - 0
-        - 0
-        version: 3.0.0
-    - - <
+    - - ~>
       - !ruby/object:Gem::Version 
         segments: 
-        - 3
         - 1
+        - 2
         - 0
-        version: 3.1.0
+        version: 1.2.0
   type: :runtime
   version_requirements: *id005
 - !ruby/object:Gem::Dependency 
-  name: sqlite3-ruby
+  name: rack
   prerelease: false
   requirement: &id006 !ruby/object:Gem::Requirement 
     requirements: 
-    - - ~>
+    - - ">="
       - !ruby/object:Gem::Version 
         segments: 
         - 1
-        - 2
         - 0
-        version: 1.2.0
+        - 1
+        version: 1.0.1
   type: :runtime
   version_requirements: *id006
 description: A simple database agnostic import/export app to transfer data to/from a remote database.
@@ -127,22 +119,21 @@ files:
 - bin/schema
 - bin/schema.cmd
 - bin/taps
-- lib/taps/adapter_hacks.rb
-- lib/taps/adapter_hacks/invalid_binary_limit.rb
-- lib/taps/adapter_hacks/invalid_text_limit.rb
-- lib/taps/adapter_hacks/mysql_invalid_primary_key.rb
-- lib/taps/adapter_hacks/non_rails_schema_dump.rb
 - lib/taps/cli.rb
-- lib/taps/client_session.rb
 - lib/taps/config.rb
+- lib/taps/data_stream.rb
 - lib/taps/db_session.rb
+- lib/taps/log.rb
+- lib/taps/monkey.rb
+- lib/taps/multipart.rb
+- lib/taps/operation.rb
 - lib/taps/progress_bar.rb
 - lib/taps/schema.rb
 - lib/taps/server.rb
 - lib/taps/utils.rb
 - spec/base.rb
-- spec/client_session_spec.rb
-- spec/schema_spec.rb
+- spec/data_stream_spec.rb
+- spec/operation_spec.rb
 - spec/server_spec.rb
 - spec/utils_spec.rb
 has_rdoc: true
@@ -176,8 +167,8 @@ signing_key:
 specification_version: 3
 summary: simple database import/export app
 test_files: 
+- spec/utils_spec.rb
+- spec/operation_spec.rb
+- spec/data_stream_spec.rb
 - spec/base.rb
-- spec/client_session_spec.rb
-- spec/schema_spec.rb
 - spec/server_spec.rb
-- spec/utils_spec.rb

data/lib/taps/adapter_hacks.rb

@@ -1,21 +0,0 @@
-module Taps
-	module AdapterHacks
-		extend self
-
-		LIST = {
-			:all => ['non_rails_schema_dump'],
-			:mysql => ['invalid_text_limit', 'mysql_invalid_primary_key'],
-			:postgresql => ['invalid_text_limit', 'invalid_binary_limit']
-		}
-
-		def load(adapter)
-			LIST[:all].each do |r|
-				require File.dirname(__FILE__) + "/adapter_hacks/#{r}"
-			end
-
-			(LIST[adapter.to_sym] || []).each do |r|
-				require File.dirname(__FILE__) + "/adapter_hacks/#{r}"
-			end
-		end
-	end
-end

data/lib/taps/adapter_hacks/invalid_binary_limit.rb

@@ -1,13 +0,0 @@
-module ActiveRecord
-	module ConnectionAdapters
-		class TableDefinition
-			alias_method :original_binary, :binary
-			def binary(*args)
-				options = args.extract_options!
-				options.delete(:limit)
-				column_names = args
-				column_names.each { |name| column(name, 'binary', options) }
-			end
-		end
-	end
-end

data/lib/taps/adapter_hacks/invalid_text_limit.rb

@@ -1,13 +0,0 @@
-module ActiveRecord
-	module ConnectionAdapters
-		class TableDefinition
-			alias_method :original_text, :text
-			def text(*args)
-				options = args.extract_options!
-				options.delete(:limit)
-				column_names = args
-				column_names.each { |name| column(name, 'text', options) }
-			end
-		end
-	end
-end

data/lib/taps/adapter_hacks/mysql_invalid_primary_key.rb

@@ -1,17 +0,0 @@
-module ActiveRecord
-	module ConnectionAdapters
-		class MysqlAdapter < AbstractAdapter
-			alias_method :orig_pk_and_sequence_for, :pk_and_sequence_for
-			# mysql accepts varchar as a primary key but most others do not.
-			# only say that a field is a primary key if mysql says so
-			# and the field is a kind of integer
-			def pk_and_sequence_for(table)
-				keys = []
-				execute("describe #{quote_table_name(table)}").each_hash do |h|
-					keys << h["Field"] if h["Key"] == "PRI" and !(h["Type"] =~ /int/).nil?
-				end
-				keys.length == 1 ? [keys.first, nil] : nil
-			end
-		end
-	end
-end

data/lib/taps/adapter_hacks/non_rails_schema_dump.rb

@@ -1,15 +0,0 @@
-module ActiveRecord
-	class SchemaDumper
-		private
-
-		def header(stream)
-			stream.puts "ActiveRecord::Schema.define do"
-		end
-
-		def tables(stream)
-			@connection.tables.sort.each do |tbl|
-				table(tbl, stream)
-			end
-		end
-	end
-end

data/lib/taps/client_session.rb

@@ -1,304 +0,0 @@
-require 'rest_client'
-require 'sequel'
-require 'zlib'
-
-require File.dirname(__FILE__) + '/progress_bar'
-require File.dirname(__FILE__) + '/config'
-require File.dirname(__FILE__) + '/utils'
-
-module Taps
-class ClientSession
-	attr_reader :database_url, :remote_url, :default_chunksize
-
-	def initialize(database_url, remote_url, default_chunksize)
-		@database_url = database_url
-		@remote_url = remote_url
-		@default_chunksize = default_chunksize
-	end
-
-	def self.start(database_url, remote_url, default_chunksize, &block)
-		s = new(database_url, remote_url, default_chunksize)
-		yield s
-		s.close_session
-	end
-
-	def self.quickstart(&block)
-		start(Taps::Config.database_url, Taps::Config.remote_url, Taps::Config.chunksize) do |s|
-			yield s
-		end
-	end
-
-	def db
-		@db ||= Sequel.connect(database_url)
-	end
-
-	def server
-		@server ||= RestClient::Resource.new(remote_url)
-	end
-
-	def session_resource
-		@session_resource ||= open_session
-	end
-
-	def open_session
-		uri = server['sessions'].post('', http_headers)
-		server[uri]
-	end
-
-	def set_session(uri)
-		@session_resource = server[uri]
-	end
-
-	def close_session
-		@session_resource.delete(http_headers) if @session_resource
-	end
-
-	def safe_url(url)
-		url.sub(/\/\/(.+?)?:(.*?)@/, '//\1:[hidden]@')
-	end
-
-	def safe_remote_url
-		safe_url(remote_url)
-	end
-
-	def safe_database_url
-		safe_url(database_url)
-	end
-
-	def http_headers(extra = {})
-		{ :taps_version => Taps.compatible_version }.merge(extra)
-	end
-
-	def cmd_send
-		begin
-			verify_server
-			cmd_send_schema
-			cmd_send_data
-			cmd_send_indexes
-			cmd_send_reset_sequences
-		rescue RestClient::Exception => e
-			if e.respond_to?(:response)
-				puts "!!! Caught Server Exception"
-				puts "#{e.response}"
-				exit(1)
-			else
-				raise
-			end
-		end
-	end
-
-	def cmd_send_indexes
-		puts "Sending indexes"
-
-		index_data = Taps::Utils.schema_bin(:indexes, database_url)
-		session_resource['indexes'].post(index_data, http_headers)
-	end
-
-	def cmd_send_schema
-		puts "Sending schema"
-
-		schema_data = Taps::Utils.schema_bin(:dump, database_url)
-		session_resource['schema'].post(schema_data, http_headers)
-	end
-
-	def cmd_send_reset_sequences
-		puts "Resetting sequences"
-
-		session_resource["reset_sequences"].post('', http_headers)
-	end
-
-	def cmd_send_data
-		puts "Sending data"
-
-		tables_with_counts, record_count = fetch_tables_info
-
-		puts "#{tables_with_counts.size} tables, #{format_number(record_count)} records"
-
-		tables_with_counts.each do |table_name, count|
-			table = db[table_name]
-			order = Taps::Utils.order_by(db, table_name)
-			chunksize = self.default_chunksize
-			string_columns = Taps::Utils.incorrect_blobs(db, table_name)
-
-			progress = ProgressBar.new(table_name.to_s, count)
-
-			offset = 0
-			loop do
-				row_size = 0
-				chunksize = Taps::Utils.calculate_chunksize(chunksize) do |c|
-					time_skip_start = Time.now
-					rows = Taps::Utils.format_data(table.order(*order).limit(c, offset).all, string_columns)
-					break if rows == { }
-
-					row_size = rows[:data].size
-					gzip_data = Taps::Utils.gzip(Marshal.dump(rows))
-					time_skip = Time.now - time_skip_start
-
-					begin
-						session_resource["tables/#{table_name}"].post(gzip_data, http_headers({
-							:content_type => 'application/octet-stream',
-							:taps_checksum => Taps::Utils.checksum(gzip_data).to_s}))
-					rescue RestClient::RequestFailed => e
-						# retry the same data, it got corrupted somehow.
-						if e.http_code == 412
-							next
-						end
-						raise
-					end
-					time_skip
-				end
-
-				progress.inc(row_size)
-				offset += row_size
-
-				break if row_size == 0
-			end
-
-			progress.finish
-		end
-	end
-
-	def fetch_tables_info
-		tables_with_counts = {}
-		record_count = db.tables.inject(0) do |record_count, table|
-			tables_with_counts[table] = db[table].count
-			record_count += tables_with_counts[table]
-		end
-
-		[ tables_with_counts, record_count ]
-	end
-
-	def cmd_receive
-		begin
-			verify_server
-			cmd_receive_schema
-			cmd_receive_data
-			cmd_receive_indexes
-			cmd_reset_sequences
-		rescue RestClient::Exception => e
-			if e.respond_to?(:response)
-				puts "!!! Caught Server Exception"
-				puts "#{e.response.body}"
-				exit(1)
-			else
-				raise
-			end
-		end
-	end
-
-	def cmd_receive_data
-		puts "Receiving data"
-
-		tables_with_counts, record_count = fetch_remote_tables_info
-
-		puts "#{tables_with_counts.size} tables, #{format_number(record_count)} records"
-
-		tables_with_counts.each do |table_name, count|
-			table = db[table_name.to_sym]
-			chunksize = default_chunksize
-
-			progress = ProgressBar.new(table_name.to_s, count)
-
-			offset = 0
-			loop do
-				begin
-					chunksize, rows = fetch_table_rows(table_name, chunksize, offset)
-				rescue CorruptedData
-					next
-				end
-				break if rows == { }
-
-				table.import(rows[:header], rows[:data])
-
-				progress.inc(rows[:data].size)
-				offset += rows[:data].size
-			end
-
-			progress.finish
-		end
-	end
-
-	class CorruptedData < Exception; end
-
-	def fetch_table_rows(table_name, chunksize, offset)
-		response = nil
-		chunksize = Taps::Utils.calculate_chunksize(chunksize) do |c|
-			response = session_resource["tables/#{table_name}/#{c}?offset=#{offset}"].get(http_headers)
-		end
-		raise CorruptedData unless Taps::Utils.valid_data?(response.to_s, response.headers[:taps_checksum])
-
-		begin
-			rows = Marshal.load(Taps::Utils.gunzip(response.to_s))
-		rescue Object => e
-			puts "Error encountered loading data, wrote the data chunk to dump.#{Process.pid}.gz"
-			File.open("dump.#{Process.pid}.gz", "w") { |f| f.write(response.to_s) }
-			raise
-		end
-		[chunksize, rows]
-	end
-
-	def fetch_remote_tables_info
-		retries = 0
-		max_retries = 1
-		begin
-			tables_with_counts = Marshal.load(session_resource['tables'].get(http_headers))
-			record_count = tables_with_counts.values.inject(0) { |a,c| a += c }
-		rescue RestClient::Exception
-			retries += 1
-			retry if retries <= max_retries
-			puts "Unable to fetch tables information from #{remote_url}. Please check the server log."
-			exit(1)
-		end
-
-		[ tables_with_counts, record_count ]
-	end
-
-	def cmd_receive_schema
-		puts "Receiving schema"
-
-		schema_data = session_resource['schema'].get(http_headers)
-		output = Taps::Utils.load_schema(database_url, schema_data)
-		puts output if output
-	end
-
-	def cmd_receive_indexes
-		puts "Receiving indexes"
-
-		index_data = session_resource['indexes'].get(http_headers)
-
-		output = Taps::Utils.load_indexes(database_url, index_data)
-		puts output if output
-	end
-
-	def cmd_reset_sequences
-		puts "Resetting sequences"
-
-		output = Taps::Utils.schema_bin(:reset_db_sequences, database_url)
-		puts output if output
-	end
-
-	def format_number(num)
-		num.to_s.gsub(/(\d)(?=(\d\d\d)+(?!\d))/, "\\1,")
-	end
-
-	def verify_server
-		begin
-			server['/'].get(http_headers)
-		rescue RestClient::RequestFailed => e
-			if e.http_code == 417
-				puts "#{safe_remote_url} is running a different minor version of taps."
-				puts "#{e.response.body}"
-				exit(1)
-			else
-				raise
-			end
-		rescue RestClient::Unauthorized
-			puts "Bad credentials given for #{safe_remote_url}"
-			exit(1)
-		rescue Errno::ECONNREFUSED
-			puts "Can't connect to #{safe_remote_url}. Please check that it's running"
-			exit(1)
-		end
-	end
-end
-end