taps 0.2.1

Sign up to get free protection for your applications and to get access to all the features.
@@ -0,0 +1,76 @@
1
+ require 'zlib'
2
+ require 'stringio'
3
+ require 'time'
4
+ require 'tempfile'
5
+
6
+ module Taps
7
+ module Utils
8
+ extend self
9
+
10
+ def checksum(data)
11
+ Zlib.crc32(data)
12
+ end
13
+
14
+ def valid_data?(data, crc32)
15
+ Zlib.crc32(data) == crc32.to_i
16
+ end
17
+
18
+ def gzip(data)
19
+ io = StringIO.new
20
+ gz = Zlib::GzipWriter.new(io)
21
+ gz.write data
22
+ gz.close
23
+ io.string
24
+ end
25
+
26
+ def gunzip(gzip_data)
27
+ io = StringIO.new(gzip_data)
28
+ gz = Zlib::GzipReader.new(io)
29
+ data = gz.read
30
+ gz.close
31
+ data
32
+ end
33
+
34
+ def format_data(data)
35
+ return {} if data.size == 0
36
+ header = data[0].keys
37
+ only_data = data.collect do |row|
38
+ header.collect { |h| row[h] }
39
+ end
40
+ { :header => header, :data => only_data }
41
+ end
42
+
43
+ def calculate_chunksize(old_chunksize)
44
+ t1 = Time.now
45
+ yield
46
+ t2 = Time.now
47
+
48
+ diff = t2 - t1
49
+ new_chunksize = if diff > 3.0
50
+ (old_chunksize / 3).ceil
51
+ elsif diff > 1.1
52
+ old_chunksize - 100
53
+ elsif diff < 0.8
54
+ old_chunksize * 2
55
+ else
56
+ old_chunksize + 100
57
+ end
58
+ new_chunksize = 100 if new_chunksize < 100
59
+ new_chunksize
60
+ end
61
+
62
+ def load_schema(database_url, schema_data)
63
+ Tempfile.open('taps') do |tmp|
64
+ File.open(tmp.path, 'w') { |f| f.write(schema_data) }
65
+ `#{File.dirname(__FILE__)}/../../bin/schema load #{database_url} #{tmp.path}`
66
+ end
67
+ end
68
+
69
+ def load_indexes(database_url, index_data)
70
+ Tempfile.open('taps') do |tmp|
71
+ File.open(tmp.path, 'w') { |f| f.write(index_data) }
72
+ `#{File.dirname(__FILE__)}/../../bin/schema load_indexes #{database_url} #{tmp.path}`
73
+ end
74
+ end
75
+ end
76
+ end
@@ -0,0 +1,21 @@
1
+ require 'rubygems'
2
+ require 'bacon'
3
+ require 'mocha'
4
+
5
+ class Bacon::Context
6
+ include Mocha::Standalone
7
+
8
+ alias_method :old_it, :it
9
+ def it(description)
10
+ old_it(description) do
11
+ mocha_setup
12
+ yield
13
+ mocha_verify
14
+ mocha_teardown
15
+ end
16
+ end
17
+ end
18
+
19
+ require File.dirname(__FILE__) + '/../lib/taps/config'
20
+ Taps::Config.taps_database_url = 'sqlite://test.db'
21
+ Sequel.connect(Taps::Config.taps_database_url)
@@ -0,0 +1,82 @@
1
+ require File.dirname(__FILE__) + '/base'
2
+ require File.dirname(__FILE__) + '/../lib/taps/client_session'
3
+
4
+ describe Taps::ClientSession do
5
+ before do
6
+ @client = Taps::ClientSession.new('sqlite://my.db', 'http://example.com:3000', 1000)
7
+ @client.stubs(:session_resource).returns(mock('session resource'))
8
+ end
9
+
10
+ it "starts a session and yields the session object to the block" do
11
+ Taps::ClientSession.start('x', 'y', 1000) do |session|
12
+ session.database_url.should == 'x'
13
+ session.remote_url.should == 'y'
14
+ session.default_chunksize.should == 1000
15
+ end
16
+ end
17
+
18
+ it "opens the local db connection via sequel and the database url" do
19
+ Sequel.expects(:connect).with('sqlite://my.db').returns(:con)
20
+ @client.db.should == :con
21
+ end
22
+
23
+ it "creates a restclient resource to the remote server" do
24
+ @client.server.url.should == 'http://example.com:3000'
25
+ end
26
+
27
+ it "verifies the db version, receive the schema, data, indexes, then reset the sequences" do
28
+ @client.expects(:verify_server)
29
+ @client.expects(:cmd_receive_schema)
30
+ @client.expects(:cmd_receive_data)
31
+ @client.expects(:cmd_receive_indexes)
32
+ @client.expects(:cmd_reset_sequences)
33
+ @client.cmd_receive.should.be.nil
34
+ end
35
+
36
+ it "checks the version of the server by seeing if it has access" do
37
+ @client.stubs(:server).returns(mock('server'))
38
+ @request = mock('request')
39
+ @client.server.expects(:[]).with('/').returns(@request)
40
+ @request.expects(:get).with({:taps_version => Taps::VERSION})
41
+
42
+ lambda { @client.verify_server }.should.not.raise
43
+ end
44
+
45
+ it "receives data from a remote taps server" do
46
+ @client.stubs(:puts)
47
+ @progressbar = mock('progressbar')
48
+ ProgressBar.stubs(:new).with('mytable', 2).returns(@progressbar)
49
+ @progressbar.stubs(:inc)
50
+ @progressbar.stubs(:finish)
51
+ @mytable = mock('mytable')
52
+ @client.expects(:fetch_tables_info).returns([ { :mytable => 2 }, 2 ])
53
+ @client.stubs(:db).returns(mock('db'))
54
+ @client.db.stubs(:[]).with(:mytable).returns(@mytable)
55
+ @client.expects(:fetch_table_rows).with(:mytable, 1000, 0).returns([ 1000, { :header => [:x, :y], :data => [[1, 2], [3, 4]] } ])
56
+ @client.expects(:fetch_table_rows).with(:mytable, 1000, 2).returns([ 1000, { }])
57
+ @mytable.expects(:multi_insert).with([:x, :y], [[1, 2], [3, 4]])
58
+
59
+ lambda { @client.cmd_receive_data }.should.not.raise
60
+ end
61
+
62
+ it "fetches tables info from taps server" do
63
+ @marshal_data = Marshal.dump({ :mytable => 2 })
64
+ @client.session_resource.stubs(:[]).with('tables').returns(mock('tables'))
65
+ @client.session_resource['tables'].stubs(:get).with(:taps_version => Taps::VERSION).returns(@marshal_data)
66
+ @client.fetch_tables_info.should == [ { :mytable => 2 }, 2 ]
67
+ end
68
+
69
+ it "fetches table rows given a chunksize and offset from taps server" do
70
+ @data = { :header => [ :x, :y ], :data => [ [1, 2], [3, 4] ] }
71
+ @gzip_data = Taps::Utils.gzip(Marshal.dump(@data))
72
+ Taps::Utils.stubs(:calculate_chunksize).with(1000).yields.returns(1000)
73
+
74
+ @response = mock('response')
75
+ @client.session_resource.stubs(:[]).with('tables/mytable/1000?offset=0').returns(mock('table resource'))
76
+ @client.session_resource['tables/mytable/1000?offset=0'].expects(:get).with(:taps_version => Taps::VERSION).returns(@response)
77
+ @response.stubs(:to_s).returns(@gzip_data)
78
+ @response.stubs(:headers).returns({ :taps_checksum => Taps::Utils.checksum(@gzip_data) })
79
+ @client.fetch_table_rows('mytable', 1000, 0).should == [ 1000, { :header => [:x, :y], :data => [[1, 2], [3, 4]] } ]
80
+ end
81
+ end
82
+
@@ -0,0 +1,38 @@
1
+ require File.dirname(__FILE__) + '/base'
2
+ require File.dirname(__FILE__) + '/../lib/taps/schema'
3
+
4
+ describe Taps::Schema do
5
+ before do
6
+ @connection = mock("AR connection")
7
+ ActiveRecord::Base.stubs(:connection).returns(@connection)
8
+ end
9
+
10
+ it "parses a database url and returns a config hash for activerecord" do
11
+ Taps::Schema.create_config("postgres://myuser:mypass@localhost/mydb").should == {
12
+ 'adapter' => 'postgresql',
13
+ 'database' => 'mydb',
14
+ 'username' => 'myuser',
15
+ 'password' => 'mypass',
16
+ 'host' => 'localhost'
17
+ }
18
+ end
19
+
20
+ it "translates sqlite in the database url to sqlite3" do
21
+ Taps::Schema.create_config("sqlite://myuser:mypass@localhost/mydb")['adapter'].should == 'sqlite3'
22
+ end
23
+
24
+ it "connects activerecord to the database" do
25
+ Taps::Schema.expects(:create_config).with("postgres://myuser:mypass@localhost/mydb").returns("db config")
26
+ ActiveRecord::Base.expects(:establish_connection).with("db config").returns(true)
27
+ Taps::Schema.connection("postgres://myuser:mypass@localhost/mydb").should == true
28
+ end
29
+
30
+ it "resets the db tables' primary keys" do
31
+ Taps::Schema.stubs(:connection)
32
+ ActiveRecord::Base.connection.expects(:respond_to?).with(:reset_pk_sequence!).returns(true)
33
+ ActiveRecord::Base.connection.stubs(:tables).returns(['table1'])
34
+ ActiveRecord::Base.connection.expects(:reset_pk_sequence!).with('table1')
35
+ should.not.raise { Taps::Schema.reset_db_sequences("postgres://myuser:mypass@localhost/mydb") }
36
+ end
37
+ end
38
+
@@ -0,0 +1,33 @@
1
+ require File.dirname(__FILE__) + '/base'
2
+ require 'sinatra'
3
+ require 'sinatra/test/bacon'
4
+
5
+ require File.dirname(__FILE__) + '/../lib/taps/server'
6
+
7
+ require 'pp'
8
+
9
+ describe Taps::Server do
10
+ before do
11
+ Taps::Config.login = 'taps'
12
+ Taps::Config.password = 'tpass'
13
+
14
+ @app = Taps::Server
15
+ @auth_header = "Basic " + ["taps:tpass"].pack("m*")
16
+ end
17
+
18
+ it "asks for http basic authentication" do
19
+ get '/'
20
+ status.should == 401
21
+ end
22
+
23
+ it "verifies the client taps version" do
24
+ get('/', { }, { 'HTTP_AUTHORIZATION' => @auth_header, 'HTTP_TAPS_VERSION' => Taps::VERSION })
25
+ status.should == 200
26
+ end
27
+
28
+ it "yells loudly if the client taps version doesn't match" do
29
+ get('/', { }, { 'HTTP_AUTHORIZATION' => @auth_header, 'HTTP_TAPS_VERSION' => '0.0.1' })
30
+ status.should == 417
31
+ end
32
+ end
33
+
@@ -0,0 +1,45 @@
1
+ require File.dirname(__FILE__) + '/base'
2
+ require File.dirname(__FILE__) + '/../lib/taps/utils'
3
+
4
+ describe Taps::Utils do
5
+ it "gunzips a string" do
6
+ @hello_world = "\037\213\b\000R\261\207I\000\003\313H\315\311\311W(\317/\312I\001\000\205\021J\r\v\000\000\000"
7
+ Taps::Utils.gunzip(@hello_world).should == "hello world"
8
+ end
9
+
10
+ it "gzips and gunzips a string and returns the same string" do
11
+ Taps::Utils.gunzip(Taps::Utils.gzip("hello world")).should == "hello world"
12
+ end
13
+
14
+ it "generates a checksum using crc32" do
15
+ Taps::Utils.checksum("hello world").should == Zlib.crc32("hello world")
16
+ end
17
+
18
+ it "formats a data hash into one hash that contains an array of headers and an array of array of data" do
19
+ Taps::Utils.format_data([ { :x => 1, :y => 1 }, { :x => 2, :y => 2 } ]).should == { :header => [ :x, :y ], :data => [ [1, 1], [2, 2] ] }
20
+ end
21
+
22
+ it "scales chunksize down slowly when the time delta of the block is just over a second" do
23
+ Time.stubs(:now).returns(10.0).returns(11.5)
24
+ Taps::Utils.calculate_chunksize(1000) { }.should == 900
25
+ end
26
+
27
+ it "scales chunksize down fast when the time delta of the block is over 3 seconds" do
28
+ Time.stubs(:now).returns(10.0).returns(15.0)
29
+ Taps::Utils.calculate_chunksize(3000) { }.should == 1000
30
+ end
31
+
32
+ it "scales up chunksize fast when the time delta of the block is under 0.8 seconds" do
33
+ Time.stubs(:now).returns(10.0).returns(10.7)
34
+ Taps::Utils.calculate_chunksize(1000) { }.should == 2000
35
+ end
36
+
37
+ it "scales up chunksize slow when the time delta of the block is between 0.8 and 1.1 seconds" do
38
+ Time.stubs(:now).returns(10.0).returns(10.8)
39
+ Taps::Utils.calculate_chunksize(1000) { }.should == 1100
40
+
41
+ Time.stubs(:now).returns(10.0).returns(11.1)
42
+ Taps::Utils.calculate_chunksize(1000) { }.should == 1100
43
+ end
44
+ end
45
+
metadata ADDED
@@ -0,0 +1,133 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: taps
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.2.1
5
+ platform: ruby
6
+ authors:
7
+ - Ricardo Chimal, Jr.
8
+ - Adam Wiggins
9
+ autorequire:
10
+ bindir: bin
11
+ cert_chain: []
12
+
13
+ date: 2009-02-10 00:00:00 -08:00
14
+ default_executable:
15
+ dependencies:
16
+ - !ruby/object:Gem::Dependency
17
+ name: sinatra
18
+ type: :runtime
19
+ version_requirement:
20
+ version_requirements: !ruby/object:Gem::Requirement
21
+ requirements:
22
+ - - ~>
23
+ - !ruby/object:Gem::Version
24
+ version: 0.9.0
25
+ version:
26
+ - !ruby/object:Gem::Dependency
27
+ name: activerecord
28
+ type: :runtime
29
+ version_requirement:
30
+ version_requirements: !ruby/object:Gem::Requirement
31
+ requirements:
32
+ - - "="
33
+ - !ruby/object:Gem::Version
34
+ version: 2.2.2
35
+ version:
36
+ - !ruby/object:Gem::Dependency
37
+ name: thor
38
+ type: :runtime
39
+ version_requirement:
40
+ version_requirements: !ruby/object:Gem::Requirement
41
+ requirements:
42
+ - - "="
43
+ - !ruby/object:Gem::Version
44
+ version: 0.9.9
45
+ version:
46
+ - !ruby/object:Gem::Dependency
47
+ name: rest-client
48
+ type: :runtime
49
+ version_requirement:
50
+ version_requirements: !ruby/object:Gem::Requirement
51
+ requirements:
52
+ - - ~>
53
+ - !ruby/object:Gem::Version
54
+ version: 0.9.0
55
+ version:
56
+ - !ruby/object:Gem::Dependency
57
+ name: sequel
58
+ type: :runtime
59
+ version_requirement:
60
+ version_requirements: !ruby/object:Gem::Requirement
61
+ requirements:
62
+ - - ~>
63
+ - !ruby/object:Gem::Version
64
+ version: 2.10.0
65
+ version:
66
+ - !ruby/object:Gem::Dependency
67
+ name: sqlite3-ruby
68
+ type: :runtime
69
+ version_requirement:
70
+ version_requirements: !ruby/object:Gem::Requirement
71
+ requirements:
72
+ - - ~>
73
+ - !ruby/object:Gem::Version
74
+ version: 1.2.0
75
+ version:
76
+ description: A simple database agnostic import/export app to transfer data to/from a remote database.
77
+ email: ricardo@heroku.com
78
+ executables:
79
+ - taps
80
+ - schema
81
+ extensions: []
82
+
83
+ extra_rdoc_files: []
84
+
85
+ files:
86
+ - spec/base.rb
87
+ - spec/client_session_spec.rb
88
+ - spec/schema_spec.rb
89
+ - spec/server_spec.rb
90
+ - spec/utils_spec.rb
91
+ - lib/taps/db_session.rb
92
+ - lib/taps/progress_bar.rb
93
+ - lib/taps/utils.rb
94
+ - lib/taps/schema.rb
95
+ - lib/taps/cli.rb
96
+ - lib/taps/client_session.rb
97
+ - lib/taps/server.rb
98
+ - lib/taps/config.rb
99
+ - README.rdoc
100
+ - LICENSE
101
+ - VERSION.yml
102
+ - Rakefile
103
+ - bin/taps
104
+ - bin/schema
105
+ has_rdoc: true
106
+ homepage: http://github.com/ricardochimal/taps
107
+ post_install_message:
108
+ rdoc_options:
109
+ - --inline-source
110
+ - --charset=UTF-8
111
+ require_paths:
112
+ - lib
113
+ required_ruby_version: !ruby/object:Gem::Requirement
114
+ requirements:
115
+ - - ">="
116
+ - !ruby/object:Gem::Version
117
+ version: "0"
118
+ version:
119
+ required_rubygems_version: !ruby/object:Gem::Requirement
120
+ requirements:
121
+ - - ">="
122
+ - !ruby/object:Gem::Version
123
+ version: "0"
124
+ version:
125
+ requirements: []
126
+
127
+ rubyforge_project: taps
128
+ rubygems_version: 1.3.1
129
+ signing_key:
130
+ specification_version: 2
131
+ summary: simple database import/export app
132
+ test_files: []
133
+