taps 0.2.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,76 @@
1
+ require 'zlib'
2
+ require 'stringio'
3
+ require 'time'
4
+ require 'tempfile'
5
+
6
+ module Taps
7
+ module Utils
8
+ extend self
9
+
10
+ def checksum(data)
11
+ Zlib.crc32(data)
12
+ end
13
+
14
+ def valid_data?(data, crc32)
15
+ Zlib.crc32(data) == crc32.to_i
16
+ end
17
+
18
+ def gzip(data)
19
+ io = StringIO.new
20
+ gz = Zlib::GzipWriter.new(io)
21
+ gz.write data
22
+ gz.close
23
+ io.string
24
+ end
25
+
26
+ def gunzip(gzip_data)
27
+ io = StringIO.new(gzip_data)
28
+ gz = Zlib::GzipReader.new(io)
29
+ data = gz.read
30
+ gz.close
31
+ data
32
+ end
33
+
34
+ def format_data(data)
35
+ return {} if data.size == 0
36
+ header = data[0].keys
37
+ only_data = data.collect do |row|
38
+ header.collect { |h| row[h] }
39
+ end
40
+ { :header => header, :data => only_data }
41
+ end
42
+
43
+ def calculate_chunksize(old_chunksize)
44
+ t1 = Time.now
45
+ yield
46
+ t2 = Time.now
47
+
48
+ diff = t2 - t1
49
+ new_chunksize = if diff > 3.0
50
+ (old_chunksize / 3).ceil
51
+ elsif diff > 1.1
52
+ old_chunksize - 100
53
+ elsif diff < 0.8
54
+ old_chunksize * 2
55
+ else
56
+ old_chunksize + 100
57
+ end
58
+ new_chunksize = 100 if new_chunksize < 100
59
+ new_chunksize
60
+ end
61
+
62
+ def load_schema(database_url, schema_data)
63
+ Tempfile.open('taps') do |tmp|
64
+ File.open(tmp.path, 'w') { |f| f.write(schema_data) }
65
+ `#{File.dirname(__FILE__)}/../../bin/schema load #{database_url} #{tmp.path}`
66
+ end
67
+ end
68
+
69
+ def load_indexes(database_url, index_data)
70
+ Tempfile.open('taps') do |tmp|
71
+ File.open(tmp.path, 'w') { |f| f.write(index_data) }
72
+ `#{File.dirname(__FILE__)}/../../bin/schema load_indexes #{database_url} #{tmp.path}`
73
+ end
74
+ end
75
+ end
76
+ end
@@ -0,0 +1,21 @@
1
+ require 'rubygems'
2
+ require 'bacon'
3
+ require 'mocha'
4
+
5
+ class Bacon::Context
6
+ include Mocha::Standalone
7
+
8
+ alias_method :old_it, :it
9
+ def it(description)
10
+ old_it(description) do
11
+ mocha_setup
12
+ yield
13
+ mocha_verify
14
+ mocha_teardown
15
+ end
16
+ end
17
+ end
18
+
19
+ require File.dirname(__FILE__) + '/../lib/taps/config'
20
+ Taps::Config.taps_database_url = 'sqlite://test.db'
21
+ Sequel.connect(Taps::Config.taps_database_url)
@@ -0,0 +1,82 @@
1
+ require File.dirname(__FILE__) + '/base'
2
+ require File.dirname(__FILE__) + '/../lib/taps/client_session'
3
+
4
+ describe Taps::ClientSession do
5
+ before do
6
+ @client = Taps::ClientSession.new('sqlite://my.db', 'http://example.com:3000', 1000)
7
+ @client.stubs(:session_resource).returns(mock('session resource'))
8
+ end
9
+
10
+ it "starts a session and yields the session object to the block" do
11
+ Taps::ClientSession.start('x', 'y', 1000) do |session|
12
+ session.database_url.should == 'x'
13
+ session.remote_url.should == 'y'
14
+ session.default_chunksize.should == 1000
15
+ end
16
+ end
17
+
18
+ it "opens the local db connection via sequel and the database url" do
19
+ Sequel.expects(:connect).with('sqlite://my.db').returns(:con)
20
+ @client.db.should == :con
21
+ end
22
+
23
+ it "creates a restclient resource to the remote server" do
24
+ @client.server.url.should == 'http://example.com:3000'
25
+ end
26
+
27
+ it "verifies the db version, receive the schema, data, indexes, then reset the sequences" do
28
+ @client.expects(:verify_server)
29
+ @client.expects(:cmd_receive_schema)
30
+ @client.expects(:cmd_receive_data)
31
+ @client.expects(:cmd_receive_indexes)
32
+ @client.expects(:cmd_reset_sequences)
33
+ @client.cmd_receive.should.be.nil
34
+ end
35
+
36
+ it "checks the version of the server by seeing if it has access" do
37
+ @client.stubs(:server).returns(mock('server'))
38
+ @request = mock('request')
39
+ @client.server.expects(:[]).with('/').returns(@request)
40
+ @request.expects(:get).with({:taps_version => Taps::VERSION})
41
+
42
+ lambda { @client.verify_server }.should.not.raise
43
+ end
44
+
45
+ it "receives data from a remote taps server" do
46
+ @client.stubs(:puts)
47
+ @progressbar = mock('progressbar')
48
+ ProgressBar.stubs(:new).with('mytable', 2).returns(@progressbar)
49
+ @progressbar.stubs(:inc)
50
+ @progressbar.stubs(:finish)
51
+ @mytable = mock('mytable')
52
+ @client.expects(:fetch_tables_info).returns([ { :mytable => 2 }, 2 ])
53
+ @client.stubs(:db).returns(mock('db'))
54
+ @client.db.stubs(:[]).with(:mytable).returns(@mytable)
55
+ @client.expects(:fetch_table_rows).with(:mytable, 1000, 0).returns([ 1000, { :header => [:x, :y], :data => [[1, 2], [3, 4]] } ])
56
+ @client.expects(:fetch_table_rows).with(:mytable, 1000, 2).returns([ 1000, { }])
57
+ @mytable.expects(:multi_insert).with([:x, :y], [[1, 2], [3, 4]])
58
+
59
+ lambda { @client.cmd_receive_data }.should.not.raise
60
+ end
61
+
62
+ it "fetches tables info from taps server" do
63
+ @marshal_data = Marshal.dump({ :mytable => 2 })
64
+ @client.session_resource.stubs(:[]).with('tables').returns(mock('tables'))
65
+ @client.session_resource['tables'].stubs(:get).with(:taps_version => Taps::VERSION).returns(@marshal_data)
66
+ @client.fetch_tables_info.should == [ { :mytable => 2 }, 2 ]
67
+ end
68
+
69
+ it "fetches table rows given a chunksize and offset from taps server" do
70
+ @data = { :header => [ :x, :y ], :data => [ [1, 2], [3, 4] ] }
71
+ @gzip_data = Taps::Utils.gzip(Marshal.dump(@data))
72
+ Taps::Utils.stubs(:calculate_chunksize).with(1000).yields.returns(1000)
73
+
74
+ @response = mock('response')
75
+ @client.session_resource.stubs(:[]).with('tables/mytable/1000?offset=0').returns(mock('table resource'))
76
+ @client.session_resource['tables/mytable/1000?offset=0'].expects(:get).with(:taps_version => Taps::VERSION).returns(@response)
77
+ @response.stubs(:to_s).returns(@gzip_data)
78
+ @response.stubs(:headers).returns({ :taps_checksum => Taps::Utils.checksum(@gzip_data) })
79
+ @client.fetch_table_rows('mytable', 1000, 0).should == [ 1000, { :header => [:x, :y], :data => [[1, 2], [3, 4]] } ]
80
+ end
81
+ end
82
+
@@ -0,0 +1,38 @@
1
+ require File.dirname(__FILE__) + '/base'
2
+ require File.dirname(__FILE__) + '/../lib/taps/schema'
3
+
4
+ describe Taps::Schema do
5
+ before do
6
+ @connection = mock("AR connection")
7
+ ActiveRecord::Base.stubs(:connection).returns(@connection)
8
+ end
9
+
10
+ it "parses a database url and returns a config hash for activerecord" do
11
+ Taps::Schema.create_config("postgres://myuser:mypass@localhost/mydb").should == {
12
+ 'adapter' => 'postgresql',
13
+ 'database' => 'mydb',
14
+ 'username' => 'myuser',
15
+ 'password' => 'mypass',
16
+ 'host' => 'localhost'
17
+ }
18
+ end
19
+
20
+ it "translates sqlite in the database url to sqlite3" do
21
+ Taps::Schema.create_config("sqlite://myuser:mypass@localhost/mydb")['adapter'].should == 'sqlite3'
22
+ end
23
+
24
+ it "connects activerecord to the database" do
25
+ Taps::Schema.expects(:create_config).with("postgres://myuser:mypass@localhost/mydb").returns("db config")
26
+ ActiveRecord::Base.expects(:establish_connection).with("db config").returns(true)
27
+ Taps::Schema.connection("postgres://myuser:mypass@localhost/mydb").should == true
28
+ end
29
+
30
+ it "resets the db tables' primary keys" do
31
+ Taps::Schema.stubs(:connection)
32
+ ActiveRecord::Base.connection.expects(:respond_to?).with(:reset_pk_sequence!).returns(true)
33
+ ActiveRecord::Base.connection.stubs(:tables).returns(['table1'])
34
+ ActiveRecord::Base.connection.expects(:reset_pk_sequence!).with('table1')
35
+ should.not.raise { Taps::Schema.reset_db_sequences("postgres://myuser:mypass@localhost/mydb") }
36
+ end
37
+ end
38
+
@@ -0,0 +1,33 @@
1
+ require File.dirname(__FILE__) + '/base'
2
+ require 'sinatra'
3
+ require 'sinatra/test/bacon'
4
+
5
+ require File.dirname(__FILE__) + '/../lib/taps/server'
6
+
7
+ require 'pp'
8
+
9
+ describe Taps::Server do
10
+ before do
11
+ Taps::Config.login = 'taps'
12
+ Taps::Config.password = 'tpass'
13
+
14
+ @app = Taps::Server
15
+ @auth_header = "Basic " + ["taps:tpass"].pack("m*")
16
+ end
17
+
18
+ it "asks for http basic authentication" do
19
+ get '/'
20
+ status.should == 401
21
+ end
22
+
23
+ it "verifies the client taps version" do
24
+ get('/', { }, { 'HTTP_AUTHORIZATION' => @auth_header, 'HTTP_TAPS_VERSION' => Taps::VERSION })
25
+ status.should == 200
26
+ end
27
+
28
+ it "yells loudly if the client taps version doesn't match" do
29
+ get('/', { }, { 'HTTP_AUTHORIZATION' => @auth_header, 'HTTP_TAPS_VERSION' => '0.0.1' })
30
+ status.should == 417
31
+ end
32
+ end
33
+
@@ -0,0 +1,45 @@
1
+ require File.dirname(__FILE__) + '/base'
2
+ require File.dirname(__FILE__) + '/../lib/taps/utils'
3
+
4
+ describe Taps::Utils do
5
+ it "gunzips a string" do
6
+ @hello_world = "\037\213\b\000R\261\207I\000\003\313H\315\311\311W(\317/\312I\001\000\205\021J\r\v\000\000\000"
7
+ Taps::Utils.gunzip(@hello_world).should == "hello world"
8
+ end
9
+
10
+ it "gzips and gunzips a string and returns the same string" do
11
+ Taps::Utils.gunzip(Taps::Utils.gzip("hello world")).should == "hello world"
12
+ end
13
+
14
+ it "generates a checksum using crc32" do
15
+ Taps::Utils.checksum("hello world").should == Zlib.crc32("hello world")
16
+ end
17
+
18
+ it "formats a data hash into one hash that contains an array of headers and an array of array of data" do
19
+ Taps::Utils.format_data([ { :x => 1, :y => 1 }, { :x => 2, :y => 2 } ]).should == { :header => [ :x, :y ], :data => [ [1, 1], [2, 2] ] }
20
+ end
21
+
22
+ it "scales chunksize down slowly when the time delta of the block is just over a second" do
23
+ Time.stubs(:now).returns(10.0).returns(11.5)
24
+ Taps::Utils.calculate_chunksize(1000) { }.should == 900
25
+ end
26
+
27
+ it "scales chunksize down fast when the time delta of the block is over 3 seconds" do
28
+ Time.stubs(:now).returns(10.0).returns(15.0)
29
+ Taps::Utils.calculate_chunksize(3000) { }.should == 1000
30
+ end
31
+
32
+ it "scales up chunksize fast when the time delta of the block is under 0.8 seconds" do
33
+ Time.stubs(:now).returns(10.0).returns(10.7)
34
+ Taps::Utils.calculate_chunksize(1000) { }.should == 2000
35
+ end
36
+
37
+ it "scales up chunksize slow when the time delta of the block is between 0.8 and 1.1 seconds" do
38
+ Time.stubs(:now).returns(10.0).returns(10.8)
39
+ Taps::Utils.calculate_chunksize(1000) { }.should == 1100
40
+
41
+ Time.stubs(:now).returns(10.0).returns(11.1)
42
+ Taps::Utils.calculate_chunksize(1000) { }.should == 1100
43
+ end
44
+ end
45
+
metadata ADDED
@@ -0,0 +1,133 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: taps
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.2.1
5
+ platform: ruby
6
+ authors:
7
+ - Ricardo Chimal, Jr.
8
+ - Adam Wiggins
9
+ autorequire:
10
+ bindir: bin
11
+ cert_chain: []
12
+
13
+ date: 2009-02-10 00:00:00 -08:00
14
+ default_executable:
15
+ dependencies:
16
+ - !ruby/object:Gem::Dependency
17
+ name: sinatra
18
+ type: :runtime
19
+ version_requirement:
20
+ version_requirements: !ruby/object:Gem::Requirement
21
+ requirements:
22
+ - - ~>
23
+ - !ruby/object:Gem::Version
24
+ version: 0.9.0
25
+ version:
26
+ - !ruby/object:Gem::Dependency
27
+ name: activerecord
28
+ type: :runtime
29
+ version_requirement:
30
+ version_requirements: !ruby/object:Gem::Requirement
31
+ requirements:
32
+ - - "="
33
+ - !ruby/object:Gem::Version
34
+ version: 2.2.2
35
+ version:
36
+ - !ruby/object:Gem::Dependency
37
+ name: thor
38
+ type: :runtime
39
+ version_requirement:
40
+ version_requirements: !ruby/object:Gem::Requirement
41
+ requirements:
42
+ - - "="
43
+ - !ruby/object:Gem::Version
44
+ version: 0.9.9
45
+ version:
46
+ - !ruby/object:Gem::Dependency
47
+ name: rest-client
48
+ type: :runtime
49
+ version_requirement:
50
+ version_requirements: !ruby/object:Gem::Requirement
51
+ requirements:
52
+ - - ~>
53
+ - !ruby/object:Gem::Version
54
+ version: 0.9.0
55
+ version:
56
+ - !ruby/object:Gem::Dependency
57
+ name: sequel
58
+ type: :runtime
59
+ version_requirement:
60
+ version_requirements: !ruby/object:Gem::Requirement
61
+ requirements:
62
+ - - ~>
63
+ - !ruby/object:Gem::Version
64
+ version: 2.10.0
65
+ version:
66
+ - !ruby/object:Gem::Dependency
67
+ name: sqlite3-ruby
68
+ type: :runtime
69
+ version_requirement:
70
+ version_requirements: !ruby/object:Gem::Requirement
71
+ requirements:
72
+ - - ~>
73
+ - !ruby/object:Gem::Version
74
+ version: 1.2.0
75
+ version:
76
+ description: A simple database agnostic import/export app to transfer data to/from a remote database.
77
+ email: ricardo@heroku.com
78
+ executables:
79
+ - taps
80
+ - schema
81
+ extensions: []
82
+
83
+ extra_rdoc_files: []
84
+
85
+ files:
86
+ - spec/base.rb
87
+ - spec/client_session_spec.rb
88
+ - spec/schema_spec.rb
89
+ - spec/server_spec.rb
90
+ - spec/utils_spec.rb
91
+ - lib/taps/db_session.rb
92
+ - lib/taps/progress_bar.rb
93
+ - lib/taps/utils.rb
94
+ - lib/taps/schema.rb
95
+ - lib/taps/cli.rb
96
+ - lib/taps/client_session.rb
97
+ - lib/taps/server.rb
98
+ - lib/taps/config.rb
99
+ - README.rdoc
100
+ - LICENSE
101
+ - VERSION.yml
102
+ - Rakefile
103
+ - bin/taps
104
+ - bin/schema
105
+ has_rdoc: true
106
+ homepage: http://github.com/ricardochimal/taps
107
+ post_install_message:
108
+ rdoc_options:
109
+ - --inline-source
110
+ - --charset=UTF-8
111
+ require_paths:
112
+ - lib
113
+ required_ruby_version: !ruby/object:Gem::Requirement
114
+ requirements:
115
+ - - ">="
116
+ - !ruby/object:Gem::Version
117
+ version: "0"
118
+ version:
119
+ required_rubygems_version: !ruby/object:Gem::Requirement
120
+ requirements:
121
+ - - ">="
122
+ - !ruby/object:Gem::Version
123
+ version: "0"
124
+ version:
125
+ requirements: []
126
+
127
+ rubyforge_project: taps
128
+ rubygems_version: 1.3.1
129
+ signing_key:
130
+ specification_version: 2
131
+ summary: simple database import/export app
132
+ test_files: []
133
+