taps 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/LICENSE +20 -0
- data/README.rdoc +35 -0
- data/Rakefile +53 -0
- data/VERSION.yml +4 -0
- data/bin/schema +38 -0
- data/bin/taps +13 -0
- data/lib/taps/cli.rb +57 -0
- data/lib/taps/client_session.rb +237 -0
- data/lib/taps/config.rb +23 -0
- data/lib/taps/db_session.rb +25 -0
- data/lib/taps/progress_bar.rb +236 -0
- data/lib/taps/schema.rb +82 -0
- data/lib/taps/server.rb +138 -0
- data/lib/taps/utils.rb +76 -0
- data/spec/base.rb +21 -0
- data/spec/client_session_spec.rb +82 -0
- data/spec/schema_spec.rb +38 -0
- data/spec/server_spec.rb +33 -0
- data/spec/utils_spec.rb +45 -0
- metadata +133 -0
data/lib/taps/utils.rb
ADDED
@@ -0,0 +1,76 @@
|
|
1
|
+
require 'zlib'
|
2
|
+
require 'stringio'
|
3
|
+
require 'time'
|
4
|
+
require 'tempfile'
|
5
|
+
|
6
|
+
module Taps
|
7
|
+
module Utils
|
8
|
+
extend self
|
9
|
+
|
10
|
+
def checksum(data)
|
11
|
+
Zlib.crc32(data)
|
12
|
+
end
|
13
|
+
|
14
|
+
def valid_data?(data, crc32)
|
15
|
+
Zlib.crc32(data) == crc32.to_i
|
16
|
+
end
|
17
|
+
|
18
|
+
def gzip(data)
|
19
|
+
io = StringIO.new
|
20
|
+
gz = Zlib::GzipWriter.new(io)
|
21
|
+
gz.write data
|
22
|
+
gz.close
|
23
|
+
io.string
|
24
|
+
end
|
25
|
+
|
26
|
+
def gunzip(gzip_data)
|
27
|
+
io = StringIO.new(gzip_data)
|
28
|
+
gz = Zlib::GzipReader.new(io)
|
29
|
+
data = gz.read
|
30
|
+
gz.close
|
31
|
+
data
|
32
|
+
end
|
33
|
+
|
34
|
+
def format_data(data)
|
35
|
+
return {} if data.size == 0
|
36
|
+
header = data[0].keys
|
37
|
+
only_data = data.collect do |row|
|
38
|
+
header.collect { |h| row[h] }
|
39
|
+
end
|
40
|
+
{ :header => header, :data => only_data }
|
41
|
+
end
|
42
|
+
|
43
|
+
def calculate_chunksize(old_chunksize)
|
44
|
+
t1 = Time.now
|
45
|
+
yield
|
46
|
+
t2 = Time.now
|
47
|
+
|
48
|
+
diff = t2 - t1
|
49
|
+
new_chunksize = if diff > 3.0
|
50
|
+
(old_chunksize / 3).ceil
|
51
|
+
elsif diff > 1.1
|
52
|
+
old_chunksize - 100
|
53
|
+
elsif diff < 0.8
|
54
|
+
old_chunksize * 2
|
55
|
+
else
|
56
|
+
old_chunksize + 100
|
57
|
+
end
|
58
|
+
new_chunksize = 100 if new_chunksize < 100
|
59
|
+
new_chunksize
|
60
|
+
end
|
61
|
+
|
62
|
+
def load_schema(database_url, schema_data)
|
63
|
+
Tempfile.open('taps') do |tmp|
|
64
|
+
File.open(tmp.path, 'w') { |f| f.write(schema_data) }
|
65
|
+
`#{File.dirname(__FILE__)}/../../bin/schema load #{database_url} #{tmp.path}`
|
66
|
+
end
|
67
|
+
end
|
68
|
+
|
69
|
+
def load_indexes(database_url, index_data)
|
70
|
+
Tempfile.open('taps') do |tmp|
|
71
|
+
File.open(tmp.path, 'w') { |f| f.write(index_data) }
|
72
|
+
`#{File.dirname(__FILE__)}/../../bin/schema load_indexes #{database_url} #{tmp.path}`
|
73
|
+
end
|
74
|
+
end
|
75
|
+
end
|
76
|
+
end
|
data/spec/base.rb
ADDED
@@ -0,0 +1,21 @@
|
|
1
|
+
require 'rubygems'
|
2
|
+
require 'bacon'
|
3
|
+
require 'mocha'
|
4
|
+
|
5
|
+
class Bacon::Context
|
6
|
+
include Mocha::Standalone
|
7
|
+
|
8
|
+
alias_method :old_it, :it
|
9
|
+
def it(description)
|
10
|
+
old_it(description) do
|
11
|
+
mocha_setup
|
12
|
+
yield
|
13
|
+
mocha_verify
|
14
|
+
mocha_teardown
|
15
|
+
end
|
16
|
+
end
|
17
|
+
end
|
18
|
+
|
19
|
+
require File.dirname(__FILE__) + '/../lib/taps/config'
|
20
|
+
Taps::Config.taps_database_url = 'sqlite://test.db'
|
21
|
+
Sequel.connect(Taps::Config.taps_database_url)
|
@@ -0,0 +1,82 @@
|
|
1
|
+
require File.dirname(__FILE__) + '/base'
|
2
|
+
require File.dirname(__FILE__) + '/../lib/taps/client_session'
|
3
|
+
|
4
|
+
describe Taps::ClientSession do
|
5
|
+
before do
|
6
|
+
@client = Taps::ClientSession.new('sqlite://my.db', 'http://example.com:3000', 1000)
|
7
|
+
@client.stubs(:session_resource).returns(mock('session resource'))
|
8
|
+
end
|
9
|
+
|
10
|
+
it "starts a session and yields the session object to the block" do
|
11
|
+
Taps::ClientSession.start('x', 'y', 1000) do |session|
|
12
|
+
session.database_url.should == 'x'
|
13
|
+
session.remote_url.should == 'y'
|
14
|
+
session.default_chunksize.should == 1000
|
15
|
+
end
|
16
|
+
end
|
17
|
+
|
18
|
+
it "opens the local db connection via sequel and the database url" do
|
19
|
+
Sequel.expects(:connect).with('sqlite://my.db').returns(:con)
|
20
|
+
@client.db.should == :con
|
21
|
+
end
|
22
|
+
|
23
|
+
it "creates a restclient resource to the remote server" do
|
24
|
+
@client.server.url.should == 'http://example.com:3000'
|
25
|
+
end
|
26
|
+
|
27
|
+
it "verifies the db version, receive the schema, data, indexes, then reset the sequences" do
|
28
|
+
@client.expects(:verify_server)
|
29
|
+
@client.expects(:cmd_receive_schema)
|
30
|
+
@client.expects(:cmd_receive_data)
|
31
|
+
@client.expects(:cmd_receive_indexes)
|
32
|
+
@client.expects(:cmd_reset_sequences)
|
33
|
+
@client.cmd_receive.should.be.nil
|
34
|
+
end
|
35
|
+
|
36
|
+
it "checks the version of the server by seeing if it has access" do
|
37
|
+
@client.stubs(:server).returns(mock('server'))
|
38
|
+
@request = mock('request')
|
39
|
+
@client.server.expects(:[]).with('/').returns(@request)
|
40
|
+
@request.expects(:get).with({:taps_version => Taps::VERSION})
|
41
|
+
|
42
|
+
lambda { @client.verify_server }.should.not.raise
|
43
|
+
end
|
44
|
+
|
45
|
+
it "receives data from a remote taps server" do
|
46
|
+
@client.stubs(:puts)
|
47
|
+
@progressbar = mock('progressbar')
|
48
|
+
ProgressBar.stubs(:new).with('mytable', 2).returns(@progressbar)
|
49
|
+
@progressbar.stubs(:inc)
|
50
|
+
@progressbar.stubs(:finish)
|
51
|
+
@mytable = mock('mytable')
|
52
|
+
@client.expects(:fetch_tables_info).returns([ { :mytable => 2 }, 2 ])
|
53
|
+
@client.stubs(:db).returns(mock('db'))
|
54
|
+
@client.db.stubs(:[]).with(:mytable).returns(@mytable)
|
55
|
+
@client.expects(:fetch_table_rows).with(:mytable, 1000, 0).returns([ 1000, { :header => [:x, :y], :data => [[1, 2], [3, 4]] } ])
|
56
|
+
@client.expects(:fetch_table_rows).with(:mytable, 1000, 2).returns([ 1000, { }])
|
57
|
+
@mytable.expects(:multi_insert).with([:x, :y], [[1, 2], [3, 4]])
|
58
|
+
|
59
|
+
lambda { @client.cmd_receive_data }.should.not.raise
|
60
|
+
end
|
61
|
+
|
62
|
+
it "fetches tables info from taps server" do
|
63
|
+
@marshal_data = Marshal.dump({ :mytable => 2 })
|
64
|
+
@client.session_resource.stubs(:[]).with('tables').returns(mock('tables'))
|
65
|
+
@client.session_resource['tables'].stubs(:get).with(:taps_version => Taps::VERSION).returns(@marshal_data)
|
66
|
+
@client.fetch_tables_info.should == [ { :mytable => 2 }, 2 ]
|
67
|
+
end
|
68
|
+
|
69
|
+
it "fetches table rows given a chunksize and offset from taps server" do
|
70
|
+
@data = { :header => [ :x, :y ], :data => [ [1, 2], [3, 4] ] }
|
71
|
+
@gzip_data = Taps::Utils.gzip(Marshal.dump(@data))
|
72
|
+
Taps::Utils.stubs(:calculate_chunksize).with(1000).yields.returns(1000)
|
73
|
+
|
74
|
+
@response = mock('response')
|
75
|
+
@client.session_resource.stubs(:[]).with('tables/mytable/1000?offset=0').returns(mock('table resource'))
|
76
|
+
@client.session_resource['tables/mytable/1000?offset=0'].expects(:get).with(:taps_version => Taps::VERSION).returns(@response)
|
77
|
+
@response.stubs(:to_s).returns(@gzip_data)
|
78
|
+
@response.stubs(:headers).returns({ :taps_checksum => Taps::Utils.checksum(@gzip_data) })
|
79
|
+
@client.fetch_table_rows('mytable', 1000, 0).should == [ 1000, { :header => [:x, :y], :data => [[1, 2], [3, 4]] } ]
|
80
|
+
end
|
81
|
+
end
|
82
|
+
|
data/spec/schema_spec.rb
ADDED
@@ -0,0 +1,38 @@
|
|
1
|
+
require File.dirname(__FILE__) + '/base'
|
2
|
+
require File.dirname(__FILE__) + '/../lib/taps/schema'
|
3
|
+
|
4
|
+
describe Taps::Schema do
|
5
|
+
before do
|
6
|
+
@connection = mock("AR connection")
|
7
|
+
ActiveRecord::Base.stubs(:connection).returns(@connection)
|
8
|
+
end
|
9
|
+
|
10
|
+
it "parses a database url and returns a config hash for activerecord" do
|
11
|
+
Taps::Schema.create_config("postgres://myuser:mypass@localhost/mydb").should == {
|
12
|
+
'adapter' => 'postgresql',
|
13
|
+
'database' => 'mydb',
|
14
|
+
'username' => 'myuser',
|
15
|
+
'password' => 'mypass',
|
16
|
+
'host' => 'localhost'
|
17
|
+
}
|
18
|
+
end
|
19
|
+
|
20
|
+
it "translates sqlite in the database url to sqlite3" do
|
21
|
+
Taps::Schema.create_config("sqlite://myuser:mypass@localhost/mydb")['adapter'].should == 'sqlite3'
|
22
|
+
end
|
23
|
+
|
24
|
+
it "connects activerecord to the database" do
|
25
|
+
Taps::Schema.expects(:create_config).with("postgres://myuser:mypass@localhost/mydb").returns("db config")
|
26
|
+
ActiveRecord::Base.expects(:establish_connection).with("db config").returns(true)
|
27
|
+
Taps::Schema.connection("postgres://myuser:mypass@localhost/mydb").should == true
|
28
|
+
end
|
29
|
+
|
30
|
+
it "resets the db tables' primary keys" do
|
31
|
+
Taps::Schema.stubs(:connection)
|
32
|
+
ActiveRecord::Base.connection.expects(:respond_to?).with(:reset_pk_sequence!).returns(true)
|
33
|
+
ActiveRecord::Base.connection.stubs(:tables).returns(['table1'])
|
34
|
+
ActiveRecord::Base.connection.expects(:reset_pk_sequence!).with('table1')
|
35
|
+
should.not.raise { Taps::Schema.reset_db_sequences("postgres://myuser:mypass@localhost/mydb") }
|
36
|
+
end
|
37
|
+
end
|
38
|
+
|
data/spec/server_spec.rb
ADDED
@@ -0,0 +1,33 @@
|
|
1
|
+
require File.dirname(__FILE__) + '/base'
|
2
|
+
require 'sinatra'
|
3
|
+
require 'sinatra/test/bacon'
|
4
|
+
|
5
|
+
require File.dirname(__FILE__) + '/../lib/taps/server'
|
6
|
+
|
7
|
+
require 'pp'
|
8
|
+
|
9
|
+
describe Taps::Server do
|
10
|
+
before do
|
11
|
+
Taps::Config.login = 'taps'
|
12
|
+
Taps::Config.password = 'tpass'
|
13
|
+
|
14
|
+
@app = Taps::Server
|
15
|
+
@auth_header = "Basic " + ["taps:tpass"].pack("m*")
|
16
|
+
end
|
17
|
+
|
18
|
+
it "asks for http basic authentication" do
|
19
|
+
get '/'
|
20
|
+
status.should == 401
|
21
|
+
end
|
22
|
+
|
23
|
+
it "verifies the client taps version" do
|
24
|
+
get('/', { }, { 'HTTP_AUTHORIZATION' => @auth_header, 'HTTP_TAPS_VERSION' => Taps::VERSION })
|
25
|
+
status.should == 200
|
26
|
+
end
|
27
|
+
|
28
|
+
it "yells loudly if the client taps version doesn't match" do
|
29
|
+
get('/', { }, { 'HTTP_AUTHORIZATION' => @auth_header, 'HTTP_TAPS_VERSION' => '0.0.1' })
|
30
|
+
status.should == 417
|
31
|
+
end
|
32
|
+
end
|
33
|
+
|
data/spec/utils_spec.rb
ADDED
@@ -0,0 +1,45 @@
|
|
1
|
+
require File.dirname(__FILE__) + '/base'
|
2
|
+
require File.dirname(__FILE__) + '/../lib/taps/utils'
|
3
|
+
|
4
|
+
describe Taps::Utils do
|
5
|
+
it "gunzips a string" do
|
6
|
+
@hello_world = "\037\213\b\000R\261\207I\000\003\313H\315\311\311W(\317/\312I\001\000\205\021J\r\v\000\000\000"
|
7
|
+
Taps::Utils.gunzip(@hello_world).should == "hello world"
|
8
|
+
end
|
9
|
+
|
10
|
+
it "gzips and gunzips a string and returns the same string" do
|
11
|
+
Taps::Utils.gunzip(Taps::Utils.gzip("hello world")).should == "hello world"
|
12
|
+
end
|
13
|
+
|
14
|
+
it "generates a checksum using crc32" do
|
15
|
+
Taps::Utils.checksum("hello world").should == Zlib.crc32("hello world")
|
16
|
+
end
|
17
|
+
|
18
|
+
it "formats a data hash into one hash that contains an array of headers and an array of array of data" do
|
19
|
+
Taps::Utils.format_data([ { :x => 1, :y => 1 }, { :x => 2, :y => 2 } ]).should == { :header => [ :x, :y ], :data => [ [1, 1], [2, 2] ] }
|
20
|
+
end
|
21
|
+
|
22
|
+
it "scales chunksize down slowly when the time delta of the block is just over a second" do
|
23
|
+
Time.stubs(:now).returns(10.0).returns(11.5)
|
24
|
+
Taps::Utils.calculate_chunksize(1000) { }.should == 900
|
25
|
+
end
|
26
|
+
|
27
|
+
it "scales chunksize down fast when the time delta of the block is over 3 seconds" do
|
28
|
+
Time.stubs(:now).returns(10.0).returns(15.0)
|
29
|
+
Taps::Utils.calculate_chunksize(3000) { }.should == 1000
|
30
|
+
end
|
31
|
+
|
32
|
+
it "scales up chunksize fast when the time delta of the block is under 0.8 seconds" do
|
33
|
+
Time.stubs(:now).returns(10.0).returns(10.7)
|
34
|
+
Taps::Utils.calculate_chunksize(1000) { }.should == 2000
|
35
|
+
end
|
36
|
+
|
37
|
+
it "scales up chunksize slow when the time delta of the block is between 0.8 and 1.1 seconds" do
|
38
|
+
Time.stubs(:now).returns(10.0).returns(10.8)
|
39
|
+
Taps::Utils.calculate_chunksize(1000) { }.should == 1100
|
40
|
+
|
41
|
+
Time.stubs(:now).returns(10.0).returns(11.1)
|
42
|
+
Taps::Utils.calculate_chunksize(1000) { }.should == 1100
|
43
|
+
end
|
44
|
+
end
|
45
|
+
|
metadata
ADDED
@@ -0,0 +1,133 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: taps
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.2.1
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Ricardo Chimal, Jr.
|
8
|
+
- Adam Wiggins
|
9
|
+
autorequire:
|
10
|
+
bindir: bin
|
11
|
+
cert_chain: []
|
12
|
+
|
13
|
+
date: 2009-02-10 00:00:00 -08:00
|
14
|
+
default_executable:
|
15
|
+
dependencies:
|
16
|
+
- !ruby/object:Gem::Dependency
|
17
|
+
name: sinatra
|
18
|
+
type: :runtime
|
19
|
+
version_requirement:
|
20
|
+
version_requirements: !ruby/object:Gem::Requirement
|
21
|
+
requirements:
|
22
|
+
- - ~>
|
23
|
+
- !ruby/object:Gem::Version
|
24
|
+
version: 0.9.0
|
25
|
+
version:
|
26
|
+
- !ruby/object:Gem::Dependency
|
27
|
+
name: activerecord
|
28
|
+
type: :runtime
|
29
|
+
version_requirement:
|
30
|
+
version_requirements: !ruby/object:Gem::Requirement
|
31
|
+
requirements:
|
32
|
+
- - "="
|
33
|
+
- !ruby/object:Gem::Version
|
34
|
+
version: 2.2.2
|
35
|
+
version:
|
36
|
+
- !ruby/object:Gem::Dependency
|
37
|
+
name: thor
|
38
|
+
type: :runtime
|
39
|
+
version_requirement:
|
40
|
+
version_requirements: !ruby/object:Gem::Requirement
|
41
|
+
requirements:
|
42
|
+
- - "="
|
43
|
+
- !ruby/object:Gem::Version
|
44
|
+
version: 0.9.9
|
45
|
+
version:
|
46
|
+
- !ruby/object:Gem::Dependency
|
47
|
+
name: rest-client
|
48
|
+
type: :runtime
|
49
|
+
version_requirement:
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - ~>
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: 0.9.0
|
55
|
+
version:
|
56
|
+
- !ruby/object:Gem::Dependency
|
57
|
+
name: sequel
|
58
|
+
type: :runtime
|
59
|
+
version_requirement:
|
60
|
+
version_requirements: !ruby/object:Gem::Requirement
|
61
|
+
requirements:
|
62
|
+
- - ~>
|
63
|
+
- !ruby/object:Gem::Version
|
64
|
+
version: 2.10.0
|
65
|
+
version:
|
66
|
+
- !ruby/object:Gem::Dependency
|
67
|
+
name: sqlite3-ruby
|
68
|
+
type: :runtime
|
69
|
+
version_requirement:
|
70
|
+
version_requirements: !ruby/object:Gem::Requirement
|
71
|
+
requirements:
|
72
|
+
- - ~>
|
73
|
+
- !ruby/object:Gem::Version
|
74
|
+
version: 1.2.0
|
75
|
+
version:
|
76
|
+
description: A simple database agnostic import/export app to transfer data to/from a remote database.
|
77
|
+
email: ricardo@heroku.com
|
78
|
+
executables:
|
79
|
+
- taps
|
80
|
+
- schema
|
81
|
+
extensions: []
|
82
|
+
|
83
|
+
extra_rdoc_files: []
|
84
|
+
|
85
|
+
files:
|
86
|
+
- spec/base.rb
|
87
|
+
- spec/client_session_spec.rb
|
88
|
+
- spec/schema_spec.rb
|
89
|
+
- spec/server_spec.rb
|
90
|
+
- spec/utils_spec.rb
|
91
|
+
- lib/taps/db_session.rb
|
92
|
+
- lib/taps/progress_bar.rb
|
93
|
+
- lib/taps/utils.rb
|
94
|
+
- lib/taps/schema.rb
|
95
|
+
- lib/taps/cli.rb
|
96
|
+
- lib/taps/client_session.rb
|
97
|
+
- lib/taps/server.rb
|
98
|
+
- lib/taps/config.rb
|
99
|
+
- README.rdoc
|
100
|
+
- LICENSE
|
101
|
+
- VERSION.yml
|
102
|
+
- Rakefile
|
103
|
+
- bin/taps
|
104
|
+
- bin/schema
|
105
|
+
has_rdoc: true
|
106
|
+
homepage: http://github.com/ricardochimal/taps
|
107
|
+
post_install_message:
|
108
|
+
rdoc_options:
|
109
|
+
- --inline-source
|
110
|
+
- --charset=UTF-8
|
111
|
+
require_paths:
|
112
|
+
- lib
|
113
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
114
|
+
requirements:
|
115
|
+
- - ">="
|
116
|
+
- !ruby/object:Gem::Version
|
117
|
+
version: "0"
|
118
|
+
version:
|
119
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
120
|
+
requirements:
|
121
|
+
- - ">="
|
122
|
+
- !ruby/object:Gem::Version
|
123
|
+
version: "0"
|
124
|
+
version:
|
125
|
+
requirements: []
|
126
|
+
|
127
|
+
rubyforge_project: taps
|
128
|
+
rubygems_version: 1.3.1
|
129
|
+
signing_key:
|
130
|
+
specification_version: 2
|
131
|
+
summary: simple database import/export app
|
132
|
+
test_files: []
|
133
|
+
|