taps-taps 0.3.24
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/README.rdoc +51 -0
- data/VERSION.yml +5 -0
- data/bin/schema +54 -0
- data/bin/schema.cmd +6 -0
- data/bin/taps +6 -0
- data/lib/taps/chunksize.rb +52 -0
- data/lib/taps/cli.rb +196 -0
- data/lib/taps/config.rb +32 -0
- data/lib/taps/data_stream.rb +343 -0
- data/lib/taps/db_session.rb +20 -0
- data/lib/taps/errors.rb +15 -0
- data/lib/taps/log.rb +15 -0
- data/lib/taps/monkey.rb +21 -0
- data/lib/taps/multipart.rb +73 -0
- data/lib/taps/operation.rb +577 -0
- data/lib/taps/progress_bar.rb +236 -0
- data/lib/taps/schema.rb +82 -0
- data/lib/taps/server.rb +210 -0
- data/lib/taps/utils.rb +182 -0
- data/lib/taps/version.rb +18 -0
- data/lib/vendor/okjson.rb +555 -0
- data/spec/base.rb +26 -0
- data/spec/chunksize_spec.rb +41 -0
- data/spec/cli_spec.rb +16 -0
- data/spec/data_stream_spec.rb +23 -0
- data/spec/operation_spec.rb +42 -0
- data/spec/server_spec.rb +40 -0
- data/spec/utils_spec.rb +30 -0
- metadata +234 -0
data/spec/base.rb
ADDED
@@ -0,0 +1,26 @@
|
|
1
|
+
require 'rubygems'
|
2
|
+
require 'bacon'
|
3
|
+
require 'mocha'
|
4
|
+
require 'rack/test'
|
5
|
+
require 'tempfile'
|
6
|
+
|
7
|
+
$:.unshift File.dirname(__FILE__) + "/../lib"
|
8
|
+
|
9
|
+
class Bacon::Context
|
10
|
+
include Mocha::Standalone
|
11
|
+
include Rack::Test::Methods
|
12
|
+
|
13
|
+
alias_method :old_it, :it
|
14
|
+
def it(description)
|
15
|
+
old_it(description) do
|
16
|
+
mocha_setup
|
17
|
+
yield
|
18
|
+
mocha_verify
|
19
|
+
mocha_teardown
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
23
|
+
|
24
|
+
require 'taps/config'
|
25
|
+
Taps::Config.taps_database_url = "sqlite://#{Tempfile.new('test.db').path}"
|
26
|
+
Sequel.connect(Taps::Config.taps_database_url)
|
@@ -0,0 +1,41 @@
|
|
1
|
+
require File.dirname(__FILE__) + '/base'
|
2
|
+
require 'taps/utils'
|
3
|
+
|
4
|
+
describe Taps::Chunksize do
|
5
|
+
it "scales chunksize down slowly when the time delta of the block is just over a second" do
|
6
|
+
Time.stubs(:now).returns(10.0).returns(11.5)
|
7
|
+
Taps::Utils.calculate_chunksize(1000) { |c| }.should == 900
|
8
|
+
end
|
9
|
+
|
10
|
+
it "scales chunksize down fast when the time delta of the block is over 3 seconds" do
|
11
|
+
Time.stubs(:now).returns(10.0).returns(15.0)
|
12
|
+
Taps::Utils.calculate_chunksize(3000) { |c| }.should == 1000
|
13
|
+
end
|
14
|
+
|
15
|
+
it "scales up chunksize fast when the time delta of the block is under 0.8 seconds" do
|
16
|
+
Time.stubs(:now).returns(10.0).returns(10.7)
|
17
|
+
Taps::Utils.calculate_chunksize(1000) { |c| }.should == 2000
|
18
|
+
end
|
19
|
+
|
20
|
+
it "scales up chunksize slow when the time delta of the block is between 0.8 and 1.1 seconds" do
|
21
|
+
Time.stubs(:now).returns(10.0).returns(10.8)
|
22
|
+
Taps::Utils.calculate_chunksize(1000) { |c| }.should == 1100
|
23
|
+
|
24
|
+
Time.stubs(:now).returns(10.0).returns(11.1)
|
25
|
+
Taps::Utils.calculate_chunksize(1000) { |c| }.should == 1100
|
26
|
+
end
|
27
|
+
|
28
|
+
it "will reset the chunksize to a small value if we got a broken pipe exception" do
|
29
|
+
Taps::Utils.calculate_chunksize(1000) do |c|
|
30
|
+
raise Errno::EPIPE if c.chunksize == 1000
|
31
|
+
c.chunksize.should == 10
|
32
|
+
end.should == 10
|
33
|
+
end
|
34
|
+
|
35
|
+
it "will reset the chunksize to a small value if we got a broken pipe exception a second time" do
|
36
|
+
Taps::Utils.calculate_chunksize(1000) do |c|
|
37
|
+
raise Errno::EPIPE if c.chunksize == 1000 || c.chunksize == 10
|
38
|
+
c.chunksize.should == 1
|
39
|
+
end.should == 1
|
40
|
+
end
|
41
|
+
end
|
data/spec/cli_spec.rb
ADDED
@@ -0,0 +1,16 @@
|
|
1
|
+
require File.dirname(__FILE__) + '/base'
|
2
|
+
require 'taps/cli'
|
3
|
+
|
4
|
+
describe Taps::Cli do
|
5
|
+
it "translates a list of tables into a regex that can be used in table_filter" do
|
6
|
+
@cli = Taps::Cli.new(["-t", "mytable1,logs", "sqlite://tmp.db", "http://x:y@localhost:5000"])
|
7
|
+
opts = @cli.clientoptparse(:pull)
|
8
|
+
opts[:table_filter].should == "(^mytable1$|^logs$)"
|
9
|
+
end
|
10
|
+
|
11
|
+
it "translates a list of tables to exclude into a regex that can be used in table_filter" do
|
12
|
+
@cli = Taps::Cli.new(["-e", "mytable1,logs", "sqlite://tmp.db", "http://x:y@localhost:5000"])
|
13
|
+
opts = @cli.clientoptparse(:pull)
|
14
|
+
opts[:exclude_tables].should == ['mytable1','logs']
|
15
|
+
end
|
16
|
+
end
|
@@ -0,0 +1,23 @@
|
|
1
|
+
require File.dirname(__FILE__) + '/base'
|
2
|
+
require 'taps/data_stream'
|
3
|
+
|
4
|
+
describe Taps::DataStream do
|
5
|
+
before do
|
6
|
+
@db = mock('db')
|
7
|
+
end
|
8
|
+
|
9
|
+
it "increments the offset" do
|
10
|
+
stream = Taps::DataStream.new(@db, :table_name => 'test_table', :chunksize => 100)
|
11
|
+
stream.state[:offset].should == 0
|
12
|
+
stream.increment(100)
|
13
|
+
stream.state[:offset].should == 100
|
14
|
+
end
|
15
|
+
|
16
|
+
it "marks the stream complete if no rows are fetched" do
|
17
|
+
stream = Taps::DataStream.new(@db, :table_name => 'test_table', :chunksize => 100)
|
18
|
+
stream.stubs(:fetch_rows).returns({})
|
19
|
+
stream.complete?.should.be.false
|
20
|
+
stream.fetch
|
21
|
+
stream.complete?.should.be.true
|
22
|
+
end
|
23
|
+
end
|
@@ -0,0 +1,42 @@
|
|
1
|
+
require File.dirname(__FILE__) + '/base'
|
2
|
+
require 'taps/operation'
|
3
|
+
|
4
|
+
describe Taps::Operation do
|
5
|
+
before do
|
6
|
+
@op = Taps::Operation.new('dummy://localhost', 'http://x:y@localhost:5000')
|
7
|
+
end
|
8
|
+
|
9
|
+
it "returns an array of tables that match the regex table_filter" do
|
10
|
+
@op = Taps::Operation.new('dummy://localhost', 'http://x:y@localhost:5000', :table_filter => 'abc')
|
11
|
+
@op.apply_table_filter(['abc', 'def']).should == ['abc']
|
12
|
+
end
|
13
|
+
|
14
|
+
it "returns a hash of tables that match the regex table_filter" do
|
15
|
+
@op = Taps::Operation.new('dummy://localhost', 'http://x:y@localhost:5000', :table_filter => 'abc')
|
16
|
+
@op.apply_table_filter({ 'abc' => 1, 'def' => 2 }).should == { 'abc' => 1 }
|
17
|
+
end
|
18
|
+
|
19
|
+
it "returns an array of tables without the exclude_tables tables" do
|
20
|
+
@op = Taps::Operation.new('dummy://localhost', 'http://x:y@localhost:5000', :exclude_tables => ['abc', 'ghi', 'jkl'])
|
21
|
+
@op.apply_table_filter(['abc', 'def', 'ghi', 'jkl', 'mno']).should == ['def', 'mno']
|
22
|
+
end
|
23
|
+
|
24
|
+
it "returns a hash of tables without the exclude_tables tables" do
|
25
|
+
@op = Taps::Operation.new('dummy://localhost', 'http://x:y@localhost:5000', :exclude_tables => ['abc', 'ghi', 'jkl'])
|
26
|
+
@op.apply_table_filter({ 'abc' => 1, 'def' => 2, 'ghi' => 3, 'jkl' => 4, 'mno' => 5 }).should == { 'def' => 2, 'mno' => 5 }
|
27
|
+
end
|
28
|
+
|
29
|
+
it "masks a url's password" do
|
30
|
+
@op.safe_url("mysql://root:password@localhost/mydb").should == "mysql://root:[hidden]@localhost/mydb"
|
31
|
+
end
|
32
|
+
|
33
|
+
it "returns http headers with compression enabled" do
|
34
|
+
@op.http_headers.should == { :taps_version => Taps.version, :accept_encoding => "gzip, deflate" }
|
35
|
+
end
|
36
|
+
|
37
|
+
it "returns http headers with compression disabled" do
|
38
|
+
@op.stubs(:compression_disabled?).returns(true)
|
39
|
+
@op.http_headers.should == { :taps_version => Taps.version, :accept_encoding => "" }
|
40
|
+
end
|
41
|
+
|
42
|
+
end
|
data/spec/server_spec.rb
ADDED
@@ -0,0 +1,40 @@
|
|
1
|
+
require File.dirname(__FILE__) + '/base'
|
2
|
+
|
3
|
+
require 'taps/server'
|
4
|
+
|
5
|
+
require 'pp'
|
6
|
+
|
7
|
+
describe Taps::Server do
|
8
|
+
def app
|
9
|
+
Taps::Server.new
|
10
|
+
end
|
11
|
+
|
12
|
+
before do
|
13
|
+
Taps::Config.login = 'taps'
|
14
|
+
Taps::Config.password = 'tpass'
|
15
|
+
|
16
|
+
@app = Taps::Server
|
17
|
+
@auth_header = "Basic " + ["taps:tpass"].pack("m*")
|
18
|
+
end
|
19
|
+
|
20
|
+
it "asks for http basic authentication" do
|
21
|
+
get '/'
|
22
|
+
last_response.status.should == 401
|
23
|
+
end
|
24
|
+
|
25
|
+
it "verifies the client taps version" do
|
26
|
+
get('/', { }, { 'HTTP_AUTHORIZATION' => @auth_header, 'HTTP_TAPS_VERSION' => Taps.version })
|
27
|
+
last_response.status.should == 200
|
28
|
+
end
|
29
|
+
|
30
|
+
it "yells loudly if the client taps version doesn't match" do
|
31
|
+
get('/', { }, { 'HTTP_AUTHORIZATION' => @auth_header, 'HTTP_TAPS_VERSION' => '0.0.1' })
|
32
|
+
last_response.status.should == 417
|
33
|
+
end
|
34
|
+
|
35
|
+
it "allows healthcheck to be accessed w/o HTTP_TAPS_VERSION" do
|
36
|
+
get('/health', { }, { 'HTTP_AUTHORIZATION' => @auth_header })
|
37
|
+
last_response.status.should == 200
|
38
|
+
end
|
39
|
+
end
|
40
|
+
|
data/spec/utils_spec.rb
ADDED
@@ -0,0 +1,30 @@
|
|
1
|
+
require File.dirname(__FILE__) + '/base'
|
2
|
+
require 'taps/utils'
|
3
|
+
|
4
|
+
describe Taps::Utils do
|
5
|
+
it "generates a checksum using crc32" do
|
6
|
+
Taps::Utils.checksum("hello world").should == Zlib.crc32("hello world")
|
7
|
+
end
|
8
|
+
|
9
|
+
it "formats a data hash into one hash that contains an array of headers and an array of array of data" do
|
10
|
+
first_row = { :x => 1, :y => 1 }
|
11
|
+
first_row.stubs(:keys).returns([:x, :y])
|
12
|
+
Taps::Utils.format_data([ first_row, { :x => 2, :y => 2 } ]).should == { :header => [ :x, :y ], :data => [ [1, 1], [2, 2] ] }
|
13
|
+
end
|
14
|
+
|
15
|
+
it "enforces length limitations on columns" do
|
16
|
+
data = [ { :a => "aaabbbccc" } ]
|
17
|
+
schema = [ [ :a, { :db_type => "varchar(3)" }]]
|
18
|
+
lambda { Taps::Utils.format_data(data, :schema => schema) }.should.raise(Taps::InvalidData)
|
19
|
+
end
|
20
|
+
|
21
|
+
it "returns a list of columns that are text fields if the database is mysql" do
|
22
|
+
@db = mock("db", :url => "mysql://localhost/mydb")
|
23
|
+
@db.stubs(:schema).with(:mytable).returns([
|
24
|
+
[:id, { :db_type => "int" }],
|
25
|
+
[:mytext, { :db_type => "text" }]
|
26
|
+
])
|
27
|
+
Taps::Utils.incorrect_blobs(@db, :mytable).should == [:mytext]
|
28
|
+
end
|
29
|
+
end
|
30
|
+
|
metadata
ADDED
@@ -0,0 +1,234 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: taps-taps
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.3.24
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Ricardo Chimal, Jr.
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
date: 2013-10-22 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: rack
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - '>='
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: 1.0.1
|
20
|
+
type: :runtime
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - '>='
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: 1.0.1
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: rest-client
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - '>='
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: 1.4.0
|
34
|
+
- - <
|
35
|
+
- !ruby/object:Gem::Version
|
36
|
+
version: 1.7.0
|
37
|
+
type: :runtime
|
38
|
+
prerelease: false
|
39
|
+
version_requirements: !ruby/object:Gem::Requirement
|
40
|
+
requirements:
|
41
|
+
- - '>='
|
42
|
+
- !ruby/object:Gem::Version
|
43
|
+
version: 1.4.0
|
44
|
+
- - <
|
45
|
+
- !ruby/object:Gem::Version
|
46
|
+
version: 1.7.0
|
47
|
+
- !ruby/object:Gem::Dependency
|
48
|
+
name: sequel
|
49
|
+
requirement: !ruby/object:Gem::Requirement
|
50
|
+
requirements:
|
51
|
+
- - ~>
|
52
|
+
- !ruby/object:Gem::Version
|
53
|
+
version: 3.20.0
|
54
|
+
type: :runtime
|
55
|
+
prerelease: false
|
56
|
+
version_requirements: !ruby/object:Gem::Requirement
|
57
|
+
requirements:
|
58
|
+
- - ~>
|
59
|
+
- !ruby/object:Gem::Version
|
60
|
+
version: 3.20.0
|
61
|
+
- !ruby/object:Gem::Dependency
|
62
|
+
name: sinatra
|
63
|
+
requirement: !ruby/object:Gem::Requirement
|
64
|
+
requirements:
|
65
|
+
- - ~>
|
66
|
+
- !ruby/object:Gem::Version
|
67
|
+
version: 1.4.4
|
68
|
+
type: :runtime
|
69
|
+
prerelease: false
|
70
|
+
version_requirements: !ruby/object:Gem::Requirement
|
71
|
+
requirements:
|
72
|
+
- - ~>
|
73
|
+
- !ruby/object:Gem::Version
|
74
|
+
version: 1.4.4
|
75
|
+
- !ruby/object:Gem::Dependency
|
76
|
+
name: sqlite3
|
77
|
+
requirement: !ruby/object:Gem::Requirement
|
78
|
+
requirements:
|
79
|
+
- - ~>
|
80
|
+
- !ruby/object:Gem::Version
|
81
|
+
version: 1.3.8
|
82
|
+
type: :runtime
|
83
|
+
prerelease: false
|
84
|
+
version_requirements: !ruby/object:Gem::Requirement
|
85
|
+
requirements:
|
86
|
+
- - ~>
|
87
|
+
- !ruby/object:Gem::Version
|
88
|
+
version: 1.3.8
|
89
|
+
- !ruby/object:Gem::Dependency
|
90
|
+
name: sqlite3
|
91
|
+
requirement: !ruby/object:Gem::Requirement
|
92
|
+
requirements:
|
93
|
+
- - ~>
|
94
|
+
- !ruby/object:Gem::Version
|
95
|
+
version: '1.2'
|
96
|
+
type: :development
|
97
|
+
prerelease: false
|
98
|
+
version_requirements: !ruby/object:Gem::Requirement
|
99
|
+
requirements:
|
100
|
+
- - ~>
|
101
|
+
- !ruby/object:Gem::Version
|
102
|
+
version: '1.2'
|
103
|
+
- !ruby/object:Gem::Dependency
|
104
|
+
name: bacon
|
105
|
+
requirement: !ruby/object:Gem::Requirement
|
106
|
+
requirements:
|
107
|
+
- - '>='
|
108
|
+
- !ruby/object:Gem::Version
|
109
|
+
version: '0'
|
110
|
+
type: :development
|
111
|
+
prerelease: false
|
112
|
+
version_requirements: !ruby/object:Gem::Requirement
|
113
|
+
requirements:
|
114
|
+
- - '>='
|
115
|
+
- !ruby/object:Gem::Version
|
116
|
+
version: '0'
|
117
|
+
- !ruby/object:Gem::Dependency
|
118
|
+
name: mocha
|
119
|
+
requirement: !ruby/object:Gem::Requirement
|
120
|
+
requirements:
|
121
|
+
- - '>='
|
122
|
+
- !ruby/object:Gem::Version
|
123
|
+
version: '0'
|
124
|
+
type: :development
|
125
|
+
prerelease: false
|
126
|
+
version_requirements: !ruby/object:Gem::Requirement
|
127
|
+
requirements:
|
128
|
+
- - '>='
|
129
|
+
- !ruby/object:Gem::Version
|
130
|
+
version: '0'
|
131
|
+
- !ruby/object:Gem::Dependency
|
132
|
+
name: rack-test
|
133
|
+
requirement: !ruby/object:Gem::Requirement
|
134
|
+
requirements:
|
135
|
+
- - '>='
|
136
|
+
- !ruby/object:Gem::Version
|
137
|
+
version: '0'
|
138
|
+
type: :development
|
139
|
+
prerelease: false
|
140
|
+
version_requirements: !ruby/object:Gem::Requirement
|
141
|
+
requirements:
|
142
|
+
- - '>='
|
143
|
+
- !ruby/object:Gem::Version
|
144
|
+
version: '0'
|
145
|
+
- !ruby/object:Gem::Dependency
|
146
|
+
name: rake
|
147
|
+
requirement: !ruby/object:Gem::Requirement
|
148
|
+
requirements:
|
149
|
+
- - '>='
|
150
|
+
- !ruby/object:Gem::Version
|
151
|
+
version: '0'
|
152
|
+
type: :development
|
153
|
+
prerelease: false
|
154
|
+
version_requirements: !ruby/object:Gem::Requirement
|
155
|
+
requirements:
|
156
|
+
- - '>='
|
157
|
+
- !ruby/object:Gem::Version
|
158
|
+
version: '0'
|
159
|
+
- !ruby/object:Gem::Dependency
|
160
|
+
name: simplecov
|
161
|
+
requirement: !ruby/object:Gem::Requirement
|
162
|
+
requirements:
|
163
|
+
- - '>='
|
164
|
+
- !ruby/object:Gem::Version
|
165
|
+
version: '0'
|
166
|
+
type: :development
|
167
|
+
prerelease: false
|
168
|
+
version_requirements: !ruby/object:Gem::Requirement
|
169
|
+
requirements:
|
170
|
+
- - '>='
|
171
|
+
- !ruby/object:Gem::Version
|
172
|
+
version: '0'
|
173
|
+
description: |-
|
174
|
+
A simple database agnostic import/export app to transfer data to/from a remote database.
|
175
|
+
Taps-taps includes fixes submitted by the community.
|
176
|
+
email: ricardo@heroku.com
|
177
|
+
executables:
|
178
|
+
- taps
|
179
|
+
- schema
|
180
|
+
extensions: []
|
181
|
+
extra_rdoc_files: []
|
182
|
+
files:
|
183
|
+
- bin/schema
|
184
|
+
- bin/schema.cmd
|
185
|
+
- bin/taps
|
186
|
+
- lib/taps/chunksize.rb
|
187
|
+
- lib/taps/cli.rb
|
188
|
+
- lib/taps/config.rb
|
189
|
+
- lib/taps/data_stream.rb
|
190
|
+
- lib/taps/db_session.rb
|
191
|
+
- lib/taps/errors.rb
|
192
|
+
- lib/taps/log.rb
|
193
|
+
- lib/taps/monkey.rb
|
194
|
+
- lib/taps/multipart.rb
|
195
|
+
- lib/taps/operation.rb
|
196
|
+
- lib/taps/progress_bar.rb
|
197
|
+
- lib/taps/schema.rb
|
198
|
+
- lib/taps/server.rb
|
199
|
+
- lib/taps/utils.rb
|
200
|
+
- lib/taps/version.rb
|
201
|
+
- lib/vendor/okjson.rb
|
202
|
+
- README.rdoc
|
203
|
+
- spec/base.rb
|
204
|
+
- spec/chunksize_spec.rb
|
205
|
+
- spec/cli_spec.rb
|
206
|
+
- spec/data_stream_spec.rb
|
207
|
+
- spec/operation_spec.rb
|
208
|
+
- spec/server_spec.rb
|
209
|
+
- spec/utils_spec.rb
|
210
|
+
- VERSION.yml
|
211
|
+
homepage: http://github.com/wijet/taps
|
212
|
+
licenses: []
|
213
|
+
metadata: {}
|
214
|
+
post_install_message:
|
215
|
+
rdoc_options: []
|
216
|
+
require_paths:
|
217
|
+
- lib
|
218
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
219
|
+
requirements:
|
220
|
+
- - '>='
|
221
|
+
- !ruby/object:Gem::Version
|
222
|
+
version: '0'
|
223
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
224
|
+
requirements:
|
225
|
+
- - '>='
|
226
|
+
- !ruby/object:Gem::Version
|
227
|
+
version: '0'
|
228
|
+
requirements: []
|
229
|
+
rubyforge_project:
|
230
|
+
rubygems_version: 2.0.3
|
231
|
+
signing_key:
|
232
|
+
specification_version: 4
|
233
|
+
summary: simple database import/export app
|
234
|
+
test_files: []
|