taps2 0.5.5 → 0.6.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/{README.rdoc → README.md} +29 -22
- data/bin/{schema → schema2} +17 -8
- data/bin/{schema.cmd → schema2.cmd} +0 -0
- data/bin/{taps → taps2} +1 -1
- data/lib/taps/chunksize.rb +16 -12
- data/lib/taps/cli.rb +33 -38
- data/lib/taps/config.rb +2 -2
- data/lib/taps/data_stream.rb +256 -262
- data/lib/taps/errors.rb +1 -1
- data/lib/taps/log.rb +1 -1
- data/lib/taps/monkey.rb +10 -9
- data/lib/taps/multipart.rb +1 -2
- data/lib/taps/operation.rb +81 -86
- data/lib/taps/progress_bar.rb +42 -45
- data/lib/taps/schema.rb +2 -2
- data/lib/taps/server.rb +37 -38
- data/lib/taps/utils.rb +31 -34
- data/lib/taps/version.rb +13 -13
- data/lib/vendor/okjson.rb +115 -161
- data/spec/base.rb +2 -2
- data/spec/chunksize_spec.rb +6 -6
- data/spec/cli_spec.rb +6 -6
- data/spec/data_stream_spec.rb +4 -4
- data/spec/operation_spec.rb +17 -18
- data/spec/server_spec.rb +7 -7
- data/spec/utils_spec.rb +20 -20
- metadata +38 -39
- data/VERSION.yml +0 -5
data/spec/base.rb
CHANGED
@@ -5,13 +5,13 @@ require 'mocha/api'
|
|
5
5
|
require 'rack/test'
|
6
6
|
require 'tempfile'
|
7
7
|
|
8
|
-
|
8
|
+
$LOAD_PATH.unshift File.dirname(__FILE__) + '/../lib'
|
9
9
|
|
10
10
|
class Bacon::Context
|
11
11
|
include Mocha::API
|
12
12
|
include Rack::Test::Methods
|
13
13
|
|
14
|
-
|
14
|
+
alias old_it it
|
15
15
|
def it(description)
|
16
16
|
old_it(description) do
|
17
17
|
mocha_setup
|
data/spec/chunksize_spec.rb
CHANGED
@@ -2,22 +2,22 @@ require File.dirname(__FILE__) + '/base'
|
|
2
2
|
require 'taps/utils'
|
3
3
|
|
4
4
|
describe Taps::Chunksize do
|
5
|
-
it
|
5
|
+
it 'scales chunksize down slowly when the time delta of the block is just over a second' do
|
6
6
|
Time.stubs(:now).returns(10.0).returns(11.5)
|
7
7
|
Taps::Utils.calculate_chunksize(1000) { |c| }.should == 900
|
8
8
|
end
|
9
9
|
|
10
|
-
it
|
10
|
+
it 'scales chunksize down fast when the time delta of the block is over 3 seconds' do
|
11
11
|
Time.stubs(:now).returns(10.0).returns(15.0)
|
12
12
|
Taps::Utils.calculate_chunksize(3000) { |c| }.should == 1000
|
13
13
|
end
|
14
14
|
|
15
|
-
it
|
15
|
+
it 'scales up chunksize fast when the time delta of the block is under 0.8 seconds' do
|
16
16
|
Time.stubs(:now).returns(10.0).returns(10.7)
|
17
17
|
Taps::Utils.calculate_chunksize(1000) { |c| }.should == 2000
|
18
18
|
end
|
19
19
|
|
20
|
-
it
|
20
|
+
it 'scales up chunksize slow when the time delta of the block is between 0.8 and 1.1 seconds' do
|
21
21
|
Time.stubs(:now).returns(10.0).returns(10.8)
|
22
22
|
Taps::Utils.calculate_chunksize(1000) { |c| }.should == 1100
|
23
23
|
|
@@ -25,14 +25,14 @@ describe Taps::Chunksize do
|
|
25
25
|
Taps::Utils.calculate_chunksize(1000) { |c| }.should == 1100
|
26
26
|
end
|
27
27
|
|
28
|
-
it
|
28
|
+
it 'will reset the chunksize to a small value if we got a broken pipe exception' do
|
29
29
|
Taps::Utils.calculate_chunksize(1000) do |c|
|
30
30
|
raise Errno::EPIPE if c.chunksize == 1000
|
31
31
|
c.chunksize.should == 10
|
32
32
|
end.should == 10
|
33
33
|
end
|
34
34
|
|
35
|
-
it
|
35
|
+
it 'will reset the chunksize to a small value if we got a broken pipe exception a second time' do
|
36
36
|
Taps::Utils.calculate_chunksize(1000) do |c|
|
37
37
|
raise Errno::EPIPE if c.chunksize == 1000 || c.chunksize == 10
|
38
38
|
c.chunksize.should == 1
|
data/spec/cli_spec.rb
CHANGED
@@ -2,15 +2,15 @@ require File.dirname(__FILE__) + '/base'
|
|
2
2
|
require 'taps/cli'
|
3
3
|
|
4
4
|
describe Taps::Cli do
|
5
|
-
it
|
6
|
-
@cli = Taps::Cli.new([
|
5
|
+
it 'translates a list of tables into a regex that can be used in table_filter' do
|
6
|
+
@cli = Taps::Cli.new(['-t', 'mytable1,logs', 'sqlite://tmp.db', 'http://x:y@localhost:5000'])
|
7
7
|
opts = @cli.clientoptparse(:pull)
|
8
|
-
opts[:table_filter].should ==
|
8
|
+
opts[:table_filter].should == '(^mytable1$|^logs$)'
|
9
9
|
end
|
10
10
|
|
11
|
-
it
|
12
|
-
@cli = Taps::Cli.new([
|
11
|
+
it 'translates a list of tables to exclude into a regex that can be used in table_filter' do
|
12
|
+
@cli = Taps::Cli.new(['-e', 'mytable1,logs', 'sqlite://tmp.db', 'http://x:y@localhost:5000'])
|
13
13
|
opts = @cli.clientoptparse(:pull)
|
14
|
-
opts[:exclude_tables].should == [
|
14
|
+
opts[:exclude_tables].should == %w[mytable1 logs]
|
15
15
|
end
|
16
16
|
end
|
data/spec/data_stream_spec.rb
CHANGED
@@ -6,15 +6,15 @@ describe Taps::DataStream do
|
|
6
6
|
@db = mock('db')
|
7
7
|
end
|
8
8
|
|
9
|
-
it
|
10
|
-
stream = Taps::DataStream.new(@db, :
|
9
|
+
it 'increments the offset' do
|
10
|
+
stream = Taps::DataStream.new(@db, table_name: 'test_table', chunksize: 100)
|
11
11
|
stream.state[:offset].should == 0
|
12
12
|
stream.increment(100)
|
13
13
|
stream.state[:offset].should == 100
|
14
14
|
end
|
15
15
|
|
16
|
-
it
|
17
|
-
stream = Taps::DataStream.new(@db, :
|
16
|
+
it 'marks the stream complete if no rows are fetched' do
|
17
|
+
stream = Taps::DataStream.new(@db, table_name: 'test_table', chunksize: 100)
|
18
18
|
stream.stubs(:fetch_rows).returns({})
|
19
19
|
stream.complete?.should.be.false
|
20
20
|
stream.fetch
|
data/spec/operation_spec.rb
CHANGED
@@ -6,37 +6,36 @@ describe Taps::Operation do
|
|
6
6
|
@op = Taps::Operation.new('dummy://localhost', 'http://x:y@localhost:5000')
|
7
7
|
end
|
8
8
|
|
9
|
-
it
|
10
|
-
@op = Taps::Operation.new('dummy://localhost', 'http://x:y@localhost:5000', :
|
11
|
-
@op.apply_table_filter([
|
9
|
+
it 'returns an array of tables that match the regex table_filter' do
|
10
|
+
@op = Taps::Operation.new('dummy://localhost', 'http://x:y@localhost:5000', table_filter: 'abc')
|
11
|
+
@op.apply_table_filter(%w[abc def]).should == ['abc']
|
12
12
|
end
|
13
13
|
|
14
|
-
it
|
15
|
-
@op = Taps::Operation.new('dummy://localhost', 'http://x:y@localhost:5000', :
|
16
|
-
@op.apply_table_filter(
|
14
|
+
it 'returns a hash of tables that match the regex table_filter' do
|
15
|
+
@op = Taps::Operation.new('dummy://localhost', 'http://x:y@localhost:5000', table_filter: 'abc')
|
16
|
+
@op.apply_table_filter('abc' => 1, 'def' => 2).should == { 'abc' => 1 }
|
17
17
|
end
|
18
18
|
|
19
|
-
it
|
20
|
-
@op = Taps::Operation.new('dummy://localhost', 'http://x:y@localhost:5000', :
|
21
|
-
@op.apply_table_filter([
|
19
|
+
it 'returns an array of tables without the exclude_tables tables' do
|
20
|
+
@op = Taps::Operation.new('dummy://localhost', 'http://x:y@localhost:5000', exclude_tables: %w[abc ghi jkl])
|
21
|
+
@op.apply_table_filter(%w[abc def ghi jkl mno]).should == %w[def mno]
|
22
22
|
end
|
23
23
|
|
24
|
-
it
|
25
|
-
@op = Taps::Operation.new('dummy://localhost', 'http://x:y@localhost:5000', :
|
26
|
-
@op.apply_table_filter(
|
24
|
+
it 'returns a hash of tables without the exclude_tables tables' do
|
25
|
+
@op = Taps::Operation.new('dummy://localhost', 'http://x:y@localhost:5000', exclude_tables: %w[abc ghi jkl])
|
26
|
+
@op.apply_table_filter('abc' => 1, 'def' => 2, 'ghi' => 3, 'jkl' => 4, 'mno' => 5).should == { 'def' => 2, 'mno' => 5 }
|
27
27
|
end
|
28
28
|
|
29
29
|
it "masks a url's password" do
|
30
|
-
@op.safe_url(
|
30
|
+
@op.safe_url('mysql://root:password@localhost/mydb').should == 'mysql://root:[hidden]@localhost/mydb'
|
31
31
|
end
|
32
32
|
|
33
|
-
it
|
34
|
-
@op.http_headers.should == { :
|
33
|
+
it 'returns http headers with compression enabled' do
|
34
|
+
@op.http_headers.should == { taps_version: Taps.version, accept_encoding: 'gzip, deflate' }
|
35
35
|
end
|
36
36
|
|
37
|
-
it
|
37
|
+
it 'returns http headers with compression disabled' do
|
38
38
|
@op.stubs(:compression_disabled?).returns(true)
|
39
|
-
@op.http_headers.should == { :
|
39
|
+
@op.http_headers.should == { taps_version: Taps.version, accept_encoding: '' }
|
40
40
|
end
|
41
|
-
|
42
41
|
end
|
data/spec/server_spec.rb
CHANGED
@@ -13,26 +13,26 @@ describe Taps::Server do
|
|
13
13
|
Taps::Config.password = 'tpass'
|
14
14
|
|
15
15
|
@app = Taps::Server
|
16
|
-
@auth_header =
|
16
|
+
@auth_header = 'Basic ' + ['taps:tpass'].pack('m*')
|
17
17
|
end
|
18
18
|
|
19
|
-
it
|
19
|
+
it 'asks for http basic authentication' do
|
20
20
|
get '/'
|
21
21
|
last_response.status.should == 401
|
22
22
|
end
|
23
23
|
|
24
|
-
it
|
25
|
-
get('/', {
|
24
|
+
it 'verifies the client taps version' do
|
25
|
+
get('/', {}, 'HTTP_AUTHORIZATION' => @auth_header, 'HTTP_TAPS_VERSION' => Taps.version)
|
26
26
|
last_response.status.should == 200
|
27
27
|
end
|
28
28
|
|
29
29
|
it "yells loudly if the client taps version doesn't match" do
|
30
|
-
get('/', {
|
30
|
+
get('/', {}, 'HTTP_AUTHORIZATION' => @auth_header, 'HTTP_TAPS_VERSION' => '0.0.1')
|
31
31
|
last_response.status.should == 417
|
32
32
|
end
|
33
33
|
|
34
|
-
it
|
35
|
-
get('/health', {
|
34
|
+
it 'allows healthcheck to be accessed w/o HTTP_TAPS_VERSION' do
|
35
|
+
get('/health', {}, 'HTTP_AUTHORIZATION' => @auth_header)
|
36
36
|
last_response.status.should == 200
|
37
37
|
end
|
38
38
|
end
|
data/spec/utils_spec.rb
CHANGED
@@ -2,37 +2,37 @@ require File.dirname(__FILE__) + '/base'
|
|
2
2
|
require 'taps/utils'
|
3
3
|
|
4
4
|
describe Taps::Utils do
|
5
|
-
it
|
6
|
-
Taps::Utils.checksum(
|
5
|
+
it 'generates a checksum using crc32' do
|
6
|
+
Taps::Utils.checksum('hello world').should == Zlib.crc32('hello world')
|
7
7
|
end
|
8
8
|
|
9
|
-
it
|
10
|
-
first_row = { :
|
11
|
-
first_row.stubs(:keys).returns([
|
12
|
-
Taps::Utils.format_data([
|
9
|
+
it 'formats a data hash into one hash that contains an array of headers and an array of array of data' do
|
10
|
+
first_row = { x: 1, y: 1 }
|
11
|
+
first_row.stubs(:keys).returns(%i[x y])
|
12
|
+
Taps::Utils.format_data([first_row, { x: 2, y: 2 }]).should == { header: %i[x y], data: [[1, 1], [2, 2]] }
|
13
13
|
end
|
14
14
|
|
15
|
-
it
|
16
|
-
data = [
|
17
|
-
schema = [
|
18
|
-
|
15
|
+
it 'enforces length limitations on columns' do
|
16
|
+
data = [{ a: 'aaabbbccc' }]
|
17
|
+
schema = [[:a, { db_type: 'varchar(3)' }]]
|
18
|
+
-> { Taps::Utils.format_data(data, schema: schema) }.should.raise(Taps::InvalidData)
|
19
19
|
end
|
20
20
|
|
21
|
-
it
|
22
|
-
@db = mock(
|
21
|
+
it 'returns a list of columns that are text fields if the database is mysql' do
|
22
|
+
@db = mock('db', url: 'mysql://localhost/mydb')
|
23
23
|
@db.stubs(:schema).with(:mytable).returns([
|
24
|
-
|
25
|
-
|
26
|
-
|
24
|
+
[:id, { db_type: 'int' }],
|
25
|
+
[:mytext, { db_type: 'text' }]
|
26
|
+
])
|
27
27
|
Taps::Utils.incorrect_blobs(@db, :mytable).should == [:mytext]
|
28
28
|
end
|
29
29
|
|
30
|
-
it
|
31
|
-
@db = mock(
|
30
|
+
it 'rejects a multiple-column primary key as a single integer primary key' do
|
31
|
+
@db = mock('db')
|
32
32
|
@db.stubs(:schema).with(:pktable.identifier).returns([
|
33
|
-
|
34
|
-
|
35
|
-
|
33
|
+
[:id1, { primary_key: true, type: :integer }],
|
34
|
+
[:id2, { primary_key: true, type: :string }]
|
35
|
+
])
|
36
36
|
Taps::Utils.single_integer_primary_key(@db, :pktable).should.be.false
|
37
37
|
end
|
38
38
|
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: taps2
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.6.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Ricardo Chimal, Jr.
|
@@ -9,8 +9,22 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2017-
|
12
|
+
date: 2017-11-15 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
|
+
- !ruby/object:Gem::Dependency
|
15
|
+
name: extlib
|
16
|
+
requirement: !ruby/object:Gem::Requirement
|
17
|
+
requirements:
|
18
|
+
- - ">="
|
19
|
+
- !ruby/object:Gem::Version
|
20
|
+
version: '0'
|
21
|
+
type: :runtime
|
22
|
+
prerelease: false
|
23
|
+
version_requirements: !ruby/object:Gem::Requirement
|
24
|
+
requirements:
|
25
|
+
- - ">="
|
26
|
+
- !ruby/object:Gem::Version
|
27
|
+
version: '0'
|
14
28
|
- !ruby/object:Gem::Dependency
|
15
29
|
name: rack
|
16
30
|
requirement: !ruby/object:Gem::Requirement
|
@@ -43,16 +57,16 @@ dependencies:
|
|
43
57
|
name: sequel
|
44
58
|
requirement: !ruby/object:Gem::Requirement
|
45
59
|
requirements:
|
46
|
-
- - "
|
60
|
+
- - "~>"
|
47
61
|
- !ruby/object:Gem::Version
|
48
|
-
version: 4.0
|
62
|
+
version: '4.0'
|
49
63
|
type: :runtime
|
50
64
|
prerelease: false
|
51
65
|
version_requirements: !ruby/object:Gem::Requirement
|
52
66
|
requirements:
|
53
|
-
- - "
|
67
|
+
- - "~>"
|
54
68
|
- !ruby/object:Gem::Version
|
55
|
-
version: 4.0
|
69
|
+
version: '4.0'
|
56
70
|
- !ruby/object:Gem::Dependency
|
57
71
|
name: sinatra
|
58
72
|
requirement: !ruby/object:Gem::Requirement
|
@@ -68,13 +82,13 @@ dependencies:
|
|
68
82
|
- !ruby/object:Gem::Version
|
69
83
|
version: 1.4.4
|
70
84
|
- !ruby/object:Gem::Dependency
|
71
|
-
name:
|
85
|
+
name: bacon
|
72
86
|
requirement: !ruby/object:Gem::Requirement
|
73
87
|
requirements:
|
74
88
|
- - ">="
|
75
89
|
- !ruby/object:Gem::Version
|
76
90
|
version: '0'
|
77
|
-
type: :
|
91
|
+
type: :development
|
78
92
|
prerelease: false
|
79
93
|
version_requirements: !ruby/object:Gem::Requirement
|
80
94
|
requirements:
|
@@ -82,21 +96,21 @@ dependencies:
|
|
82
96
|
- !ruby/object:Gem::Version
|
83
97
|
version: '0'
|
84
98
|
- !ruby/object:Gem::Dependency
|
85
|
-
name:
|
99
|
+
name: mocha
|
86
100
|
requirement: !ruby/object:Gem::Requirement
|
87
101
|
requirements:
|
88
102
|
- - ">="
|
89
103
|
- !ruby/object:Gem::Version
|
90
|
-
version: 1.
|
104
|
+
version: 1.2.1
|
91
105
|
type: :development
|
92
106
|
prerelease: false
|
93
107
|
version_requirements: !ruby/object:Gem::Requirement
|
94
108
|
requirements:
|
95
109
|
- - ">="
|
96
110
|
- !ruby/object:Gem::Version
|
97
|
-
version: 1.
|
111
|
+
version: 1.2.1
|
98
112
|
- !ruby/object:Gem::Dependency
|
99
|
-
name:
|
113
|
+
name: rack-test
|
100
114
|
requirement: !ruby/object:Gem::Requirement
|
101
115
|
requirements:
|
102
116
|
- - ">="
|
@@ -110,21 +124,7 @@ dependencies:
|
|
110
124
|
- !ruby/object:Gem::Version
|
111
125
|
version: '0'
|
112
126
|
- !ruby/object:Gem::Dependency
|
113
|
-
name:
|
114
|
-
requirement: !ruby/object:Gem::Requirement
|
115
|
-
requirements:
|
116
|
-
- - ">="
|
117
|
-
- !ruby/object:Gem::Version
|
118
|
-
version: 1.2.1
|
119
|
-
type: :development
|
120
|
-
prerelease: false
|
121
|
-
version_requirements: !ruby/object:Gem::Requirement
|
122
|
-
requirements:
|
123
|
-
- - ">="
|
124
|
-
- !ruby/object:Gem::Version
|
125
|
-
version: 1.2.1
|
126
|
-
- !ruby/object:Gem::Dependency
|
127
|
-
name: rack-test
|
127
|
+
name: rake
|
128
128
|
requirement: !ruby/object:Gem::Requirement
|
129
129
|
requirements:
|
130
130
|
- - ">="
|
@@ -138,7 +138,7 @@ dependencies:
|
|
138
138
|
- !ruby/object:Gem::Version
|
139
139
|
version: '0'
|
140
140
|
- !ruby/object:Gem::Dependency
|
141
|
-
name:
|
141
|
+
name: simplecov
|
142
142
|
requirement: !ruby/object:Gem::Requirement
|
143
143
|
requirements:
|
144
144
|
- - ">="
|
@@ -152,35 +152,34 @@ dependencies:
|
|
152
152
|
- !ruby/object:Gem::Version
|
153
153
|
version: '0'
|
154
154
|
- !ruby/object:Gem::Dependency
|
155
|
-
name:
|
155
|
+
name: sqlite3
|
156
156
|
requirement: !ruby/object:Gem::Requirement
|
157
157
|
requirements:
|
158
158
|
- - ">="
|
159
159
|
- !ruby/object:Gem::Version
|
160
|
-
version:
|
160
|
+
version: 1.3.8
|
161
161
|
type: :development
|
162
162
|
prerelease: false
|
163
163
|
version_requirements: !ruby/object:Gem::Requirement
|
164
164
|
requirements:
|
165
165
|
- - ">="
|
166
166
|
- !ruby/object:Gem::Version
|
167
|
-
version:
|
167
|
+
version: 1.3.8
|
168
168
|
description: A simple database agnostic import/export app to transfer data to/from
|
169
169
|
a remote database.
|
170
170
|
email:
|
171
171
|
- ricardo@heroku.com
|
172
172
|
- joel@joelvanhorn.com
|
173
173
|
executables:
|
174
|
-
-
|
175
|
-
-
|
174
|
+
- taps2
|
175
|
+
- schema2
|
176
176
|
extensions: []
|
177
177
|
extra_rdoc_files: []
|
178
178
|
files:
|
179
|
-
- README.
|
180
|
-
-
|
181
|
-
- bin/
|
182
|
-
- bin/
|
183
|
-
- bin/taps
|
179
|
+
- README.md
|
180
|
+
- bin/schema2
|
181
|
+
- bin/schema2.cmd
|
182
|
+
- bin/taps2
|
184
183
|
- lib/taps/chunksize.rb
|
185
184
|
- lib/taps/cli.rb
|
186
185
|
- lib/taps/config.rb
|
@@ -223,7 +222,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
223
222
|
version: '0'
|
224
223
|
requirements: []
|
225
224
|
rubyforge_project:
|
226
|
-
rubygems_version: 2.6.
|
225
|
+
rubygems_version: 2.6.14
|
227
226
|
signing_key:
|
228
227
|
specification_version: 4
|
229
228
|
summary: Simple database import/export app
|