taps2 0.5.5 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/{README.rdoc → README.md} +29 -22
- data/bin/{schema → schema2} +17 -8
- data/bin/{schema.cmd → schema2.cmd} +0 -0
- data/bin/{taps → taps2} +1 -1
- data/lib/taps/chunksize.rb +16 -12
- data/lib/taps/cli.rb +33 -38
- data/lib/taps/config.rb +2 -2
- data/lib/taps/data_stream.rb +256 -262
- data/lib/taps/errors.rb +1 -1
- data/lib/taps/log.rb +1 -1
- data/lib/taps/monkey.rb +10 -9
- data/lib/taps/multipart.rb +1 -2
- data/lib/taps/operation.rb +81 -86
- data/lib/taps/progress_bar.rb +42 -45
- data/lib/taps/schema.rb +2 -2
- data/lib/taps/server.rb +37 -38
- data/lib/taps/utils.rb +31 -34
- data/lib/taps/version.rb +13 -13
- data/lib/vendor/okjson.rb +115 -161
- data/spec/base.rb +2 -2
- data/spec/chunksize_spec.rb +6 -6
- data/spec/cli_spec.rb +6 -6
- data/spec/data_stream_spec.rb +4 -4
- data/spec/operation_spec.rb +17 -18
- data/spec/server_spec.rb +7 -7
- data/spec/utils_spec.rb +20 -20
- metadata +38 -39
- data/VERSION.yml +0 -5
data/spec/base.rb
CHANGED
@@ -5,13 +5,13 @@ require 'mocha/api'
|
|
5
5
|
require 'rack/test'
|
6
6
|
require 'tempfile'
|
7
7
|
|
8
|
-
|
8
|
+
$LOAD_PATH.unshift File.dirname(__FILE__) + '/../lib'
|
9
9
|
|
10
10
|
class Bacon::Context
|
11
11
|
include Mocha::API
|
12
12
|
include Rack::Test::Methods
|
13
13
|
|
14
|
-
|
14
|
+
alias old_it it
|
15
15
|
def it(description)
|
16
16
|
old_it(description) do
|
17
17
|
mocha_setup
|
data/spec/chunksize_spec.rb
CHANGED
@@ -2,22 +2,22 @@ require File.dirname(__FILE__) + '/base'
|
|
2
2
|
require 'taps/utils'
|
3
3
|
|
4
4
|
describe Taps::Chunksize do
|
5
|
-
it
|
5
|
+
it 'scales chunksize down slowly when the time delta of the block is just over a second' do
|
6
6
|
Time.stubs(:now).returns(10.0).returns(11.5)
|
7
7
|
Taps::Utils.calculate_chunksize(1000) { |c| }.should == 900
|
8
8
|
end
|
9
9
|
|
10
|
-
it
|
10
|
+
it 'scales chunksize down fast when the time delta of the block is over 3 seconds' do
|
11
11
|
Time.stubs(:now).returns(10.0).returns(15.0)
|
12
12
|
Taps::Utils.calculate_chunksize(3000) { |c| }.should == 1000
|
13
13
|
end
|
14
14
|
|
15
|
-
it
|
15
|
+
it 'scales up chunksize fast when the time delta of the block is under 0.8 seconds' do
|
16
16
|
Time.stubs(:now).returns(10.0).returns(10.7)
|
17
17
|
Taps::Utils.calculate_chunksize(1000) { |c| }.should == 2000
|
18
18
|
end
|
19
19
|
|
20
|
-
it
|
20
|
+
it 'scales up chunksize slow when the time delta of the block is between 0.8 and 1.1 seconds' do
|
21
21
|
Time.stubs(:now).returns(10.0).returns(10.8)
|
22
22
|
Taps::Utils.calculate_chunksize(1000) { |c| }.should == 1100
|
23
23
|
|
@@ -25,14 +25,14 @@ describe Taps::Chunksize do
|
|
25
25
|
Taps::Utils.calculate_chunksize(1000) { |c| }.should == 1100
|
26
26
|
end
|
27
27
|
|
28
|
-
it
|
28
|
+
it 'will reset the chunksize to a small value if we got a broken pipe exception' do
|
29
29
|
Taps::Utils.calculate_chunksize(1000) do |c|
|
30
30
|
raise Errno::EPIPE if c.chunksize == 1000
|
31
31
|
c.chunksize.should == 10
|
32
32
|
end.should == 10
|
33
33
|
end
|
34
34
|
|
35
|
-
it
|
35
|
+
it 'will reset the chunksize to a small value if we got a broken pipe exception a second time' do
|
36
36
|
Taps::Utils.calculate_chunksize(1000) do |c|
|
37
37
|
raise Errno::EPIPE if c.chunksize == 1000 || c.chunksize == 10
|
38
38
|
c.chunksize.should == 1
|
data/spec/cli_spec.rb
CHANGED
@@ -2,15 +2,15 @@ require File.dirname(__FILE__) + '/base'
|
|
2
2
|
require 'taps/cli'
|
3
3
|
|
4
4
|
describe Taps::Cli do
|
5
|
-
it
|
6
|
-
@cli = Taps::Cli.new([
|
5
|
+
it 'translates a list of tables into a regex that can be used in table_filter' do
|
6
|
+
@cli = Taps::Cli.new(['-t', 'mytable1,logs', 'sqlite://tmp.db', 'http://x:y@localhost:5000'])
|
7
7
|
opts = @cli.clientoptparse(:pull)
|
8
|
-
opts[:table_filter].should ==
|
8
|
+
opts[:table_filter].should == '(^mytable1$|^logs$)'
|
9
9
|
end
|
10
10
|
|
11
|
-
it
|
12
|
-
@cli = Taps::Cli.new([
|
11
|
+
it 'translates a list of tables to exclude into a regex that can be used in table_filter' do
|
12
|
+
@cli = Taps::Cli.new(['-e', 'mytable1,logs', 'sqlite://tmp.db', 'http://x:y@localhost:5000'])
|
13
13
|
opts = @cli.clientoptparse(:pull)
|
14
|
-
opts[:exclude_tables].should == [
|
14
|
+
opts[:exclude_tables].should == %w[mytable1 logs]
|
15
15
|
end
|
16
16
|
end
|
data/spec/data_stream_spec.rb
CHANGED
@@ -6,15 +6,15 @@ describe Taps::DataStream do
|
|
6
6
|
@db = mock('db')
|
7
7
|
end
|
8
8
|
|
9
|
-
it
|
10
|
-
stream = Taps::DataStream.new(@db, :
|
9
|
+
it 'increments the offset' do
|
10
|
+
stream = Taps::DataStream.new(@db, table_name: 'test_table', chunksize: 100)
|
11
11
|
stream.state[:offset].should == 0
|
12
12
|
stream.increment(100)
|
13
13
|
stream.state[:offset].should == 100
|
14
14
|
end
|
15
15
|
|
16
|
-
it
|
17
|
-
stream = Taps::DataStream.new(@db, :
|
16
|
+
it 'marks the stream complete if no rows are fetched' do
|
17
|
+
stream = Taps::DataStream.new(@db, table_name: 'test_table', chunksize: 100)
|
18
18
|
stream.stubs(:fetch_rows).returns({})
|
19
19
|
stream.complete?.should.be.false
|
20
20
|
stream.fetch
|
data/spec/operation_spec.rb
CHANGED
@@ -6,37 +6,36 @@ describe Taps::Operation do
|
|
6
6
|
@op = Taps::Operation.new('dummy://localhost', 'http://x:y@localhost:5000')
|
7
7
|
end
|
8
8
|
|
9
|
-
it
|
10
|
-
@op = Taps::Operation.new('dummy://localhost', 'http://x:y@localhost:5000', :
|
11
|
-
@op.apply_table_filter([
|
9
|
+
it 'returns an array of tables that match the regex table_filter' do
|
10
|
+
@op = Taps::Operation.new('dummy://localhost', 'http://x:y@localhost:5000', table_filter: 'abc')
|
11
|
+
@op.apply_table_filter(%w[abc def]).should == ['abc']
|
12
12
|
end
|
13
13
|
|
14
|
-
it
|
15
|
-
@op = Taps::Operation.new('dummy://localhost', 'http://x:y@localhost:5000', :
|
16
|
-
@op.apply_table_filter(
|
14
|
+
it 'returns a hash of tables that match the regex table_filter' do
|
15
|
+
@op = Taps::Operation.new('dummy://localhost', 'http://x:y@localhost:5000', table_filter: 'abc')
|
16
|
+
@op.apply_table_filter('abc' => 1, 'def' => 2).should == { 'abc' => 1 }
|
17
17
|
end
|
18
18
|
|
19
|
-
it
|
20
|
-
@op = Taps::Operation.new('dummy://localhost', 'http://x:y@localhost:5000', :
|
21
|
-
@op.apply_table_filter([
|
19
|
+
it 'returns an array of tables without the exclude_tables tables' do
|
20
|
+
@op = Taps::Operation.new('dummy://localhost', 'http://x:y@localhost:5000', exclude_tables: %w[abc ghi jkl])
|
21
|
+
@op.apply_table_filter(%w[abc def ghi jkl mno]).should == %w[def mno]
|
22
22
|
end
|
23
23
|
|
24
|
-
it
|
25
|
-
@op = Taps::Operation.new('dummy://localhost', 'http://x:y@localhost:5000', :
|
26
|
-
@op.apply_table_filter(
|
24
|
+
it 'returns a hash of tables without the exclude_tables tables' do
|
25
|
+
@op = Taps::Operation.new('dummy://localhost', 'http://x:y@localhost:5000', exclude_tables: %w[abc ghi jkl])
|
26
|
+
@op.apply_table_filter('abc' => 1, 'def' => 2, 'ghi' => 3, 'jkl' => 4, 'mno' => 5).should == { 'def' => 2, 'mno' => 5 }
|
27
27
|
end
|
28
28
|
|
29
29
|
it "masks a url's password" do
|
30
|
-
@op.safe_url(
|
30
|
+
@op.safe_url('mysql://root:password@localhost/mydb').should == 'mysql://root:[hidden]@localhost/mydb'
|
31
31
|
end
|
32
32
|
|
33
|
-
it
|
34
|
-
@op.http_headers.should == { :
|
33
|
+
it 'returns http headers with compression enabled' do
|
34
|
+
@op.http_headers.should == { taps_version: Taps.version, accept_encoding: 'gzip, deflate' }
|
35
35
|
end
|
36
36
|
|
37
|
-
it
|
37
|
+
it 'returns http headers with compression disabled' do
|
38
38
|
@op.stubs(:compression_disabled?).returns(true)
|
39
|
-
@op.http_headers.should == { :
|
39
|
+
@op.http_headers.should == { taps_version: Taps.version, accept_encoding: '' }
|
40
40
|
end
|
41
|
-
|
42
41
|
end
|
data/spec/server_spec.rb
CHANGED
@@ -13,26 +13,26 @@ describe Taps::Server do
|
|
13
13
|
Taps::Config.password = 'tpass'
|
14
14
|
|
15
15
|
@app = Taps::Server
|
16
|
-
@auth_header =
|
16
|
+
@auth_header = 'Basic ' + ['taps:tpass'].pack('m*')
|
17
17
|
end
|
18
18
|
|
19
|
-
it
|
19
|
+
it 'asks for http basic authentication' do
|
20
20
|
get '/'
|
21
21
|
last_response.status.should == 401
|
22
22
|
end
|
23
23
|
|
24
|
-
it
|
25
|
-
get('/', {
|
24
|
+
it 'verifies the client taps version' do
|
25
|
+
get('/', {}, 'HTTP_AUTHORIZATION' => @auth_header, 'HTTP_TAPS_VERSION' => Taps.version)
|
26
26
|
last_response.status.should == 200
|
27
27
|
end
|
28
28
|
|
29
29
|
it "yells loudly if the client taps version doesn't match" do
|
30
|
-
get('/', {
|
30
|
+
get('/', {}, 'HTTP_AUTHORIZATION' => @auth_header, 'HTTP_TAPS_VERSION' => '0.0.1')
|
31
31
|
last_response.status.should == 417
|
32
32
|
end
|
33
33
|
|
34
|
-
it
|
35
|
-
get('/health', {
|
34
|
+
it 'allows healthcheck to be accessed w/o HTTP_TAPS_VERSION' do
|
35
|
+
get('/health', {}, 'HTTP_AUTHORIZATION' => @auth_header)
|
36
36
|
last_response.status.should == 200
|
37
37
|
end
|
38
38
|
end
|
data/spec/utils_spec.rb
CHANGED
@@ -2,37 +2,37 @@ require File.dirname(__FILE__) + '/base'
|
|
2
2
|
require 'taps/utils'
|
3
3
|
|
4
4
|
describe Taps::Utils do
|
5
|
-
it
|
6
|
-
Taps::Utils.checksum(
|
5
|
+
it 'generates a checksum using crc32' do
|
6
|
+
Taps::Utils.checksum('hello world').should == Zlib.crc32('hello world')
|
7
7
|
end
|
8
8
|
|
9
|
-
it
|
10
|
-
first_row = { :
|
11
|
-
first_row.stubs(:keys).returns([
|
12
|
-
Taps::Utils.format_data([
|
9
|
+
it 'formats a data hash into one hash that contains an array of headers and an array of array of data' do
|
10
|
+
first_row = { x: 1, y: 1 }
|
11
|
+
first_row.stubs(:keys).returns(%i[x y])
|
12
|
+
Taps::Utils.format_data([first_row, { x: 2, y: 2 }]).should == { header: %i[x y], data: [[1, 1], [2, 2]] }
|
13
13
|
end
|
14
14
|
|
15
|
-
it
|
16
|
-
data = [
|
17
|
-
schema = [
|
18
|
-
|
15
|
+
it 'enforces length limitations on columns' do
|
16
|
+
data = [{ a: 'aaabbbccc' }]
|
17
|
+
schema = [[:a, { db_type: 'varchar(3)' }]]
|
18
|
+
-> { Taps::Utils.format_data(data, schema: schema) }.should.raise(Taps::InvalidData)
|
19
19
|
end
|
20
20
|
|
21
|
-
it
|
22
|
-
@db = mock(
|
21
|
+
it 'returns a list of columns that are text fields if the database is mysql' do
|
22
|
+
@db = mock('db', url: 'mysql://localhost/mydb')
|
23
23
|
@db.stubs(:schema).with(:mytable).returns([
|
24
|
-
|
25
|
-
|
26
|
-
|
24
|
+
[:id, { db_type: 'int' }],
|
25
|
+
[:mytext, { db_type: 'text' }]
|
26
|
+
])
|
27
27
|
Taps::Utils.incorrect_blobs(@db, :mytable).should == [:mytext]
|
28
28
|
end
|
29
29
|
|
30
|
-
it
|
31
|
-
@db = mock(
|
30
|
+
it 'rejects a multiple-column primary key as a single integer primary key' do
|
31
|
+
@db = mock('db')
|
32
32
|
@db.stubs(:schema).with(:pktable.identifier).returns([
|
33
|
-
|
34
|
-
|
35
|
-
|
33
|
+
[:id1, { primary_key: true, type: :integer }],
|
34
|
+
[:id2, { primary_key: true, type: :string }]
|
35
|
+
])
|
36
36
|
Taps::Utils.single_integer_primary_key(@db, :pktable).should.be.false
|
37
37
|
end
|
38
38
|
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: taps2
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.6.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Ricardo Chimal, Jr.
|
@@ -9,8 +9,22 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2017-
|
12
|
+
date: 2017-11-15 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
|
+
- !ruby/object:Gem::Dependency
|
15
|
+
name: extlib
|
16
|
+
requirement: !ruby/object:Gem::Requirement
|
17
|
+
requirements:
|
18
|
+
- - ">="
|
19
|
+
- !ruby/object:Gem::Version
|
20
|
+
version: '0'
|
21
|
+
type: :runtime
|
22
|
+
prerelease: false
|
23
|
+
version_requirements: !ruby/object:Gem::Requirement
|
24
|
+
requirements:
|
25
|
+
- - ">="
|
26
|
+
- !ruby/object:Gem::Version
|
27
|
+
version: '0'
|
14
28
|
- !ruby/object:Gem::Dependency
|
15
29
|
name: rack
|
16
30
|
requirement: !ruby/object:Gem::Requirement
|
@@ -43,16 +57,16 @@ dependencies:
|
|
43
57
|
name: sequel
|
44
58
|
requirement: !ruby/object:Gem::Requirement
|
45
59
|
requirements:
|
46
|
-
- - "
|
60
|
+
- - "~>"
|
47
61
|
- !ruby/object:Gem::Version
|
48
|
-
version: 4.0
|
62
|
+
version: '4.0'
|
49
63
|
type: :runtime
|
50
64
|
prerelease: false
|
51
65
|
version_requirements: !ruby/object:Gem::Requirement
|
52
66
|
requirements:
|
53
|
-
- - "
|
67
|
+
- - "~>"
|
54
68
|
- !ruby/object:Gem::Version
|
55
|
-
version: 4.0
|
69
|
+
version: '4.0'
|
56
70
|
- !ruby/object:Gem::Dependency
|
57
71
|
name: sinatra
|
58
72
|
requirement: !ruby/object:Gem::Requirement
|
@@ -68,13 +82,13 @@ dependencies:
|
|
68
82
|
- !ruby/object:Gem::Version
|
69
83
|
version: 1.4.4
|
70
84
|
- !ruby/object:Gem::Dependency
|
71
|
-
name:
|
85
|
+
name: bacon
|
72
86
|
requirement: !ruby/object:Gem::Requirement
|
73
87
|
requirements:
|
74
88
|
- - ">="
|
75
89
|
- !ruby/object:Gem::Version
|
76
90
|
version: '0'
|
77
|
-
type: :
|
91
|
+
type: :development
|
78
92
|
prerelease: false
|
79
93
|
version_requirements: !ruby/object:Gem::Requirement
|
80
94
|
requirements:
|
@@ -82,21 +96,21 @@ dependencies:
|
|
82
96
|
- !ruby/object:Gem::Version
|
83
97
|
version: '0'
|
84
98
|
- !ruby/object:Gem::Dependency
|
85
|
-
name:
|
99
|
+
name: mocha
|
86
100
|
requirement: !ruby/object:Gem::Requirement
|
87
101
|
requirements:
|
88
102
|
- - ">="
|
89
103
|
- !ruby/object:Gem::Version
|
90
|
-
version: 1.
|
104
|
+
version: 1.2.1
|
91
105
|
type: :development
|
92
106
|
prerelease: false
|
93
107
|
version_requirements: !ruby/object:Gem::Requirement
|
94
108
|
requirements:
|
95
109
|
- - ">="
|
96
110
|
- !ruby/object:Gem::Version
|
97
|
-
version: 1.
|
111
|
+
version: 1.2.1
|
98
112
|
- !ruby/object:Gem::Dependency
|
99
|
-
name:
|
113
|
+
name: rack-test
|
100
114
|
requirement: !ruby/object:Gem::Requirement
|
101
115
|
requirements:
|
102
116
|
- - ">="
|
@@ -110,21 +124,7 @@ dependencies:
|
|
110
124
|
- !ruby/object:Gem::Version
|
111
125
|
version: '0'
|
112
126
|
- !ruby/object:Gem::Dependency
|
113
|
-
name:
|
114
|
-
requirement: !ruby/object:Gem::Requirement
|
115
|
-
requirements:
|
116
|
-
- - ">="
|
117
|
-
- !ruby/object:Gem::Version
|
118
|
-
version: 1.2.1
|
119
|
-
type: :development
|
120
|
-
prerelease: false
|
121
|
-
version_requirements: !ruby/object:Gem::Requirement
|
122
|
-
requirements:
|
123
|
-
- - ">="
|
124
|
-
- !ruby/object:Gem::Version
|
125
|
-
version: 1.2.1
|
126
|
-
- !ruby/object:Gem::Dependency
|
127
|
-
name: rack-test
|
127
|
+
name: rake
|
128
128
|
requirement: !ruby/object:Gem::Requirement
|
129
129
|
requirements:
|
130
130
|
- - ">="
|
@@ -138,7 +138,7 @@ dependencies:
|
|
138
138
|
- !ruby/object:Gem::Version
|
139
139
|
version: '0'
|
140
140
|
- !ruby/object:Gem::Dependency
|
141
|
-
name:
|
141
|
+
name: simplecov
|
142
142
|
requirement: !ruby/object:Gem::Requirement
|
143
143
|
requirements:
|
144
144
|
- - ">="
|
@@ -152,35 +152,34 @@ dependencies:
|
|
152
152
|
- !ruby/object:Gem::Version
|
153
153
|
version: '0'
|
154
154
|
- !ruby/object:Gem::Dependency
|
155
|
-
name:
|
155
|
+
name: sqlite3
|
156
156
|
requirement: !ruby/object:Gem::Requirement
|
157
157
|
requirements:
|
158
158
|
- - ">="
|
159
159
|
- !ruby/object:Gem::Version
|
160
|
-
version:
|
160
|
+
version: 1.3.8
|
161
161
|
type: :development
|
162
162
|
prerelease: false
|
163
163
|
version_requirements: !ruby/object:Gem::Requirement
|
164
164
|
requirements:
|
165
165
|
- - ">="
|
166
166
|
- !ruby/object:Gem::Version
|
167
|
-
version:
|
167
|
+
version: 1.3.8
|
168
168
|
description: A simple database agnostic import/export app to transfer data to/from
|
169
169
|
a remote database.
|
170
170
|
email:
|
171
171
|
- ricardo@heroku.com
|
172
172
|
- joel@joelvanhorn.com
|
173
173
|
executables:
|
174
|
-
-
|
175
|
-
-
|
174
|
+
- taps2
|
175
|
+
- schema2
|
176
176
|
extensions: []
|
177
177
|
extra_rdoc_files: []
|
178
178
|
files:
|
179
|
-
- README.
|
180
|
-
-
|
181
|
-
- bin/
|
182
|
-
- bin/
|
183
|
-
- bin/taps
|
179
|
+
- README.md
|
180
|
+
- bin/schema2
|
181
|
+
- bin/schema2.cmd
|
182
|
+
- bin/taps2
|
184
183
|
- lib/taps/chunksize.rb
|
185
184
|
- lib/taps/cli.rb
|
186
185
|
- lib/taps/config.rb
|
@@ -223,7 +222,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
223
222
|
version: '0'
|
224
223
|
requirements: []
|
225
224
|
rubyforge_project:
|
226
|
-
rubygems_version: 2.6.
|
225
|
+
rubygems_version: 2.6.14
|
227
226
|
signing_key:
|
228
227
|
specification_version: 4
|
229
228
|
summary: Simple database import/export app
|