taps-taps 0.3.24
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/README.rdoc +51 -0
- data/VERSION.yml +5 -0
- data/bin/schema +54 -0
- data/bin/schema.cmd +6 -0
- data/bin/taps +6 -0
- data/lib/taps/chunksize.rb +52 -0
- data/lib/taps/cli.rb +196 -0
- data/lib/taps/config.rb +32 -0
- data/lib/taps/data_stream.rb +343 -0
- data/lib/taps/db_session.rb +20 -0
- data/lib/taps/errors.rb +15 -0
- data/lib/taps/log.rb +15 -0
- data/lib/taps/monkey.rb +21 -0
- data/lib/taps/multipart.rb +73 -0
- data/lib/taps/operation.rb +577 -0
- data/lib/taps/progress_bar.rb +236 -0
- data/lib/taps/schema.rb +82 -0
- data/lib/taps/server.rb +210 -0
- data/lib/taps/utils.rb +182 -0
- data/lib/taps/version.rb +18 -0
- data/lib/vendor/okjson.rb +555 -0
- data/spec/base.rb +26 -0
- data/spec/chunksize_spec.rb +41 -0
- data/spec/cli_spec.rb +16 -0
- data/spec/data_stream_spec.rb +23 -0
- data/spec/operation_spec.rb +42 -0
- data/spec/server_spec.rb +40 -0
- data/spec/utils_spec.rb +30 -0
- metadata +234 -0
data/spec/base.rb
ADDED
@@ -0,0 +1,26 @@
|
|
1
|
+
require 'rubygems'
|
2
|
+
require 'bacon'
|
3
|
+
require 'mocha'
|
4
|
+
require 'rack/test'
|
5
|
+
require 'tempfile'
|
6
|
+
|
7
|
+
$:.unshift File.dirname(__FILE__) + "/../lib"
|
8
|
+
|
9
|
+
class Bacon::Context
|
10
|
+
include Mocha::Standalone
|
11
|
+
include Rack::Test::Methods
|
12
|
+
|
13
|
+
alias_method :old_it, :it
|
14
|
+
def it(description)
|
15
|
+
old_it(description) do
|
16
|
+
mocha_setup
|
17
|
+
yield
|
18
|
+
mocha_verify
|
19
|
+
mocha_teardown
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
23
|
+
|
24
|
+
require 'taps/config'
|
25
|
+
Taps::Config.taps_database_url = "sqlite://#{Tempfile.new('test.db').path}"
|
26
|
+
Sequel.connect(Taps::Config.taps_database_url)
|
@@ -0,0 +1,41 @@
|
|
1
|
+
require File.dirname(__FILE__) + '/base'
|
2
|
+
require 'taps/utils'
|
3
|
+
|
4
|
+
describe Taps::Chunksize do
|
5
|
+
it "scales chunksize down slowly when the time delta of the block is just over a second" do
|
6
|
+
Time.stubs(:now).returns(10.0).returns(11.5)
|
7
|
+
Taps::Utils.calculate_chunksize(1000) { |c| }.should == 900
|
8
|
+
end
|
9
|
+
|
10
|
+
it "scales chunksize down fast when the time delta of the block is over 3 seconds" do
|
11
|
+
Time.stubs(:now).returns(10.0).returns(15.0)
|
12
|
+
Taps::Utils.calculate_chunksize(3000) { |c| }.should == 1000
|
13
|
+
end
|
14
|
+
|
15
|
+
it "scales up chunksize fast when the time delta of the block is under 0.8 seconds" do
|
16
|
+
Time.stubs(:now).returns(10.0).returns(10.7)
|
17
|
+
Taps::Utils.calculate_chunksize(1000) { |c| }.should == 2000
|
18
|
+
end
|
19
|
+
|
20
|
+
it "scales up chunksize slow when the time delta of the block is between 0.8 and 1.1 seconds" do
|
21
|
+
Time.stubs(:now).returns(10.0).returns(10.8)
|
22
|
+
Taps::Utils.calculate_chunksize(1000) { |c| }.should == 1100
|
23
|
+
|
24
|
+
Time.stubs(:now).returns(10.0).returns(11.1)
|
25
|
+
Taps::Utils.calculate_chunksize(1000) { |c| }.should == 1100
|
26
|
+
end
|
27
|
+
|
28
|
+
it "will reset the chunksize to a small value if we got a broken pipe exception" do
|
29
|
+
Taps::Utils.calculate_chunksize(1000) do |c|
|
30
|
+
raise Errno::EPIPE if c.chunksize == 1000
|
31
|
+
c.chunksize.should == 10
|
32
|
+
end.should == 10
|
33
|
+
end
|
34
|
+
|
35
|
+
it "will reset the chunksize to a small value if we got a broken pipe exception a second time" do
|
36
|
+
Taps::Utils.calculate_chunksize(1000) do |c|
|
37
|
+
raise Errno::EPIPE if c.chunksize == 1000 || c.chunksize == 10
|
38
|
+
c.chunksize.should == 1
|
39
|
+
end.should == 1
|
40
|
+
end
|
41
|
+
end
|
data/spec/cli_spec.rb
ADDED
@@ -0,0 +1,16 @@
|
|
1
|
+
require File.dirname(__FILE__) + '/base'
|
2
|
+
require 'taps/cli'
|
3
|
+
|
4
|
+
describe Taps::Cli do
|
5
|
+
it "translates a list of tables into a regex that can be used in table_filter" do
|
6
|
+
@cli = Taps::Cli.new(["-t", "mytable1,logs", "sqlite://tmp.db", "http://x:y@localhost:5000"])
|
7
|
+
opts = @cli.clientoptparse(:pull)
|
8
|
+
opts[:table_filter].should == "(^mytable1$|^logs$)"
|
9
|
+
end
|
10
|
+
|
11
|
+
it "translates a list of tables to exclude into a regex that can be used in table_filter" do
|
12
|
+
@cli = Taps::Cli.new(["-e", "mytable1,logs", "sqlite://tmp.db", "http://x:y@localhost:5000"])
|
13
|
+
opts = @cli.clientoptparse(:pull)
|
14
|
+
opts[:exclude_tables].should == ['mytable1','logs']
|
15
|
+
end
|
16
|
+
end
|
@@ -0,0 +1,23 @@
|
|
1
|
+
require File.dirname(__FILE__) + '/base'
|
2
|
+
require 'taps/data_stream'
|
3
|
+
|
4
|
+
describe Taps::DataStream do
|
5
|
+
before do
|
6
|
+
@db = mock('db')
|
7
|
+
end
|
8
|
+
|
9
|
+
it "increments the offset" do
|
10
|
+
stream = Taps::DataStream.new(@db, :table_name => 'test_table', :chunksize => 100)
|
11
|
+
stream.state[:offset].should == 0
|
12
|
+
stream.increment(100)
|
13
|
+
stream.state[:offset].should == 100
|
14
|
+
end
|
15
|
+
|
16
|
+
it "marks the stream complete if no rows are fetched" do
|
17
|
+
stream = Taps::DataStream.new(@db, :table_name => 'test_table', :chunksize => 100)
|
18
|
+
stream.stubs(:fetch_rows).returns({})
|
19
|
+
stream.complete?.should.be.false
|
20
|
+
stream.fetch
|
21
|
+
stream.complete?.should.be.true
|
22
|
+
end
|
23
|
+
end
|
@@ -0,0 +1,42 @@
|
|
1
|
+
require File.dirname(__FILE__) + '/base'
|
2
|
+
require 'taps/operation'
|
3
|
+
|
4
|
+
describe Taps::Operation do
|
5
|
+
before do
|
6
|
+
@op = Taps::Operation.new('dummy://localhost', 'http://x:y@localhost:5000')
|
7
|
+
end
|
8
|
+
|
9
|
+
it "returns an array of tables that match the regex table_filter" do
|
10
|
+
@op = Taps::Operation.new('dummy://localhost', 'http://x:y@localhost:5000', :table_filter => 'abc')
|
11
|
+
@op.apply_table_filter(['abc', 'def']).should == ['abc']
|
12
|
+
end
|
13
|
+
|
14
|
+
it "returns a hash of tables that match the regex table_filter" do
|
15
|
+
@op = Taps::Operation.new('dummy://localhost', 'http://x:y@localhost:5000', :table_filter => 'abc')
|
16
|
+
@op.apply_table_filter({ 'abc' => 1, 'def' => 2 }).should == { 'abc' => 1 }
|
17
|
+
end
|
18
|
+
|
19
|
+
it "returns an array of tables without the exclude_tables tables" do
|
20
|
+
@op = Taps::Operation.new('dummy://localhost', 'http://x:y@localhost:5000', :exclude_tables => ['abc', 'ghi', 'jkl'])
|
21
|
+
@op.apply_table_filter(['abc', 'def', 'ghi', 'jkl', 'mno']).should == ['def', 'mno']
|
22
|
+
end
|
23
|
+
|
24
|
+
it "returns a hash of tables without the exclude_tables tables" do
|
25
|
+
@op = Taps::Operation.new('dummy://localhost', 'http://x:y@localhost:5000', :exclude_tables => ['abc', 'ghi', 'jkl'])
|
26
|
+
@op.apply_table_filter({ 'abc' => 1, 'def' => 2, 'ghi' => 3, 'jkl' => 4, 'mno' => 5 }).should == { 'def' => 2, 'mno' => 5 }
|
27
|
+
end
|
28
|
+
|
29
|
+
it "masks a url's password" do
|
30
|
+
@op.safe_url("mysql://root:password@localhost/mydb").should == "mysql://root:[hidden]@localhost/mydb"
|
31
|
+
end
|
32
|
+
|
33
|
+
it "returns http headers with compression enabled" do
|
34
|
+
@op.http_headers.should == { :taps_version => Taps.version, :accept_encoding => "gzip, deflate" }
|
35
|
+
end
|
36
|
+
|
37
|
+
it "returns http headers with compression disabled" do
|
38
|
+
@op.stubs(:compression_disabled?).returns(true)
|
39
|
+
@op.http_headers.should == { :taps_version => Taps.version, :accept_encoding => "" }
|
40
|
+
end
|
41
|
+
|
42
|
+
end
|
data/spec/server_spec.rb
ADDED
@@ -0,0 +1,40 @@
|
|
1
|
+
require File.dirname(__FILE__) + '/base'
|
2
|
+
|
3
|
+
require 'taps/server'
|
4
|
+
|
5
|
+
require 'pp'
|
6
|
+
|
7
|
+
describe Taps::Server do
|
8
|
+
def app
|
9
|
+
Taps::Server.new
|
10
|
+
end
|
11
|
+
|
12
|
+
before do
|
13
|
+
Taps::Config.login = 'taps'
|
14
|
+
Taps::Config.password = 'tpass'
|
15
|
+
|
16
|
+
@app = Taps::Server
|
17
|
+
@auth_header = "Basic " + ["taps:tpass"].pack("m*")
|
18
|
+
end
|
19
|
+
|
20
|
+
it "asks for http basic authentication" do
|
21
|
+
get '/'
|
22
|
+
last_response.status.should == 401
|
23
|
+
end
|
24
|
+
|
25
|
+
it "verifies the client taps version" do
|
26
|
+
get('/', { }, { 'HTTP_AUTHORIZATION' => @auth_header, 'HTTP_TAPS_VERSION' => Taps.version })
|
27
|
+
last_response.status.should == 200
|
28
|
+
end
|
29
|
+
|
30
|
+
it "yells loudly if the client taps version doesn't match" do
|
31
|
+
get('/', { }, { 'HTTP_AUTHORIZATION' => @auth_header, 'HTTP_TAPS_VERSION' => '0.0.1' })
|
32
|
+
last_response.status.should == 417
|
33
|
+
end
|
34
|
+
|
35
|
+
it "allows healthcheck to be accessed w/o HTTP_TAPS_VERSION" do
|
36
|
+
get('/health', { }, { 'HTTP_AUTHORIZATION' => @auth_header })
|
37
|
+
last_response.status.should == 200
|
38
|
+
end
|
39
|
+
end
|
40
|
+
|
data/spec/utils_spec.rb
ADDED
@@ -0,0 +1,30 @@
|
|
1
|
+
require File.dirname(__FILE__) + '/base'
|
2
|
+
require 'taps/utils'
|
3
|
+
|
4
|
+
describe Taps::Utils do
|
5
|
+
it "generates a checksum using crc32" do
|
6
|
+
Taps::Utils.checksum("hello world").should == Zlib.crc32("hello world")
|
7
|
+
end
|
8
|
+
|
9
|
+
it "formats a data hash into one hash that contains an array of headers and an array of array of data" do
|
10
|
+
first_row = { :x => 1, :y => 1 }
|
11
|
+
first_row.stubs(:keys).returns([:x, :y])
|
12
|
+
Taps::Utils.format_data([ first_row, { :x => 2, :y => 2 } ]).should == { :header => [ :x, :y ], :data => [ [1, 1], [2, 2] ] }
|
13
|
+
end
|
14
|
+
|
15
|
+
it "enforces length limitations on columns" do
|
16
|
+
data = [ { :a => "aaabbbccc" } ]
|
17
|
+
schema = [ [ :a, { :db_type => "varchar(3)" }]]
|
18
|
+
lambda { Taps::Utils.format_data(data, :schema => schema) }.should.raise(Taps::InvalidData)
|
19
|
+
end
|
20
|
+
|
21
|
+
it "returns a list of columns that are text fields if the database is mysql" do
|
22
|
+
@db = mock("db", :url => "mysql://localhost/mydb")
|
23
|
+
@db.stubs(:schema).with(:mytable).returns([
|
24
|
+
[:id, { :db_type => "int" }],
|
25
|
+
[:mytext, { :db_type => "text" }]
|
26
|
+
])
|
27
|
+
Taps::Utils.incorrect_blobs(@db, :mytable).should == [:mytext]
|
28
|
+
end
|
29
|
+
end
|
30
|
+
|
metadata
ADDED
@@ -0,0 +1,234 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: taps-taps
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.3.24
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Ricardo Chimal, Jr.
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
date: 2013-10-22 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: rack
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - '>='
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: 1.0.1
|
20
|
+
type: :runtime
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - '>='
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: 1.0.1
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: rest-client
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - '>='
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: 1.4.0
|
34
|
+
- - <
|
35
|
+
- !ruby/object:Gem::Version
|
36
|
+
version: 1.7.0
|
37
|
+
type: :runtime
|
38
|
+
prerelease: false
|
39
|
+
version_requirements: !ruby/object:Gem::Requirement
|
40
|
+
requirements:
|
41
|
+
- - '>='
|
42
|
+
- !ruby/object:Gem::Version
|
43
|
+
version: 1.4.0
|
44
|
+
- - <
|
45
|
+
- !ruby/object:Gem::Version
|
46
|
+
version: 1.7.0
|
47
|
+
- !ruby/object:Gem::Dependency
|
48
|
+
name: sequel
|
49
|
+
requirement: !ruby/object:Gem::Requirement
|
50
|
+
requirements:
|
51
|
+
- - ~>
|
52
|
+
- !ruby/object:Gem::Version
|
53
|
+
version: 3.20.0
|
54
|
+
type: :runtime
|
55
|
+
prerelease: false
|
56
|
+
version_requirements: !ruby/object:Gem::Requirement
|
57
|
+
requirements:
|
58
|
+
- - ~>
|
59
|
+
- !ruby/object:Gem::Version
|
60
|
+
version: 3.20.0
|
61
|
+
- !ruby/object:Gem::Dependency
|
62
|
+
name: sinatra
|
63
|
+
requirement: !ruby/object:Gem::Requirement
|
64
|
+
requirements:
|
65
|
+
- - ~>
|
66
|
+
- !ruby/object:Gem::Version
|
67
|
+
version: 1.4.4
|
68
|
+
type: :runtime
|
69
|
+
prerelease: false
|
70
|
+
version_requirements: !ruby/object:Gem::Requirement
|
71
|
+
requirements:
|
72
|
+
- - ~>
|
73
|
+
- !ruby/object:Gem::Version
|
74
|
+
version: 1.4.4
|
75
|
+
- !ruby/object:Gem::Dependency
|
76
|
+
name: sqlite3
|
77
|
+
requirement: !ruby/object:Gem::Requirement
|
78
|
+
requirements:
|
79
|
+
- - ~>
|
80
|
+
- !ruby/object:Gem::Version
|
81
|
+
version: 1.3.8
|
82
|
+
type: :runtime
|
83
|
+
prerelease: false
|
84
|
+
version_requirements: !ruby/object:Gem::Requirement
|
85
|
+
requirements:
|
86
|
+
- - ~>
|
87
|
+
- !ruby/object:Gem::Version
|
88
|
+
version: 1.3.8
|
89
|
+
- !ruby/object:Gem::Dependency
|
90
|
+
name: sqlite3
|
91
|
+
requirement: !ruby/object:Gem::Requirement
|
92
|
+
requirements:
|
93
|
+
- - ~>
|
94
|
+
- !ruby/object:Gem::Version
|
95
|
+
version: '1.2'
|
96
|
+
type: :development
|
97
|
+
prerelease: false
|
98
|
+
version_requirements: !ruby/object:Gem::Requirement
|
99
|
+
requirements:
|
100
|
+
- - ~>
|
101
|
+
- !ruby/object:Gem::Version
|
102
|
+
version: '1.2'
|
103
|
+
- !ruby/object:Gem::Dependency
|
104
|
+
name: bacon
|
105
|
+
requirement: !ruby/object:Gem::Requirement
|
106
|
+
requirements:
|
107
|
+
- - '>='
|
108
|
+
- !ruby/object:Gem::Version
|
109
|
+
version: '0'
|
110
|
+
type: :development
|
111
|
+
prerelease: false
|
112
|
+
version_requirements: !ruby/object:Gem::Requirement
|
113
|
+
requirements:
|
114
|
+
- - '>='
|
115
|
+
- !ruby/object:Gem::Version
|
116
|
+
version: '0'
|
117
|
+
- !ruby/object:Gem::Dependency
|
118
|
+
name: mocha
|
119
|
+
requirement: !ruby/object:Gem::Requirement
|
120
|
+
requirements:
|
121
|
+
- - '>='
|
122
|
+
- !ruby/object:Gem::Version
|
123
|
+
version: '0'
|
124
|
+
type: :development
|
125
|
+
prerelease: false
|
126
|
+
version_requirements: !ruby/object:Gem::Requirement
|
127
|
+
requirements:
|
128
|
+
- - '>='
|
129
|
+
- !ruby/object:Gem::Version
|
130
|
+
version: '0'
|
131
|
+
- !ruby/object:Gem::Dependency
|
132
|
+
name: rack-test
|
133
|
+
requirement: !ruby/object:Gem::Requirement
|
134
|
+
requirements:
|
135
|
+
- - '>='
|
136
|
+
- !ruby/object:Gem::Version
|
137
|
+
version: '0'
|
138
|
+
type: :development
|
139
|
+
prerelease: false
|
140
|
+
version_requirements: !ruby/object:Gem::Requirement
|
141
|
+
requirements:
|
142
|
+
- - '>='
|
143
|
+
- !ruby/object:Gem::Version
|
144
|
+
version: '0'
|
145
|
+
- !ruby/object:Gem::Dependency
|
146
|
+
name: rake
|
147
|
+
requirement: !ruby/object:Gem::Requirement
|
148
|
+
requirements:
|
149
|
+
- - '>='
|
150
|
+
- !ruby/object:Gem::Version
|
151
|
+
version: '0'
|
152
|
+
type: :development
|
153
|
+
prerelease: false
|
154
|
+
version_requirements: !ruby/object:Gem::Requirement
|
155
|
+
requirements:
|
156
|
+
- - '>='
|
157
|
+
- !ruby/object:Gem::Version
|
158
|
+
version: '0'
|
159
|
+
- !ruby/object:Gem::Dependency
|
160
|
+
name: simplecov
|
161
|
+
requirement: !ruby/object:Gem::Requirement
|
162
|
+
requirements:
|
163
|
+
- - '>='
|
164
|
+
- !ruby/object:Gem::Version
|
165
|
+
version: '0'
|
166
|
+
type: :development
|
167
|
+
prerelease: false
|
168
|
+
version_requirements: !ruby/object:Gem::Requirement
|
169
|
+
requirements:
|
170
|
+
- - '>='
|
171
|
+
- !ruby/object:Gem::Version
|
172
|
+
version: '0'
|
173
|
+
description: |-
|
174
|
+
A simple database agnostic import/export app to transfer data to/from a remote database.
|
175
|
+
Taps-taps includes fixes submitted by the community.
|
176
|
+
email: ricardo@heroku.com
|
177
|
+
executables:
|
178
|
+
- taps
|
179
|
+
- schema
|
180
|
+
extensions: []
|
181
|
+
extra_rdoc_files: []
|
182
|
+
files:
|
183
|
+
- bin/schema
|
184
|
+
- bin/schema.cmd
|
185
|
+
- bin/taps
|
186
|
+
- lib/taps/chunksize.rb
|
187
|
+
- lib/taps/cli.rb
|
188
|
+
- lib/taps/config.rb
|
189
|
+
- lib/taps/data_stream.rb
|
190
|
+
- lib/taps/db_session.rb
|
191
|
+
- lib/taps/errors.rb
|
192
|
+
- lib/taps/log.rb
|
193
|
+
- lib/taps/monkey.rb
|
194
|
+
- lib/taps/multipart.rb
|
195
|
+
- lib/taps/operation.rb
|
196
|
+
- lib/taps/progress_bar.rb
|
197
|
+
- lib/taps/schema.rb
|
198
|
+
- lib/taps/server.rb
|
199
|
+
- lib/taps/utils.rb
|
200
|
+
- lib/taps/version.rb
|
201
|
+
- lib/vendor/okjson.rb
|
202
|
+
- README.rdoc
|
203
|
+
- spec/base.rb
|
204
|
+
- spec/chunksize_spec.rb
|
205
|
+
- spec/cli_spec.rb
|
206
|
+
- spec/data_stream_spec.rb
|
207
|
+
- spec/operation_spec.rb
|
208
|
+
- spec/server_spec.rb
|
209
|
+
- spec/utils_spec.rb
|
210
|
+
- VERSION.yml
|
211
|
+
homepage: http://github.com/wijet/taps
|
212
|
+
licenses: []
|
213
|
+
metadata: {}
|
214
|
+
post_install_message:
|
215
|
+
rdoc_options: []
|
216
|
+
require_paths:
|
217
|
+
- lib
|
218
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
219
|
+
requirements:
|
220
|
+
- - '>='
|
221
|
+
- !ruby/object:Gem::Version
|
222
|
+
version: '0'
|
223
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
224
|
+
requirements:
|
225
|
+
- - '>='
|
226
|
+
- !ruby/object:Gem::Version
|
227
|
+
version: '0'
|
228
|
+
requirements: []
|
229
|
+
rubyforge_project:
|
230
|
+
rubygems_version: 2.0.3
|
231
|
+
signing_key:
|
232
|
+
specification_version: 4
|
233
|
+
summary: simple database import/export app
|
234
|
+
test_files: []
|