taps-taps 0.3.24

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,26 @@
1
+ require 'rubygems'
2
+ require 'bacon'
3
+ require 'mocha'
4
+ require 'rack/test'
5
+ require 'tempfile'
6
+
7
+ $:.unshift File.dirname(__FILE__) + "/../lib"
8
+
9
+ class Bacon::Context
10
+ include Mocha::Standalone
11
+ include Rack::Test::Methods
12
+
13
+ alias_method :old_it, :it
14
+ def it(description)
15
+ old_it(description) do
16
+ mocha_setup
17
+ yield
18
+ mocha_verify
19
+ mocha_teardown
20
+ end
21
+ end
22
+ end
23
+
24
+ require 'taps/config'
25
+ Taps::Config.taps_database_url = "sqlite://#{Tempfile.new('test.db').path}"
26
+ Sequel.connect(Taps::Config.taps_database_url)
@@ -0,0 +1,41 @@
1
+ require File.dirname(__FILE__) + '/base'
2
+ require 'taps/utils'
3
+
4
+ describe Taps::Chunksize do
5
+ it "scales chunksize down slowly when the time delta of the block is just over a second" do
6
+ Time.stubs(:now).returns(10.0).returns(11.5)
7
+ Taps::Utils.calculate_chunksize(1000) { |c| }.should == 900
8
+ end
9
+
10
+ it "scales chunksize down fast when the time delta of the block is over 3 seconds" do
11
+ Time.stubs(:now).returns(10.0).returns(15.0)
12
+ Taps::Utils.calculate_chunksize(3000) { |c| }.should == 1000
13
+ end
14
+
15
+ it "scales up chunksize fast when the time delta of the block is under 0.8 seconds" do
16
+ Time.stubs(:now).returns(10.0).returns(10.7)
17
+ Taps::Utils.calculate_chunksize(1000) { |c| }.should == 2000
18
+ end
19
+
20
+ it "scales up chunksize slow when the time delta of the block is between 0.8 and 1.1 seconds" do
21
+ Time.stubs(:now).returns(10.0).returns(10.8)
22
+ Taps::Utils.calculate_chunksize(1000) { |c| }.should == 1100
23
+
24
+ Time.stubs(:now).returns(10.0).returns(11.1)
25
+ Taps::Utils.calculate_chunksize(1000) { |c| }.should == 1100
26
+ end
27
+
28
+ it "will reset the chunksize to a small value if we got a broken pipe exception" do
29
+ Taps::Utils.calculate_chunksize(1000) do |c|
30
+ raise Errno::EPIPE if c.chunksize == 1000
31
+ c.chunksize.should == 10
32
+ end.should == 10
33
+ end
34
+
35
+ it "will reset the chunksize to a small value if we got a broken pipe exception a second time" do
36
+ Taps::Utils.calculate_chunksize(1000) do |c|
37
+ raise Errno::EPIPE if c.chunksize == 1000 || c.chunksize == 10
38
+ c.chunksize.should == 1
39
+ end.should == 1
40
+ end
41
+ end
@@ -0,0 +1,16 @@
1
+ require File.dirname(__FILE__) + '/base'
2
+ require 'taps/cli'
3
+
4
+ describe Taps::Cli do
5
+ it "translates a list of tables into a regex that can be used in table_filter" do
6
+ @cli = Taps::Cli.new(["-t", "mytable1,logs", "sqlite://tmp.db", "http://x:y@localhost:5000"])
7
+ opts = @cli.clientoptparse(:pull)
8
+ opts[:table_filter].should == "(^mytable1$|^logs$)"
9
+ end
10
+
11
+ it "translates a list of tables to exclude into a regex that can be used in table_filter" do
12
+ @cli = Taps::Cli.new(["-e", "mytable1,logs", "sqlite://tmp.db", "http://x:y@localhost:5000"])
13
+ opts = @cli.clientoptparse(:pull)
14
+ opts[:exclude_tables].should == ['mytable1','logs']
15
+ end
16
+ end
@@ -0,0 +1,23 @@
1
+ require File.dirname(__FILE__) + '/base'
2
+ require 'taps/data_stream'
3
+
4
+ describe Taps::DataStream do
5
+ before do
6
+ @db = mock('db')
7
+ end
8
+
9
+ it "increments the offset" do
10
+ stream = Taps::DataStream.new(@db, :table_name => 'test_table', :chunksize => 100)
11
+ stream.state[:offset].should == 0
12
+ stream.increment(100)
13
+ stream.state[:offset].should == 100
14
+ end
15
+
16
+ it "marks the stream complete if no rows are fetched" do
17
+ stream = Taps::DataStream.new(@db, :table_name => 'test_table', :chunksize => 100)
18
+ stream.stubs(:fetch_rows).returns({})
19
+ stream.complete?.should.be.false
20
+ stream.fetch
21
+ stream.complete?.should.be.true
22
+ end
23
+ end
@@ -0,0 +1,42 @@
1
+ require File.dirname(__FILE__) + '/base'
2
+ require 'taps/operation'
3
+
4
+ describe Taps::Operation do
5
+ before do
6
+ @op = Taps::Operation.new('dummy://localhost', 'http://x:y@localhost:5000')
7
+ end
8
+
9
+ it "returns an array of tables that match the regex table_filter" do
10
+ @op = Taps::Operation.new('dummy://localhost', 'http://x:y@localhost:5000', :table_filter => 'abc')
11
+ @op.apply_table_filter(['abc', 'def']).should == ['abc']
12
+ end
13
+
14
+ it "returns a hash of tables that match the regex table_filter" do
15
+ @op = Taps::Operation.new('dummy://localhost', 'http://x:y@localhost:5000', :table_filter => 'abc')
16
+ @op.apply_table_filter({ 'abc' => 1, 'def' => 2 }).should == { 'abc' => 1 }
17
+ end
18
+
19
+ it "returns an array of tables without the exclude_tables tables" do
20
+ @op = Taps::Operation.new('dummy://localhost', 'http://x:y@localhost:5000', :exclude_tables => ['abc', 'ghi', 'jkl'])
21
+ @op.apply_table_filter(['abc', 'def', 'ghi', 'jkl', 'mno']).should == ['def', 'mno']
22
+ end
23
+
24
+ it "returns a hash of tables without the exclude_tables tables" do
25
+ @op = Taps::Operation.new('dummy://localhost', 'http://x:y@localhost:5000', :exclude_tables => ['abc', 'ghi', 'jkl'])
26
+ @op.apply_table_filter({ 'abc' => 1, 'def' => 2, 'ghi' => 3, 'jkl' => 4, 'mno' => 5 }).should == { 'def' => 2, 'mno' => 5 }
27
+ end
28
+
29
+ it "masks a url's password" do
30
+ @op.safe_url("mysql://root:password@localhost/mydb").should == "mysql://root:[hidden]@localhost/mydb"
31
+ end
32
+
33
+ it "returns http headers with compression enabled" do
34
+ @op.http_headers.should == { :taps_version => Taps.version, :accept_encoding => "gzip, deflate" }
35
+ end
36
+
37
+ it "returns http headers with compression disabled" do
38
+ @op.stubs(:compression_disabled?).returns(true)
39
+ @op.http_headers.should == { :taps_version => Taps.version, :accept_encoding => "" }
40
+ end
41
+
42
+ end
@@ -0,0 +1,40 @@
1
+ require File.dirname(__FILE__) + '/base'
2
+
3
+ require 'taps/server'
4
+
5
+ require 'pp'
6
+
7
+ describe Taps::Server do
8
+ def app
9
+ Taps::Server.new
10
+ end
11
+
12
+ before do
13
+ Taps::Config.login = 'taps'
14
+ Taps::Config.password = 'tpass'
15
+
16
+ @app = Taps::Server
17
+ @auth_header = "Basic " + ["taps:tpass"].pack("m*")
18
+ end
19
+
20
+ it "asks for http basic authentication" do
21
+ get '/'
22
+ last_response.status.should == 401
23
+ end
24
+
25
+ it "verifies the client taps version" do
26
+ get('/', { }, { 'HTTP_AUTHORIZATION' => @auth_header, 'HTTP_TAPS_VERSION' => Taps.version })
27
+ last_response.status.should == 200
28
+ end
29
+
30
+ it "yells loudly if the client taps version doesn't match" do
31
+ get('/', { }, { 'HTTP_AUTHORIZATION' => @auth_header, 'HTTP_TAPS_VERSION' => '0.0.1' })
32
+ last_response.status.should == 417
33
+ end
34
+
35
+ it "allows healthcheck to be accessed w/o HTTP_TAPS_VERSION" do
36
+ get('/health', { }, { 'HTTP_AUTHORIZATION' => @auth_header })
37
+ last_response.status.should == 200
38
+ end
39
+ end
40
+
@@ -0,0 +1,30 @@
1
+ require File.dirname(__FILE__) + '/base'
2
+ require 'taps/utils'
3
+
4
+ describe Taps::Utils do
5
+ it "generates a checksum using crc32" do
6
+ Taps::Utils.checksum("hello world").should == Zlib.crc32("hello world")
7
+ end
8
+
9
+ it "formats a data hash into one hash that contains an array of headers and an array of array of data" do
10
+ first_row = { :x => 1, :y => 1 }
11
+ first_row.stubs(:keys).returns([:x, :y])
12
+ Taps::Utils.format_data([ first_row, { :x => 2, :y => 2 } ]).should == { :header => [ :x, :y ], :data => [ [1, 1], [2, 2] ] }
13
+ end
14
+
15
+ it "enforces length limitations on columns" do
16
+ data = [ { :a => "aaabbbccc" } ]
17
+ schema = [ [ :a, { :db_type => "varchar(3)" }]]
18
+ lambda { Taps::Utils.format_data(data, :schema => schema) }.should.raise(Taps::InvalidData)
19
+ end
20
+
21
+ it "returns a list of columns that are text fields if the database is mysql" do
22
+ @db = mock("db", :url => "mysql://localhost/mydb")
23
+ @db.stubs(:schema).with(:mytable).returns([
24
+ [:id, { :db_type => "int" }],
25
+ [:mytext, { :db_type => "text" }]
26
+ ])
27
+ Taps::Utils.incorrect_blobs(@db, :mytable).should == [:mytext]
28
+ end
29
+ end
30
+
metadata ADDED
@@ -0,0 +1,234 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: taps-taps
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.3.24
5
+ platform: ruby
6
+ authors:
7
+ - Ricardo Chimal, Jr.
8
+ autorequire:
9
+ bindir: bin
10
+ cert_chain: []
11
+ date: 2013-10-22 00:00:00.000000000 Z
12
+ dependencies:
13
+ - !ruby/object:Gem::Dependency
14
+ name: rack
15
+ requirement: !ruby/object:Gem::Requirement
16
+ requirements:
17
+ - - '>='
18
+ - !ruby/object:Gem::Version
19
+ version: 1.0.1
20
+ type: :runtime
21
+ prerelease: false
22
+ version_requirements: !ruby/object:Gem::Requirement
23
+ requirements:
24
+ - - '>='
25
+ - !ruby/object:Gem::Version
26
+ version: 1.0.1
27
+ - !ruby/object:Gem::Dependency
28
+ name: rest-client
29
+ requirement: !ruby/object:Gem::Requirement
30
+ requirements:
31
+ - - '>='
32
+ - !ruby/object:Gem::Version
33
+ version: 1.4.0
34
+ - - <
35
+ - !ruby/object:Gem::Version
36
+ version: 1.7.0
37
+ type: :runtime
38
+ prerelease: false
39
+ version_requirements: !ruby/object:Gem::Requirement
40
+ requirements:
41
+ - - '>='
42
+ - !ruby/object:Gem::Version
43
+ version: 1.4.0
44
+ - - <
45
+ - !ruby/object:Gem::Version
46
+ version: 1.7.0
47
+ - !ruby/object:Gem::Dependency
48
+ name: sequel
49
+ requirement: !ruby/object:Gem::Requirement
50
+ requirements:
51
+ - - ~>
52
+ - !ruby/object:Gem::Version
53
+ version: 3.20.0
54
+ type: :runtime
55
+ prerelease: false
56
+ version_requirements: !ruby/object:Gem::Requirement
57
+ requirements:
58
+ - - ~>
59
+ - !ruby/object:Gem::Version
60
+ version: 3.20.0
61
+ - !ruby/object:Gem::Dependency
62
+ name: sinatra
63
+ requirement: !ruby/object:Gem::Requirement
64
+ requirements:
65
+ - - ~>
66
+ - !ruby/object:Gem::Version
67
+ version: 1.4.4
68
+ type: :runtime
69
+ prerelease: false
70
+ version_requirements: !ruby/object:Gem::Requirement
71
+ requirements:
72
+ - - ~>
73
+ - !ruby/object:Gem::Version
74
+ version: 1.4.4
75
+ - !ruby/object:Gem::Dependency
76
+ name: sqlite3
77
+ requirement: !ruby/object:Gem::Requirement
78
+ requirements:
79
+ - - ~>
80
+ - !ruby/object:Gem::Version
81
+ version: 1.3.8
82
+ type: :runtime
83
+ prerelease: false
84
+ version_requirements: !ruby/object:Gem::Requirement
85
+ requirements:
86
+ - - ~>
87
+ - !ruby/object:Gem::Version
88
+ version: 1.3.8
89
+ - !ruby/object:Gem::Dependency
90
+ name: sqlite3
91
+ requirement: !ruby/object:Gem::Requirement
92
+ requirements:
93
+ - - ~>
94
+ - !ruby/object:Gem::Version
95
+ version: '1.2'
96
+ type: :development
97
+ prerelease: false
98
+ version_requirements: !ruby/object:Gem::Requirement
99
+ requirements:
100
+ - - ~>
101
+ - !ruby/object:Gem::Version
102
+ version: '1.2'
103
+ - !ruby/object:Gem::Dependency
104
+ name: bacon
105
+ requirement: !ruby/object:Gem::Requirement
106
+ requirements:
107
+ - - '>='
108
+ - !ruby/object:Gem::Version
109
+ version: '0'
110
+ type: :development
111
+ prerelease: false
112
+ version_requirements: !ruby/object:Gem::Requirement
113
+ requirements:
114
+ - - '>='
115
+ - !ruby/object:Gem::Version
116
+ version: '0'
117
+ - !ruby/object:Gem::Dependency
118
+ name: mocha
119
+ requirement: !ruby/object:Gem::Requirement
120
+ requirements:
121
+ - - '>='
122
+ - !ruby/object:Gem::Version
123
+ version: '0'
124
+ type: :development
125
+ prerelease: false
126
+ version_requirements: !ruby/object:Gem::Requirement
127
+ requirements:
128
+ - - '>='
129
+ - !ruby/object:Gem::Version
130
+ version: '0'
131
+ - !ruby/object:Gem::Dependency
132
+ name: rack-test
133
+ requirement: !ruby/object:Gem::Requirement
134
+ requirements:
135
+ - - '>='
136
+ - !ruby/object:Gem::Version
137
+ version: '0'
138
+ type: :development
139
+ prerelease: false
140
+ version_requirements: !ruby/object:Gem::Requirement
141
+ requirements:
142
+ - - '>='
143
+ - !ruby/object:Gem::Version
144
+ version: '0'
145
+ - !ruby/object:Gem::Dependency
146
+ name: rake
147
+ requirement: !ruby/object:Gem::Requirement
148
+ requirements:
149
+ - - '>='
150
+ - !ruby/object:Gem::Version
151
+ version: '0'
152
+ type: :development
153
+ prerelease: false
154
+ version_requirements: !ruby/object:Gem::Requirement
155
+ requirements:
156
+ - - '>='
157
+ - !ruby/object:Gem::Version
158
+ version: '0'
159
+ - !ruby/object:Gem::Dependency
160
+ name: simplecov
161
+ requirement: !ruby/object:Gem::Requirement
162
+ requirements:
163
+ - - '>='
164
+ - !ruby/object:Gem::Version
165
+ version: '0'
166
+ type: :development
167
+ prerelease: false
168
+ version_requirements: !ruby/object:Gem::Requirement
169
+ requirements:
170
+ - - '>='
171
+ - !ruby/object:Gem::Version
172
+ version: '0'
173
+ description: |-
174
+ A simple database agnostic import/export app to transfer data to/from a remote database.
175
+ Taps-taps includes fixes submitted by the community.
176
+ email: ricardo@heroku.com
177
+ executables:
178
+ - taps
179
+ - schema
180
+ extensions: []
181
+ extra_rdoc_files: []
182
+ files:
183
+ - bin/schema
184
+ - bin/schema.cmd
185
+ - bin/taps
186
+ - lib/taps/chunksize.rb
187
+ - lib/taps/cli.rb
188
+ - lib/taps/config.rb
189
+ - lib/taps/data_stream.rb
190
+ - lib/taps/db_session.rb
191
+ - lib/taps/errors.rb
192
+ - lib/taps/log.rb
193
+ - lib/taps/monkey.rb
194
+ - lib/taps/multipart.rb
195
+ - lib/taps/operation.rb
196
+ - lib/taps/progress_bar.rb
197
+ - lib/taps/schema.rb
198
+ - lib/taps/server.rb
199
+ - lib/taps/utils.rb
200
+ - lib/taps/version.rb
201
+ - lib/vendor/okjson.rb
202
+ - README.rdoc
203
+ - spec/base.rb
204
+ - spec/chunksize_spec.rb
205
+ - spec/cli_spec.rb
206
+ - spec/data_stream_spec.rb
207
+ - spec/operation_spec.rb
208
+ - spec/server_spec.rb
209
+ - spec/utils_spec.rb
210
+ - VERSION.yml
211
+ homepage: http://github.com/wijet/taps
212
+ licenses: []
213
+ metadata: {}
214
+ post_install_message:
215
+ rdoc_options: []
216
+ require_paths:
217
+ - lib
218
+ required_ruby_version: !ruby/object:Gem::Requirement
219
+ requirements:
220
+ - - '>='
221
+ - !ruby/object:Gem::Version
222
+ version: '0'
223
+ required_rubygems_version: !ruby/object:Gem::Requirement
224
+ requirements:
225
+ - - '>='
226
+ - !ruby/object:Gem::Version
227
+ version: '0'
228
+ requirements: []
229
+ rubyforge_project:
230
+ rubygems_version: 2.0.3
231
+ signing_key:
232
+ specification_version: 4
233
+ summary: simple database import/export app
234
+ test_files: []