taps 0.3.11 → 0.3.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/Rakefile +3 -3
- data/VERSION.yml +1 -1
- data/bin/schema +28 -28
- data/lib/taps/cli.rb +171 -166
- data/lib/taps/config.rb +39 -39
- data/lib/taps/data_stream.rb +291 -291
- data/lib/taps/db_session.rb +13 -13
- data/lib/taps/log.rb +12 -12
- data/lib/taps/monkey.rb +17 -17
- data/lib/taps/multipart.rb +51 -51
- data/lib/taps/operation.rb +525 -525
- data/lib/taps/schema.rb +58 -58
- data/lib/taps/server.rb +154 -154
- data/lib/taps/utils.rb +145 -145
- data/spec/base.rb +11 -11
- data/spec/cli_spec.rb +5 -5
- data/spec/data_stream_spec.rb +16 -16
- data/spec/operation_spec.rb +21 -21
- data/spec/server_spec.rb +26 -26
- data/spec/utils_spec.rb +49 -49
- metadata +5 -5
data/lib/taps/utils.rb
CHANGED
@@ -5,150 +5,150 @@ require 'tempfile'
|
|
5
5
|
|
6
6
|
module Taps
|
7
7
|
module Utils
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
|
71
|
-
|
72
|
-
|
73
|
-
|
74
|
-
|
75
|
-
|
76
|
-
|
77
|
-
|
78
|
-
|
79
|
-
|
80
|
-
|
81
|
-
|
82
|
-
|
83
|
-
|
84
|
-
|
85
|
-
|
86
|
-
|
87
|
-
|
88
|
-
|
89
|
-
|
90
|
-
|
91
|
-
|
92
|
-
|
93
|
-
|
94
|
-
|
95
|
-
|
96
|
-
|
97
|
-
|
98
|
-
|
99
|
-
|
100
|
-
|
101
|
-
|
102
|
-
|
103
|
-
|
104
|
-
|
105
|
-
|
106
|
-
|
107
|
-
|
108
|
-
|
109
|
-
|
110
|
-
|
111
|
-
|
112
|
-
|
113
|
-
|
114
|
-
|
115
|
-
|
116
|
-
|
117
|
-
|
118
|
-
|
119
|
-
|
120
|
-
|
121
|
-
|
122
|
-
|
123
|
-
|
124
|
-
|
125
|
-
|
126
|
-
|
127
|
-
|
128
|
-
|
129
|
-
|
130
|
-
|
131
|
-
|
132
|
-
|
133
|
-
|
134
|
-
|
135
|
-
|
136
|
-
|
137
|
-
|
138
|
-
|
139
|
-
|
140
|
-
|
141
|
-
|
142
|
-
|
143
|
-
|
144
|
-
|
145
|
-
|
146
|
-
|
147
|
-
|
148
|
-
|
149
|
-
|
150
|
-
|
151
|
-
|
152
|
-
|
8
|
+
extend self
|
9
|
+
|
10
|
+
def windows?
|
11
|
+
return @windows if defined?(@windows)
|
12
|
+
require 'rbconfig'
|
13
|
+
@windows = !!(::Config::CONFIG['host_os'] =~ /mswin|mingw/)
|
14
|
+
end
|
15
|
+
|
16
|
+
def bin(cmd)
|
17
|
+
cmd = "#{cmd}.cmd" if windows?
|
18
|
+
cmd
|
19
|
+
end
|
20
|
+
|
21
|
+
def checksum(data)
|
22
|
+
Zlib.crc32(data)
|
23
|
+
end
|
24
|
+
|
25
|
+
def valid_data?(data, crc32)
|
26
|
+
Zlib.crc32(data) == crc32.to_i
|
27
|
+
end
|
28
|
+
|
29
|
+
def base64encode(data)
|
30
|
+
[data].pack("m")
|
31
|
+
end
|
32
|
+
|
33
|
+
def base64decode(data)
|
34
|
+
data.unpack("m").first
|
35
|
+
end
|
36
|
+
|
37
|
+
def format_data(data, opts={})
|
38
|
+
return {} if data.size == 0
|
39
|
+
string_columns = opts[:string_columns] || []
|
40
|
+
|
41
|
+
header = data[0].keys
|
42
|
+
only_data = data.collect do |row|
|
43
|
+
row = blobs_to_string(row, string_columns)
|
44
|
+
header.collect { |h| row[h] }
|
45
|
+
end
|
46
|
+
{ :header => header, :data => only_data }
|
47
|
+
end
|
48
|
+
|
49
|
+
# mysql text and blobs fields are handled the same way internally
|
50
|
+
# this is not true for other databases so we must check if the field is
|
51
|
+
# actually text and manually convert it back to a string
|
52
|
+
def incorrect_blobs(db, table)
|
53
|
+
return [] if (db.url =~ /mysql:\/\//).nil?
|
54
|
+
|
55
|
+
columns = []
|
56
|
+
db.schema(table).each do |data|
|
57
|
+
column, cdata = data
|
58
|
+
columns << column if cdata[:db_type] =~ /text/
|
59
|
+
end
|
60
|
+
columns
|
61
|
+
end
|
62
|
+
|
63
|
+
def blobs_to_string(row, columns)
|
64
|
+
return row if columns.size == 0
|
65
|
+
columns.each do |c|
|
66
|
+
row[c] = row[c].to_s if row[c].kind_of?(Sequel::SQL::Blob)
|
67
|
+
end
|
68
|
+
row
|
69
|
+
end
|
70
|
+
|
71
|
+
def calculate_chunksize(old_chunksize)
|
72
|
+
chunksize = old_chunksize
|
73
|
+
|
74
|
+
retries = 0
|
75
|
+
time_in_db = 0
|
76
|
+
begin
|
77
|
+
t1 = Time.now
|
78
|
+
time_in_db = yield chunksize
|
79
|
+
time_in_db = time_in_db.to_f rescue 0
|
80
|
+
rescue Errno::EPIPE, RestClient::RequestFailed, RestClient::RequestTimeout
|
81
|
+
retries += 1
|
82
|
+
raise if retries > 2
|
83
|
+
|
84
|
+
# we got disconnected, the chunksize could be too large
|
85
|
+
# on first retry change to 10, on successive retries go down to 1
|
86
|
+
chunksize = (retries == 1) ? 10 : 1
|
87
|
+
|
88
|
+
retry
|
89
|
+
end
|
90
|
+
|
91
|
+
t2 = Time.now
|
92
|
+
|
93
|
+
diff = t2 - t1 - time_in_db
|
94
|
+
|
95
|
+
new_chunksize = if retries > 0
|
96
|
+
chunksize
|
97
|
+
elsif diff > 3.0
|
98
|
+
(chunksize / 3).ceil
|
99
|
+
elsif diff > 1.1
|
100
|
+
chunksize - 100
|
101
|
+
elsif diff < 0.8
|
102
|
+
chunksize * 2
|
103
|
+
else
|
104
|
+
chunksize + 100
|
105
|
+
end
|
106
|
+
new_chunksize = 1 if new_chunksize < 1
|
107
|
+
new_chunksize
|
108
|
+
end
|
109
|
+
|
110
|
+
def load_schema(database_url, schema_data)
|
111
|
+
Tempfile.open('taps') do |tmp|
|
112
|
+
File.open(tmp.path, 'w') { |f| f.write(schema_data) }
|
113
|
+
schema_bin(:load, database_url, tmp.path)
|
114
|
+
end
|
115
|
+
end
|
116
|
+
|
117
|
+
def load_indexes(database_url, index_data)
|
118
|
+
Tempfile.open('taps') do |tmp|
|
119
|
+
File.open(tmp.path, 'w') { |f| f.write(index_data) }
|
120
|
+
schema_bin(:load_indexes, database_url, tmp.path)
|
121
|
+
end
|
122
|
+
end
|
123
|
+
|
124
|
+
def schema_bin(*args)
|
125
|
+
bin_path = File.expand_path("#{File.dirname(__FILE__)}/../../bin/#{bin('schema')}")
|
126
|
+
`"#{bin_path}" #{args.map { |a| "'#{a}'" }.join(' ')}`
|
127
|
+
end
|
128
|
+
|
129
|
+
def primary_key(db, table)
|
130
|
+
table = table.to_sym.identifier unless table.kind_of?(Sequel::SQL::Identifier)
|
131
|
+
if db.respond_to?(:primary_key)
|
132
|
+
db.primary_key(table)
|
133
|
+
else
|
134
|
+
db.schema(table).select { |c| c[1][:primary_key] }.map { |c| c.first.to_sym }
|
135
|
+
end
|
136
|
+
end
|
137
|
+
|
138
|
+
def single_integer_primary_key(db, table)
|
139
|
+
table = table.to_sym.identifier unless table.kind_of?(Sequel::SQL::Identifier)
|
140
|
+
keys = db.schema(table).select { |c| c[1][:primary_key] and c[1][:type] == :integer }
|
141
|
+
not keys.nil? and keys.size == 1
|
142
|
+
end
|
143
|
+
|
144
|
+
def order_by(db, table)
|
145
|
+
pkey = primary_key(db, table)
|
146
|
+
if pkey
|
147
|
+
pkey.kind_of?(Array) ? pkey : [pkey.to_sym]
|
148
|
+
else
|
149
|
+
table = table.to_sym.identifier unless table.kind_of?(Sequel::SQL::Identifier)
|
150
|
+
db[table].columns
|
151
|
+
end
|
152
|
+
end
|
153
153
|
end
|
154
154
|
end
|
data/spec/base.rb
CHANGED
@@ -7,18 +7,18 @@ require 'tempfile'
|
|
7
7
|
$:.unshift File.dirname(__FILE__) + "/../lib"
|
8
8
|
|
9
9
|
class Bacon::Context
|
10
|
-
|
11
|
-
|
10
|
+
include Mocha::Standalone
|
11
|
+
include Rack::Test::Methods
|
12
12
|
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
13
|
+
alias_method :old_it, :it
|
14
|
+
def it(description)
|
15
|
+
old_it(description) do
|
16
|
+
mocha_setup
|
17
|
+
yield
|
18
|
+
mocha_verify
|
19
|
+
mocha_teardown
|
20
|
+
end
|
21
|
+
end
|
22
22
|
end
|
23
23
|
|
24
24
|
require 'taps/config'
|
data/spec/cli_spec.rb
CHANGED
@@ -2,9 +2,9 @@ require File.dirname(__FILE__) + '/base'
|
|
2
2
|
require 'taps/cli'
|
3
3
|
|
4
4
|
describe Taps::Cli do
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
5
|
+
it "translates a list of tables into a regex that can be used in table_filter" do
|
6
|
+
@cli = Taps::Cli.new(["-t", "mytable1,logs", "sqlite://tmp.db", "http://x:y@localhost:5000"])
|
7
|
+
opts = @cli.clientoptparse(:pull)
|
8
|
+
opts[:table_filter].should == "(^mytable1$|^logs$)"
|
9
|
+
end
|
10
10
|
end
|
data/spec/data_stream_spec.rb
CHANGED
@@ -2,22 +2,22 @@ require File.dirname(__FILE__) + '/base'
|
|
2
2
|
require 'taps/data_stream'
|
3
3
|
|
4
4
|
describe Taps::DataStream do
|
5
|
-
|
6
|
-
|
7
|
-
|
5
|
+
before do
|
6
|
+
@db = mock('db')
|
7
|
+
end
|
8
8
|
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
9
|
+
it "increments the offset" do
|
10
|
+
stream = Taps::DataStream.new(@db, :table_name => 'test_table', :chunksize => 100)
|
11
|
+
stream.state[:offset].should == 0
|
12
|
+
stream.increment(100)
|
13
|
+
stream.state[:offset].should == 100
|
14
|
+
end
|
15
15
|
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
16
|
+
it "marks the stream complete if no rows are fetched" do
|
17
|
+
stream = Taps::DataStream.new(@db, :table_name => 'test_table', :chunksize => 100)
|
18
|
+
stream.stubs(:fetch_rows).returns({})
|
19
|
+
stream.complete?.should.be.false
|
20
|
+
stream.fetch
|
21
|
+
stream.complete?.should.be.true
|
22
|
+
end
|
23
23
|
end
|
data/spec/operation_spec.rb
CHANGED
@@ -2,31 +2,31 @@ require File.dirname(__FILE__) + '/base'
|
|
2
2
|
require 'taps/operation'
|
3
3
|
|
4
4
|
describe Taps::Operation do
|
5
|
-
|
6
|
-
|
7
|
-
|
5
|
+
before do
|
6
|
+
@op = Taps::Operation.new('dummy://localhost', 'http://x:y@localhost:5000')
|
7
|
+
end
|
8
8
|
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
9
|
+
it "returns an array of tables that match the regex table_filter" do
|
10
|
+
@op = Taps::Operation.new('dummy://localhost', 'http://x:y@localhost:5000', :table_filter => 'abc')
|
11
|
+
@op.apply_table_filter(['abc', 'def']).should == ['abc']
|
12
|
+
end
|
13
13
|
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
14
|
+
it "returns a hash of tables that match the regex table_filter" do
|
15
|
+
@op = Taps::Operation.new('dummy://localhost', 'http://x:y@localhost:5000', :table_filter => 'abc')
|
16
|
+
@op.apply_table_filter({ 'abc' => 1, 'def' => 2 }).should == { 'abc' => 1 }
|
17
|
+
end
|
18
18
|
|
19
|
-
|
20
|
-
|
21
|
-
|
19
|
+
it "masks a url's password" do
|
20
|
+
@op.safe_url("mysql://root:password@localhost/mydb").should == "mysql://root:[hidden]@localhost/mydb"
|
21
|
+
end
|
22
22
|
|
23
|
-
|
24
|
-
|
25
|
-
|
23
|
+
it "returns http headers with compression enabled" do
|
24
|
+
@op.http_headers.should == { :taps_version => Taps.version, :accept_encoding => "gzip, deflate" }
|
25
|
+
end
|
26
26
|
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
27
|
+
it "returns http headers with compression disabled" do
|
28
|
+
@op.stubs(:compression_disabled?).returns(true)
|
29
|
+
@op.http_headers.should == { :taps_version => Taps.version, :accept_encoding => "" }
|
30
|
+
end
|
31
31
|
|
32
32
|
end
|
data/spec/server_spec.rb
CHANGED
@@ -5,31 +5,31 @@ require 'taps/server'
|
|
5
5
|
require 'pp'
|
6
6
|
|
7
7
|
describe Taps::Server do
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
8
|
+
def app
|
9
|
+
Taps::Server.new
|
10
|
+
end
|
11
|
+
|
12
|
+
before do
|
13
|
+
Taps::Config.login = 'taps'
|
14
|
+
Taps::Config.password = 'tpass'
|
15
|
+
|
16
|
+
@app = Taps::Server
|
17
|
+
@auth_header = "Basic " + ["taps:tpass"].pack("m*")
|
18
|
+
end
|
19
|
+
|
20
|
+
it "asks for http basic authentication" do
|
21
|
+
get '/'
|
22
|
+
last_response.status.should == 401
|
23
|
+
end
|
24
|
+
|
25
|
+
it "verifies the client taps version" do
|
26
|
+
get('/', { }, { 'HTTP_AUTHORIZATION' => @auth_header, 'HTTP_TAPS_VERSION' => Taps.compatible_version })
|
27
|
+
last_response.status.should == 200
|
28
|
+
end
|
29
|
+
|
30
|
+
it "yells loudly if the client taps version doesn't match" do
|
31
|
+
get('/', { }, { 'HTTP_AUTHORIZATION' => @auth_header, 'HTTP_TAPS_VERSION' => '0.0.1' })
|
32
|
+
last_response.status.should == 417
|
33
|
+
end
|
34
34
|
end
|
35
35
|
|