taps 0.3.11 → 0.3.12

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
data/lib/taps/utils.rb CHANGED
@@ -5,150 +5,150 @@ require 'tempfile'
5
5
 
6
6
  module Taps
7
7
  module Utils
8
- extend self
9
-
10
- def windows?
11
- return @windows if defined?(@windows)
12
- require 'rbconfig'
13
- @windows = !!(::Config::CONFIG['host_os'] =~ /mswin|mingw/)
14
- end
15
-
16
- def bin(cmd)
17
- cmd = "#{cmd}.cmd" if windows?
18
- cmd
19
- end
20
-
21
- def checksum(data)
22
- Zlib.crc32(data)
23
- end
24
-
25
- def valid_data?(data, crc32)
26
- Zlib.crc32(data) == crc32.to_i
27
- end
28
-
29
- def base64encode(data)
30
- [data].pack("m")
31
- end
32
-
33
- def base64decode(data)
34
- data.unpack("m").first
35
- end
36
-
37
- def format_data(data, opts={})
38
- return {} if data.size == 0
39
- string_columns = opts[:string_columns] || []
40
-
41
- header = data[0].keys
42
- only_data = data.collect do |row|
43
- row = blobs_to_string(row, string_columns)
44
- header.collect { |h| row[h] }
45
- end
46
- { :header => header, :data => only_data }
47
- end
48
-
49
- # mysql text and blobs fields are handled the same way internally
50
- # this is not true for other databases so we must check if the field is
51
- # actually text and manually convert it back to a string
52
- def incorrect_blobs(db, table)
53
- return [] if (db.url =~ /mysql:\/\//).nil?
54
-
55
- columns = []
56
- db.schema(table).each do |data|
57
- column, cdata = data
58
- columns << column if cdata[:db_type] =~ /text/
59
- end
60
- columns
61
- end
62
-
63
- def blobs_to_string(row, columns)
64
- return row if columns.size == 0
65
- columns.each do |c|
66
- row[c] = row[c].to_s if row[c].kind_of?(Sequel::SQL::Blob)
67
- end
68
- row
69
- end
70
-
71
- def calculate_chunksize(old_chunksize)
72
- chunksize = old_chunksize
73
-
74
- retries = 0
75
- time_in_db = 0
76
- begin
77
- t1 = Time.now
78
- time_in_db = yield chunksize
79
- time_in_db = time_in_db.to_f rescue 0
80
- rescue Errno::EPIPE, RestClient::RequestFailed, RestClient::RequestTimeout
81
- retries += 1
82
- raise if retries > 2
83
-
84
- # we got disconnected, the chunksize could be too large
85
- # on first retry change to 10, on successive retries go down to 1
86
- chunksize = (retries == 1) ? 10 : 1
87
-
88
- retry
89
- end
90
-
91
- t2 = Time.now
92
-
93
- diff = t2 - t1 - time_in_db
94
-
95
- new_chunksize = if retries > 0
96
- chunksize
97
- elsif diff > 3.0
98
- (chunksize / 3).ceil
99
- elsif diff > 1.1
100
- chunksize - 100
101
- elsif diff < 0.8
102
- chunksize * 2
103
- else
104
- chunksize + 100
105
- end
106
- new_chunksize = 1 if new_chunksize < 1
107
- new_chunksize
108
- end
109
-
110
- def load_schema(database_url, schema_data)
111
- Tempfile.open('taps') do |tmp|
112
- File.open(tmp.path, 'w') { |f| f.write(schema_data) }
113
- schema_bin(:load, database_url, tmp.path)
114
- end
115
- end
116
-
117
- def load_indexes(database_url, index_data)
118
- Tempfile.open('taps') do |tmp|
119
- File.open(tmp.path, 'w') { |f| f.write(index_data) }
120
- schema_bin(:load_indexes, database_url, tmp.path)
121
- end
122
- end
123
-
124
- def schema_bin(*args)
125
- bin_path = File.expand_path("#{File.dirname(__FILE__)}/../../bin/#{bin('schema')}")
126
- `"#{bin_path}" #{args.map { |a| "'#{a}'" }.join(' ')}`
127
- end
128
-
129
- def primary_key(db, table)
130
- table = table.to_sym.identifier unless table.kind_of?(Sequel::SQL::Identifier)
131
- if db.respond_to?(:primary_key)
132
- db.primary_key(table)
133
- else
134
- db.schema(table).select { |c| c[1][:primary_key] }.map { |c| c.first.to_sym }
135
- end
136
- end
137
-
138
- def single_integer_primary_key(db, table)
139
- table = table.to_sym.identifier unless table.kind_of?(Sequel::SQL::Identifier)
140
- keys = db.schema(table).select { |c| c[1][:primary_key] and c[1][:type] == :integer }
141
- not keys.nil? and keys.size == 1
142
- end
143
-
144
- def order_by(db, table)
145
- pkey = primary_key(db, table)
146
- if pkey
147
- pkey.kind_of?(Array) ? pkey : [pkey.to_sym]
148
- else
149
- table = table.to_sym.identifier unless table.kind_of?(Sequel::SQL::Identifier)
150
- db[table].columns
151
- end
152
- end
8
+ extend self
9
+
10
+ def windows?
11
+ return @windows if defined?(@windows)
12
+ require 'rbconfig'
13
+ @windows = !!(::Config::CONFIG['host_os'] =~ /mswin|mingw/)
14
+ end
15
+
16
+ def bin(cmd)
17
+ cmd = "#{cmd}.cmd" if windows?
18
+ cmd
19
+ end
20
+
21
+ def checksum(data)
22
+ Zlib.crc32(data)
23
+ end
24
+
25
+ def valid_data?(data, crc32)
26
+ Zlib.crc32(data) == crc32.to_i
27
+ end
28
+
29
+ def base64encode(data)
30
+ [data].pack("m")
31
+ end
32
+
33
+ def base64decode(data)
34
+ data.unpack("m").first
35
+ end
36
+
37
+ def format_data(data, opts={})
38
+ return {} if data.size == 0
39
+ string_columns = opts[:string_columns] || []
40
+
41
+ header = data[0].keys
42
+ only_data = data.collect do |row|
43
+ row = blobs_to_string(row, string_columns)
44
+ header.collect { |h| row[h] }
45
+ end
46
+ { :header => header, :data => only_data }
47
+ end
48
+
49
+ # mysql text and blobs fields are handled the same way internally
50
+ # this is not true for other databases so we must check if the field is
51
+ # actually text and manually convert it back to a string
52
+ def incorrect_blobs(db, table)
53
+ return [] if (db.url =~ /mysql:\/\//).nil?
54
+
55
+ columns = []
56
+ db.schema(table).each do |data|
57
+ column, cdata = data
58
+ columns << column if cdata[:db_type] =~ /text/
59
+ end
60
+ columns
61
+ end
62
+
63
+ def blobs_to_string(row, columns)
64
+ return row if columns.size == 0
65
+ columns.each do |c|
66
+ row[c] = row[c].to_s if row[c].kind_of?(Sequel::SQL::Blob)
67
+ end
68
+ row
69
+ end
70
+
71
+ def calculate_chunksize(old_chunksize)
72
+ chunksize = old_chunksize
73
+
74
+ retries = 0
75
+ time_in_db = 0
76
+ begin
77
+ t1 = Time.now
78
+ time_in_db = yield chunksize
79
+ time_in_db = time_in_db.to_f rescue 0
80
+ rescue Errno::EPIPE, RestClient::RequestFailed, RestClient::RequestTimeout
81
+ retries += 1
82
+ raise if retries > 2
83
+
84
+ # we got disconnected, the chunksize could be too large
85
+ # on first retry change to 10, on successive retries go down to 1
86
+ chunksize = (retries == 1) ? 10 : 1
87
+
88
+ retry
89
+ end
90
+
91
+ t2 = Time.now
92
+
93
+ diff = t2 - t1 - time_in_db
94
+
95
+ new_chunksize = if retries > 0
96
+ chunksize
97
+ elsif diff > 3.0
98
+ (chunksize / 3).ceil
99
+ elsif diff > 1.1
100
+ chunksize - 100
101
+ elsif diff < 0.8
102
+ chunksize * 2
103
+ else
104
+ chunksize + 100
105
+ end
106
+ new_chunksize = 1 if new_chunksize < 1
107
+ new_chunksize
108
+ end
109
+
110
+ def load_schema(database_url, schema_data)
111
+ Tempfile.open('taps') do |tmp|
112
+ File.open(tmp.path, 'w') { |f| f.write(schema_data) }
113
+ schema_bin(:load, database_url, tmp.path)
114
+ end
115
+ end
116
+
117
+ def load_indexes(database_url, index_data)
118
+ Tempfile.open('taps') do |tmp|
119
+ File.open(tmp.path, 'w') { |f| f.write(index_data) }
120
+ schema_bin(:load_indexes, database_url, tmp.path)
121
+ end
122
+ end
123
+
124
+ def schema_bin(*args)
125
+ bin_path = File.expand_path("#{File.dirname(__FILE__)}/../../bin/#{bin('schema')}")
126
+ `"#{bin_path}" #{args.map { |a| "'#{a}'" }.join(' ')}`
127
+ end
128
+
129
+ def primary_key(db, table)
130
+ table = table.to_sym.identifier unless table.kind_of?(Sequel::SQL::Identifier)
131
+ if db.respond_to?(:primary_key)
132
+ db.primary_key(table)
133
+ else
134
+ db.schema(table).select { |c| c[1][:primary_key] }.map { |c| c.first.to_sym }
135
+ end
136
+ end
137
+
138
+ def single_integer_primary_key(db, table)
139
+ table = table.to_sym.identifier unless table.kind_of?(Sequel::SQL::Identifier)
140
+ keys = db.schema(table).select { |c| c[1][:primary_key] and c[1][:type] == :integer }
141
+ not keys.nil? and keys.size == 1
142
+ end
143
+
144
+ def order_by(db, table)
145
+ pkey = primary_key(db, table)
146
+ if pkey
147
+ pkey.kind_of?(Array) ? pkey : [pkey.to_sym]
148
+ else
149
+ table = table.to_sym.identifier unless table.kind_of?(Sequel::SQL::Identifier)
150
+ db[table].columns
151
+ end
152
+ end
153
153
  end
154
154
  end
data/spec/base.rb CHANGED
@@ -7,18 +7,18 @@ require 'tempfile'
7
7
  $:.unshift File.dirname(__FILE__) + "/../lib"
8
8
 
9
9
  class Bacon::Context
10
- include Mocha::Standalone
11
- include Rack::Test::Methods
10
+ include Mocha::Standalone
11
+ include Rack::Test::Methods
12
12
 
13
- alias_method :old_it, :it
14
- def it(description)
15
- old_it(description) do
16
- mocha_setup
17
- yield
18
- mocha_verify
19
- mocha_teardown
20
- end
21
- end
13
+ alias_method :old_it, :it
14
+ def it(description)
15
+ old_it(description) do
16
+ mocha_setup
17
+ yield
18
+ mocha_verify
19
+ mocha_teardown
20
+ end
21
+ end
22
22
  end
23
23
 
24
24
  require 'taps/config'
data/spec/cli_spec.rb CHANGED
@@ -2,9 +2,9 @@ require File.dirname(__FILE__) + '/base'
2
2
  require 'taps/cli'
3
3
 
4
4
  describe Taps::Cli do
5
- it "translates a list of tables into a regex that can be used in table_filter" do
6
- @cli = Taps::Cli.new(["-t", "mytable1,logs", "sqlite://tmp.db", "http://x:y@localhost:5000"])
7
- opts = @cli.clientoptparse(:pull)
8
- opts[:table_filter].should == "(^mytable1$|^logs$)"
9
- end
5
+ it "translates a list of tables into a regex that can be used in table_filter" do
6
+ @cli = Taps::Cli.new(["-t", "mytable1,logs", "sqlite://tmp.db", "http://x:y@localhost:5000"])
7
+ opts = @cli.clientoptparse(:pull)
8
+ opts[:table_filter].should == "(^mytable1$|^logs$)"
9
+ end
10
10
  end
@@ -2,22 +2,22 @@ require File.dirname(__FILE__) + '/base'
2
2
  require 'taps/data_stream'
3
3
 
4
4
  describe Taps::DataStream do
5
- before do
6
- @db = mock('db')
7
- end
5
+ before do
6
+ @db = mock('db')
7
+ end
8
8
 
9
- it "increments the offset" do
10
- stream = Taps::DataStream.new(@db, :table_name => 'test_table', :chunksize => 100)
11
- stream.state[:offset].should == 0
12
- stream.increment(100)
13
- stream.state[:offset].should == 100
14
- end
9
+ it "increments the offset" do
10
+ stream = Taps::DataStream.new(@db, :table_name => 'test_table', :chunksize => 100)
11
+ stream.state[:offset].should == 0
12
+ stream.increment(100)
13
+ stream.state[:offset].should == 100
14
+ end
15
15
 
16
- it "marks the stream complete if no rows are fetched" do
17
- stream = Taps::DataStream.new(@db, :table_name => 'test_table', :chunksize => 100)
18
- stream.stubs(:fetch_rows).returns({})
19
- stream.complete?.should.be.false
20
- stream.fetch
21
- stream.complete?.should.be.true
22
- end
16
+ it "marks the stream complete if no rows are fetched" do
17
+ stream = Taps::DataStream.new(@db, :table_name => 'test_table', :chunksize => 100)
18
+ stream.stubs(:fetch_rows).returns({})
19
+ stream.complete?.should.be.false
20
+ stream.fetch
21
+ stream.complete?.should.be.true
22
+ end
23
23
  end
@@ -2,31 +2,31 @@ require File.dirname(__FILE__) + '/base'
2
2
  require 'taps/operation'
3
3
 
4
4
  describe Taps::Operation do
5
- before do
6
- @op = Taps::Operation.new('dummy://localhost', 'http://x:y@localhost:5000')
7
- end
5
+ before do
6
+ @op = Taps::Operation.new('dummy://localhost', 'http://x:y@localhost:5000')
7
+ end
8
8
 
9
- it "returns an array of tables that match the regex table_filter" do
10
- @op = Taps::Operation.new('dummy://localhost', 'http://x:y@localhost:5000', :table_filter => 'abc')
11
- @op.apply_table_filter(['abc', 'def']).should == ['abc']
12
- end
9
+ it "returns an array of tables that match the regex table_filter" do
10
+ @op = Taps::Operation.new('dummy://localhost', 'http://x:y@localhost:5000', :table_filter => 'abc')
11
+ @op.apply_table_filter(['abc', 'def']).should == ['abc']
12
+ end
13
13
 
14
- it "returns a hash of tables that match the regex table_filter" do
15
- @op = Taps::Operation.new('dummy://localhost', 'http://x:y@localhost:5000', :table_filter => 'abc')
16
- @op.apply_table_filter({ 'abc' => 1, 'def' => 2 }).should == { 'abc' => 1 }
17
- end
14
+ it "returns a hash of tables that match the regex table_filter" do
15
+ @op = Taps::Operation.new('dummy://localhost', 'http://x:y@localhost:5000', :table_filter => 'abc')
16
+ @op.apply_table_filter({ 'abc' => 1, 'def' => 2 }).should == { 'abc' => 1 }
17
+ end
18
18
 
19
- it "masks a url's password" do
20
- @op.safe_url("mysql://root:password@localhost/mydb").should == "mysql://root:[hidden]@localhost/mydb"
21
- end
19
+ it "masks a url's password" do
20
+ @op.safe_url("mysql://root:password@localhost/mydb").should == "mysql://root:[hidden]@localhost/mydb"
21
+ end
22
22
 
23
- it "returns http headers with compression enabled" do
24
- @op.http_headers.should == { :taps_version => Taps.compatible_version, :accept_encoding => "gzip, deflate" }
25
- end
23
+ it "returns http headers with compression enabled" do
24
+ @op.http_headers.should == { :taps_version => Taps.version, :accept_encoding => "gzip, deflate" }
25
+ end
26
26
 
27
- it "returns http headers with compression disabled" do
28
- @op.stubs(:compression_disabled?).returns(true)
29
- @op.http_headers.should == { :taps_version => Taps.compatible_version, :accept_encoding => "" }
30
- end
27
+ it "returns http headers with compression disabled" do
28
+ @op.stubs(:compression_disabled?).returns(true)
29
+ @op.http_headers.should == { :taps_version => Taps.version, :accept_encoding => "" }
30
+ end
31
31
 
32
32
  end
data/spec/server_spec.rb CHANGED
@@ -5,31 +5,31 @@ require 'taps/server'
5
5
  require 'pp'
6
6
 
7
7
  describe Taps::Server do
8
- def app
9
- Taps::Server.new
10
- end
11
-
12
- before do
13
- Taps::Config.login = 'taps'
14
- Taps::Config.password = 'tpass'
15
-
16
- @app = Taps::Server
17
- @auth_header = "Basic " + ["taps:tpass"].pack("m*")
18
- end
19
-
20
- it "asks for http basic authentication" do
21
- get '/'
22
- last_response.status.should == 401
23
- end
24
-
25
- it "verifies the client taps version" do
26
- get('/', { }, { 'HTTP_AUTHORIZATION' => @auth_header, 'HTTP_TAPS_VERSION' => Taps.compatible_version })
27
- last_response.status.should == 200
28
- end
29
-
30
- it "yells loudly if the client taps version doesn't match" do
31
- get('/', { }, { 'HTTP_AUTHORIZATION' => @auth_header, 'HTTP_TAPS_VERSION' => '0.0.1' })
32
- last_response.status.should == 417
33
- end
8
+ def app
9
+ Taps::Server.new
10
+ end
11
+
12
+ before do
13
+ Taps::Config.login = 'taps'
14
+ Taps::Config.password = 'tpass'
15
+
16
+ @app = Taps::Server
17
+ @auth_header = "Basic " + ["taps:tpass"].pack("m*")
18
+ end
19
+
20
+ it "asks for http basic authentication" do
21
+ get '/'
22
+ last_response.status.should == 401
23
+ end
24
+
25
+ it "verifies the client taps version" do
26
+ get('/', { }, { 'HTTP_AUTHORIZATION' => @auth_header, 'HTTP_TAPS_VERSION' => Taps.compatible_version })
27
+ last_response.status.should == 200
28
+ end
29
+
30
+ it "yells loudly if the client taps version doesn't match" do
31
+ get('/', { }, { 'HTTP_AUTHORIZATION' => @auth_header, 'HTTP_TAPS_VERSION' => '0.0.1' })
32
+ last_response.status.should == 417
33
+ end
34
34
  end
35
35