taps 0.2.26 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,23 @@
1
+ require File.dirname(__FILE__) + '/base'
2
+ require 'taps/data_stream'
3
+
4
+ describe Taps::DataStream do
5
+ before do
6
+ @db = mock('db')
7
+ end
8
+
9
+ it "increments the offset" do
10
+ stream = Taps::DataStream.new(@db, :table_name => 'test_table', :chunksize => 100)
11
+ stream.state[:offset].should == 0
12
+ stream.increment(100)
13
+ stream.state[:offset].should == 100
14
+ end
15
+
16
+ it "marks the stream complete if no rows are fetched" do
17
+ stream = Taps::DataStream.new(@db, :table_name => 'test_table', :chunksize => 100)
18
+ stream.stubs(:fetch_rows).returns({})
19
+ stream.complete?.should.be.false
20
+ stream.fetch
21
+ stream.complete?.should.be.true
22
+ end
23
+ end
@@ -0,0 +1,32 @@
1
+ require File.dirname(__FILE__) + '/base'
2
+ require 'taps/operation'
3
+
4
+ describe Taps::Operation do
5
+ before do
6
+ @op = Taps::Operation.new('dummy://localhost', 'http://x:y@localhost:5000')
7
+ end
8
+
9
+ it "returns an array of tables that match the regex table_filter" do
10
+ @op = Taps::Operation.new('dummy://localhost', 'http://x:y@localhost:5000', :table_filter => 'abc')
11
+ @op.apply_table_filter(['abc', 'def']).should == ['abc']
12
+ end
13
+
14
+ it "returns a hash of tables that match the regex table_filter" do
15
+ @op = Taps::Operation.new('dummy://localhost', 'http://x:y@localhost:5000', :table_filter => 'abc')
16
+ @op.apply_table_filter({ 'abc' => 1, 'def' => 2 }).should == { 'abc' => 1 }
17
+ end
18
+
19
+ it "masks a url's password" do
20
+ @op.safe_url("mysql://root:password@localhost/mydb").should == "mysql://root:[hidden]@localhost/mydb"
21
+ end
22
+
23
+ it "returns http headers with compression enabled" do
24
+ @op.http_headers.should == { :taps_version => Taps.compatible_version, :accept_encoding => "gzip, deflate" }
25
+ end
26
+
27
+ it "returns http headers with compression disabled" do
28
+ @op.stubs(:compression_disabled?).returns(true)
29
+ @op.http_headers.should == { :taps_version => Taps.compatible_version, :accept_encoding => "" }
30
+ end
31
+
32
+ end
data/spec/server_spec.rb CHANGED
@@ -1,6 +1,6 @@
1
1
  require File.dirname(__FILE__) + '/base'
2
2
 
3
- require File.dirname(__FILE__) + '/../lib/taps/server'
3
+ require 'taps/server'
4
4
 
5
5
  require 'pp'
6
6
 
data/spec/utils_spec.rb CHANGED
@@ -1,16 +1,7 @@
1
1
  require File.dirname(__FILE__) + '/base'
2
- require File.dirname(__FILE__) + '/../lib/taps/utils'
2
+ require 'taps/utils'
3
3
 
4
4
  describe Taps::Utils do
5
- it "gunzips a string" do
6
- @hello_world = "\037\213\b\000R\261\207I\000\003\313H\315\311\311W(\317/\312I\001\000\205\021J\r\v\000\000\000"
7
- Taps::Utils.gunzip(@hello_world).should == "hello world"
8
- end
9
-
10
- it "gzips and gunzips a string and returns the same string" do
11
- Taps::Utils.gunzip(Taps::Utils.gzip("hello world")).should == "hello world"
12
- end
13
-
14
5
  it "generates a checksum using crc32" do
15
6
  Taps::Utils.checksum("hello world").should == Zlib.crc32("hello world")
16
7
  end
@@ -18,7 +9,7 @@ describe Taps::Utils do
18
9
  it "formats a data hash into one hash that contains an array of headers and an array of array of data" do
19
10
  first_row = { :x => 1, :y => 1 }
20
11
  first_row.stubs(:keys).returns([:x, :y])
21
- Taps::Utils.format_data([ first_row, { :x => 2, :y => 2 } ], []).should == { :header => [ :x, :y ], :data => [ [1, 1], [2, 2] ] }
12
+ Taps::Utils.format_data([ first_row, { :x => 2, :y => 2 } ]).should == { :header => [ :x, :y ], :data => [ [1, 1], [2, 2] ] }
22
13
  end
23
14
 
24
15
  it "scales chunksize down slowly when the time delta of the block is just over a second" do
@@ -53,8 +44,7 @@ describe Taps::Utils do
53
44
  end
54
45
 
55
46
  it "returns a list of columns that are text fields if the database is mysql" do
56
- @db = mock("db")
57
- @db.class.stubs(:to_s).returns("Sequel::MySQL::Database")
47
+ @db = mock("db", :url => "mysql://localhost/mydb")
58
48
  @db.stubs(:schema).with(:mytable).returns([
59
49
  [:id, { :db_type => "int" }],
60
50
  [:mytext, { :db_type => "text" }]
metadata CHANGED
@@ -4,109 +4,101 @@ version: !ruby/object:Gem::Version
4
4
  prerelease: false
5
5
  segments:
6
6
  - 0
7
- - 2
8
- - 26
9
- version: 0.2.26
7
+ - 3
8
+ - 0
9
+ version: 0.3.0
10
10
  platform: ruby
11
11
  authors:
12
12
  - Ricardo Chimal, Jr.
13
- - Adam Wiggins
14
13
  autorequire:
15
14
  bindir: bin
16
15
  cert_chain: []
17
16
 
18
- date: 2010-02-25 00:00:00 -08:00
17
+ date: 2010-04-14 00:00:00 -07:00
19
18
  default_executable:
20
19
  dependencies:
21
20
  - !ruby/object:Gem::Dependency
22
- name: sinatra
21
+ name: json_pure
23
22
  prerelease: false
24
23
  requirement: &id001 !ruby/object:Gem::Requirement
25
24
  requirements:
26
- - - "="
25
+ - - ~>
27
26
  - !ruby/object:Gem::Version
28
27
  segments:
29
- - 0
30
- - 9
28
+ - 1
31
29
  - 2
32
- version: 0.9.2
30
+ - 0
31
+ version: 1.2.0
33
32
  type: :runtime
34
33
  version_requirements: *id001
35
34
  - !ruby/object:Gem::Dependency
36
- name: activerecord
35
+ name: sinatra
37
36
  prerelease: false
38
37
  requirement: &id002 !ruby/object:Gem::Requirement
39
38
  requirements:
40
- - - "="
39
+ - - ~>
41
40
  - !ruby/object:Gem::Version
42
41
  segments:
43
- - 2
44
- - 2
45
- - 2
46
- version: 2.2.2
42
+ - 1
43
+ - 0
44
+ - 0
45
+ version: 1.0.0
47
46
  type: :runtime
48
47
  version_requirements: *id002
49
48
  - !ruby/object:Gem::Dependency
50
- name: thor
49
+ name: rest-client
51
50
  prerelease: false
52
51
  requirement: &id003 !ruby/object:Gem::Requirement
53
52
  requirements:
54
- - - "="
53
+ - - ~>
55
54
  - !ruby/object:Gem::Version
56
55
  segments:
56
+ - 1
57
+ - 4
57
58
  - 0
58
- - 9
59
- - 9
60
- version: 0.9.9
59
+ version: 1.4.0
61
60
  type: :runtime
62
61
  version_requirements: *id003
63
62
  - !ruby/object:Gem::Dependency
64
- name: rest-client
63
+ name: sequel
65
64
  prerelease: false
66
65
  requirement: &id004 !ruby/object:Gem::Requirement
67
66
  requirements:
68
67
  - - ~>
69
68
  - !ruby/object:Gem::Version
70
69
  segments:
71
- - 1
72
70
  - 3
71
+ - 10
73
72
  - 0
74
- version: 1.3.0
73
+ version: 3.10.0
75
74
  type: :runtime
76
75
  version_requirements: *id004
77
76
  - !ruby/object:Gem::Dependency
78
- name: sequel
77
+ name: sqlite3-ruby
79
78
  prerelease: false
80
79
  requirement: &id005 !ruby/object:Gem::Requirement
81
80
  requirements:
82
- - - ">="
83
- - !ruby/object:Gem::Version
84
- segments:
85
- - 3
86
- - 0
87
- - 0
88
- version: 3.0.0
89
- - - <
81
+ - - ~>
90
82
  - !ruby/object:Gem::Version
91
83
  segments:
92
- - 3
93
84
  - 1
85
+ - 2
94
86
  - 0
95
- version: 3.1.0
87
+ version: 1.2.0
96
88
  type: :runtime
97
89
  version_requirements: *id005
98
90
  - !ruby/object:Gem::Dependency
99
- name: sqlite3-ruby
91
+ name: rack
100
92
  prerelease: false
101
93
  requirement: &id006 !ruby/object:Gem::Requirement
102
94
  requirements:
103
- - - ~>
95
+ - - ">="
104
96
  - !ruby/object:Gem::Version
105
97
  segments:
106
98
  - 1
107
- - 2
108
99
  - 0
109
- version: 1.2.0
100
+ - 1
101
+ version: 1.0.1
110
102
  type: :runtime
111
103
  version_requirements: *id006
112
104
  description: A simple database agnostic import/export app to transfer data to/from a remote database.
@@ -127,22 +119,21 @@ files:
127
119
  - bin/schema
128
120
  - bin/schema.cmd
129
121
  - bin/taps
130
- - lib/taps/adapter_hacks.rb
131
- - lib/taps/adapter_hacks/invalid_binary_limit.rb
132
- - lib/taps/adapter_hacks/invalid_text_limit.rb
133
- - lib/taps/adapter_hacks/mysql_invalid_primary_key.rb
134
- - lib/taps/adapter_hacks/non_rails_schema_dump.rb
135
122
  - lib/taps/cli.rb
136
- - lib/taps/client_session.rb
137
123
  - lib/taps/config.rb
124
+ - lib/taps/data_stream.rb
138
125
  - lib/taps/db_session.rb
126
+ - lib/taps/log.rb
127
+ - lib/taps/monkey.rb
128
+ - lib/taps/multipart.rb
129
+ - lib/taps/operation.rb
139
130
  - lib/taps/progress_bar.rb
140
131
  - lib/taps/schema.rb
141
132
  - lib/taps/server.rb
142
133
  - lib/taps/utils.rb
143
134
  - spec/base.rb
144
- - spec/client_session_spec.rb
145
- - spec/schema_spec.rb
135
+ - spec/data_stream_spec.rb
136
+ - spec/operation_spec.rb
146
137
  - spec/server_spec.rb
147
138
  - spec/utils_spec.rb
148
139
  has_rdoc: true
@@ -176,8 +167,8 @@ signing_key:
176
167
  specification_version: 3
177
168
  summary: simple database import/export app
178
169
  test_files:
170
+ - spec/utils_spec.rb
171
+ - spec/operation_spec.rb
172
+ - spec/data_stream_spec.rb
179
173
  - spec/base.rb
180
- - spec/client_session_spec.rb
181
- - spec/schema_spec.rb
182
174
  - spec/server_spec.rb
183
- - spec/utils_spec.rb
@@ -1,21 +0,0 @@
1
- module Taps
2
- module AdapterHacks
3
- extend self
4
-
5
- LIST = {
6
- :all => ['non_rails_schema_dump'],
7
- :mysql => ['invalid_text_limit', 'mysql_invalid_primary_key'],
8
- :postgresql => ['invalid_text_limit', 'invalid_binary_limit']
9
- }
10
-
11
- def load(adapter)
12
- LIST[:all].each do |r|
13
- require File.dirname(__FILE__) + "/adapter_hacks/#{r}"
14
- end
15
-
16
- (LIST[adapter.to_sym] || []).each do |r|
17
- require File.dirname(__FILE__) + "/adapter_hacks/#{r}"
18
- end
19
- end
20
- end
21
- end
@@ -1,13 +0,0 @@
1
- module ActiveRecord
2
- module ConnectionAdapters
3
- class TableDefinition
4
- alias_method :original_binary, :binary
5
- def binary(*args)
6
- options = args.extract_options!
7
- options.delete(:limit)
8
- column_names = args
9
- column_names.each { |name| column(name, 'binary', options) }
10
- end
11
- end
12
- end
13
- end
@@ -1,13 +0,0 @@
1
- module ActiveRecord
2
- module ConnectionAdapters
3
- class TableDefinition
4
- alias_method :original_text, :text
5
- def text(*args)
6
- options = args.extract_options!
7
- options.delete(:limit)
8
- column_names = args
9
- column_names.each { |name| column(name, 'text', options) }
10
- end
11
- end
12
- end
13
- end
@@ -1,17 +0,0 @@
1
- module ActiveRecord
2
- module ConnectionAdapters
3
- class MysqlAdapter < AbstractAdapter
4
- alias_method :orig_pk_and_sequence_for, :pk_and_sequence_for
5
- # mysql accepts varchar as a primary key but most others do not.
6
- # only say that a field is a primary key if mysql says so
7
- # and the field is a kind of integer
8
- def pk_and_sequence_for(table)
9
- keys = []
10
- execute("describe #{quote_table_name(table)}").each_hash do |h|
11
- keys << h["Field"] if h["Key"] == "PRI" and !(h["Type"] =~ /int/).nil?
12
- end
13
- keys.length == 1 ? [keys.first, nil] : nil
14
- end
15
- end
16
- end
17
- end
@@ -1,15 +0,0 @@
1
- module ActiveRecord
2
- class SchemaDumper
3
- private
4
-
5
- def header(stream)
6
- stream.puts "ActiveRecord::Schema.define do"
7
- end
8
-
9
- def tables(stream)
10
- @connection.tables.sort.each do |tbl|
11
- table(tbl, stream)
12
- end
13
- end
14
- end
15
- end
@@ -1,304 +0,0 @@
1
- require 'rest_client'
2
- require 'sequel'
3
- require 'zlib'
4
-
5
- require File.dirname(__FILE__) + '/progress_bar'
6
- require File.dirname(__FILE__) + '/config'
7
- require File.dirname(__FILE__) + '/utils'
8
-
9
- module Taps
10
- class ClientSession
11
- attr_reader :database_url, :remote_url, :default_chunksize
12
-
13
- def initialize(database_url, remote_url, default_chunksize)
14
- @database_url = database_url
15
- @remote_url = remote_url
16
- @default_chunksize = default_chunksize
17
- end
18
-
19
- def self.start(database_url, remote_url, default_chunksize, &block)
20
- s = new(database_url, remote_url, default_chunksize)
21
- yield s
22
- s.close_session
23
- end
24
-
25
- def self.quickstart(&block)
26
- start(Taps::Config.database_url, Taps::Config.remote_url, Taps::Config.chunksize) do |s|
27
- yield s
28
- end
29
- end
30
-
31
- def db
32
- @db ||= Sequel.connect(database_url)
33
- end
34
-
35
- def server
36
- @server ||= RestClient::Resource.new(remote_url)
37
- end
38
-
39
- def session_resource
40
- @session_resource ||= open_session
41
- end
42
-
43
- def open_session
44
- uri = server['sessions'].post('', http_headers)
45
- server[uri]
46
- end
47
-
48
- def set_session(uri)
49
- @session_resource = server[uri]
50
- end
51
-
52
- def close_session
53
- @session_resource.delete(http_headers) if @session_resource
54
- end
55
-
56
- def safe_url(url)
57
- url.sub(/\/\/(.+?)?:(.*?)@/, '//\1:[hidden]@')
58
- end
59
-
60
- def safe_remote_url
61
- safe_url(remote_url)
62
- end
63
-
64
- def safe_database_url
65
- safe_url(database_url)
66
- end
67
-
68
- def http_headers(extra = {})
69
- { :taps_version => Taps.compatible_version }.merge(extra)
70
- end
71
-
72
- def cmd_send
73
- begin
74
- verify_server
75
- cmd_send_schema
76
- cmd_send_data
77
- cmd_send_indexes
78
- cmd_send_reset_sequences
79
- rescue RestClient::Exception => e
80
- if e.respond_to?(:response)
81
- puts "!!! Caught Server Exception"
82
- puts "#{e.response}"
83
- exit(1)
84
- else
85
- raise
86
- end
87
- end
88
- end
89
-
90
- def cmd_send_indexes
91
- puts "Sending indexes"
92
-
93
- index_data = Taps::Utils.schema_bin(:indexes, database_url)
94
- session_resource['indexes'].post(index_data, http_headers)
95
- end
96
-
97
- def cmd_send_schema
98
- puts "Sending schema"
99
-
100
- schema_data = Taps::Utils.schema_bin(:dump, database_url)
101
- session_resource['schema'].post(schema_data, http_headers)
102
- end
103
-
104
- def cmd_send_reset_sequences
105
- puts "Resetting sequences"
106
-
107
- session_resource["reset_sequences"].post('', http_headers)
108
- end
109
-
110
- def cmd_send_data
111
- puts "Sending data"
112
-
113
- tables_with_counts, record_count = fetch_tables_info
114
-
115
- puts "#{tables_with_counts.size} tables, #{format_number(record_count)} records"
116
-
117
- tables_with_counts.each do |table_name, count|
118
- table = db[table_name]
119
- order = Taps::Utils.order_by(db, table_name)
120
- chunksize = self.default_chunksize
121
- string_columns = Taps::Utils.incorrect_blobs(db, table_name)
122
-
123
- progress = ProgressBar.new(table_name.to_s, count)
124
-
125
- offset = 0
126
- loop do
127
- row_size = 0
128
- chunksize = Taps::Utils.calculate_chunksize(chunksize) do |c|
129
- time_skip_start = Time.now
130
- rows = Taps::Utils.format_data(table.order(*order).limit(c, offset).all, string_columns)
131
- break if rows == { }
132
-
133
- row_size = rows[:data].size
134
- gzip_data = Taps::Utils.gzip(Marshal.dump(rows))
135
- time_skip = Time.now - time_skip_start
136
-
137
- begin
138
- session_resource["tables/#{table_name}"].post(gzip_data, http_headers({
139
- :content_type => 'application/octet-stream',
140
- :taps_checksum => Taps::Utils.checksum(gzip_data).to_s}))
141
- rescue RestClient::RequestFailed => e
142
- # retry the same data, it got corrupted somehow.
143
- if e.http_code == 412
144
- next
145
- end
146
- raise
147
- end
148
- time_skip
149
- end
150
-
151
- progress.inc(row_size)
152
- offset += row_size
153
-
154
- break if row_size == 0
155
- end
156
-
157
- progress.finish
158
- end
159
- end
160
-
161
- def fetch_tables_info
162
- tables_with_counts = {}
163
- record_count = db.tables.inject(0) do |record_count, table|
164
- tables_with_counts[table] = db[table].count
165
- record_count += tables_with_counts[table]
166
- end
167
-
168
- [ tables_with_counts, record_count ]
169
- end
170
-
171
- def cmd_receive
172
- begin
173
- verify_server
174
- cmd_receive_schema
175
- cmd_receive_data
176
- cmd_receive_indexes
177
- cmd_reset_sequences
178
- rescue RestClient::Exception => e
179
- if e.respond_to?(:response)
180
- puts "!!! Caught Server Exception"
181
- puts "#{e.response.body}"
182
- exit(1)
183
- else
184
- raise
185
- end
186
- end
187
- end
188
-
189
- def cmd_receive_data
190
- puts "Receiving data"
191
-
192
- tables_with_counts, record_count = fetch_remote_tables_info
193
-
194
- puts "#{tables_with_counts.size} tables, #{format_number(record_count)} records"
195
-
196
- tables_with_counts.each do |table_name, count|
197
- table = db[table_name.to_sym]
198
- chunksize = default_chunksize
199
-
200
- progress = ProgressBar.new(table_name.to_s, count)
201
-
202
- offset = 0
203
- loop do
204
- begin
205
- chunksize, rows = fetch_table_rows(table_name, chunksize, offset)
206
- rescue CorruptedData
207
- next
208
- end
209
- break if rows == { }
210
-
211
- table.import(rows[:header], rows[:data])
212
-
213
- progress.inc(rows[:data].size)
214
- offset += rows[:data].size
215
- end
216
-
217
- progress.finish
218
- end
219
- end
220
-
221
- class CorruptedData < Exception; end
222
-
223
- def fetch_table_rows(table_name, chunksize, offset)
224
- response = nil
225
- chunksize = Taps::Utils.calculate_chunksize(chunksize) do |c|
226
- response = session_resource["tables/#{table_name}/#{c}?offset=#{offset}"].get(http_headers)
227
- end
228
- raise CorruptedData unless Taps::Utils.valid_data?(response.to_s, response.headers[:taps_checksum])
229
-
230
- begin
231
- rows = Marshal.load(Taps::Utils.gunzip(response.to_s))
232
- rescue Object => e
233
- puts "Error encountered loading data, wrote the data chunk to dump.#{Process.pid}.gz"
234
- File.open("dump.#{Process.pid}.gz", "w") { |f| f.write(response.to_s) }
235
- raise
236
- end
237
- [chunksize, rows]
238
- end
239
-
240
- def fetch_remote_tables_info
241
- retries = 0
242
- max_retries = 1
243
- begin
244
- tables_with_counts = Marshal.load(session_resource['tables'].get(http_headers))
245
- record_count = tables_with_counts.values.inject(0) { |a,c| a += c }
246
- rescue RestClient::Exception
247
- retries += 1
248
- retry if retries <= max_retries
249
- puts "Unable to fetch tables information from #{remote_url}. Please check the server log."
250
- exit(1)
251
- end
252
-
253
- [ tables_with_counts, record_count ]
254
- end
255
-
256
- def cmd_receive_schema
257
- puts "Receiving schema"
258
-
259
- schema_data = session_resource['schema'].get(http_headers)
260
- output = Taps::Utils.load_schema(database_url, schema_data)
261
- puts output if output
262
- end
263
-
264
- def cmd_receive_indexes
265
- puts "Receiving indexes"
266
-
267
- index_data = session_resource['indexes'].get(http_headers)
268
-
269
- output = Taps::Utils.load_indexes(database_url, index_data)
270
- puts output if output
271
- end
272
-
273
- def cmd_reset_sequences
274
- puts "Resetting sequences"
275
-
276
- output = Taps::Utils.schema_bin(:reset_db_sequences, database_url)
277
- puts output if output
278
- end
279
-
280
- def format_number(num)
281
- num.to_s.gsub(/(\d)(?=(\d\d\d)+(?!\d))/, "\\1,")
282
- end
283
-
284
- def verify_server
285
- begin
286
- server['/'].get(http_headers)
287
- rescue RestClient::RequestFailed => e
288
- if e.http_code == 417
289
- puts "#{safe_remote_url} is running a different minor version of taps."
290
- puts "#{e.response.body}"
291
- exit(1)
292
- else
293
- raise
294
- end
295
- rescue RestClient::Unauthorized
296
- puts "Bad credentials given for #{safe_remote_url}"
297
- exit(1)
298
- rescue Errno::ECONNREFUSED
299
- puts "Can't connect to #{safe_remote_url}. Please check that it's running"
300
- exit(1)
301
- end
302
- end
303
- end
304
- end