taps 0.2.26 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/README.rdoc +8 -2
- data/Rakefile +20 -15
- data/VERSION.yml +3 -3
- data/bin/schema +13 -5
- data/bin/taps +3 -10
- data/lib/taps/cli.rb +157 -37
- data/lib/taps/config.rb +14 -3
- data/lib/taps/data_stream.rb +299 -0
- data/lib/taps/db_session.rb +4 -46
- data/lib/taps/log.rb +15 -0
- data/lib/taps/monkey.rb +21 -0
- data/lib/taps/multipart.rb +73 -0
- data/lib/taps/operation.rb +537 -0
- data/lib/taps/schema.rb +48 -66
- data/lib/taps/server.rb +76 -50
- data/lib/taps/utils.rb +20 -19
- data/spec/base.rb +3 -1
- data/spec/data_stream_spec.rb +23 -0
- data/spec/operation_spec.rb +32 -0
- data/spec/server_spec.rb +1 -1
- data/spec/utils_spec.rb +3 -13
- metadata +41 -50
- data/lib/taps/adapter_hacks.rb +0 -21
- data/lib/taps/adapter_hacks/invalid_binary_limit.rb +0 -13
- data/lib/taps/adapter_hacks/invalid_text_limit.rb +0 -13
- data/lib/taps/adapter_hacks/mysql_invalid_primary_key.rb +0 -17
- data/lib/taps/adapter_hacks/non_rails_schema_dump.rb +0 -15
- data/lib/taps/client_session.rb +0 -304
- data/spec/client_session_spec.rb +0 -88
- data/spec/schema_spec.rb +0 -45
@@ -0,0 +1,23 @@
|
|
1
|
+
require File.dirname(__FILE__) + '/base'
|
2
|
+
require 'taps/data_stream'
|
3
|
+
|
4
|
+
describe Taps::DataStream do
|
5
|
+
before do
|
6
|
+
@db = mock('db')
|
7
|
+
end
|
8
|
+
|
9
|
+
it "increments the offset" do
|
10
|
+
stream = Taps::DataStream.new(@db, :table_name => 'test_table', :chunksize => 100)
|
11
|
+
stream.state[:offset].should == 0
|
12
|
+
stream.increment(100)
|
13
|
+
stream.state[:offset].should == 100
|
14
|
+
end
|
15
|
+
|
16
|
+
it "marks the stream complete if no rows are fetched" do
|
17
|
+
stream = Taps::DataStream.new(@db, :table_name => 'test_table', :chunksize => 100)
|
18
|
+
stream.stubs(:fetch_rows).returns({})
|
19
|
+
stream.complete?.should.be.false
|
20
|
+
stream.fetch
|
21
|
+
stream.complete?.should.be.true
|
22
|
+
end
|
23
|
+
end
|
@@ -0,0 +1,32 @@
|
|
1
|
+
require File.dirname(__FILE__) + '/base'
|
2
|
+
require 'taps/operation'
|
3
|
+
|
4
|
+
describe Taps::Operation do
|
5
|
+
before do
|
6
|
+
@op = Taps::Operation.new('dummy://localhost', 'http://x:y@localhost:5000')
|
7
|
+
end
|
8
|
+
|
9
|
+
it "returns an array of tables that match the regex table_filter" do
|
10
|
+
@op = Taps::Operation.new('dummy://localhost', 'http://x:y@localhost:5000', :table_filter => 'abc')
|
11
|
+
@op.apply_table_filter(['abc', 'def']).should == ['abc']
|
12
|
+
end
|
13
|
+
|
14
|
+
it "returns a hash of tables that match the regex table_filter" do
|
15
|
+
@op = Taps::Operation.new('dummy://localhost', 'http://x:y@localhost:5000', :table_filter => 'abc')
|
16
|
+
@op.apply_table_filter({ 'abc' => 1, 'def' => 2 }).should == { 'abc' => 1 }
|
17
|
+
end
|
18
|
+
|
19
|
+
it "masks a url's password" do
|
20
|
+
@op.safe_url("mysql://root:password@localhost/mydb").should == "mysql://root:[hidden]@localhost/mydb"
|
21
|
+
end
|
22
|
+
|
23
|
+
it "returns http headers with compression enabled" do
|
24
|
+
@op.http_headers.should == { :taps_version => Taps.compatible_version, :accept_encoding => "gzip, deflate" }
|
25
|
+
end
|
26
|
+
|
27
|
+
it "returns http headers with compression disabled" do
|
28
|
+
@op.stubs(:compression_disabled?).returns(true)
|
29
|
+
@op.http_headers.should == { :taps_version => Taps.compatible_version, :accept_encoding => "" }
|
30
|
+
end
|
31
|
+
|
32
|
+
end
|
data/spec/server_spec.rb
CHANGED
data/spec/utils_spec.rb
CHANGED
@@ -1,16 +1,7 @@
|
|
1
1
|
require File.dirname(__FILE__) + '/base'
|
2
|
-
require
|
2
|
+
require 'taps/utils'
|
3
3
|
|
4
4
|
describe Taps::Utils do
|
5
|
-
it "gunzips a string" do
|
6
|
-
@hello_world = "\037\213\b\000R\261\207I\000\003\313H\315\311\311W(\317/\312I\001\000\205\021J\r\v\000\000\000"
|
7
|
-
Taps::Utils.gunzip(@hello_world).should == "hello world"
|
8
|
-
end
|
9
|
-
|
10
|
-
it "gzips and gunzips a string and returns the same string" do
|
11
|
-
Taps::Utils.gunzip(Taps::Utils.gzip("hello world")).should == "hello world"
|
12
|
-
end
|
13
|
-
|
14
5
|
it "generates a checksum using crc32" do
|
15
6
|
Taps::Utils.checksum("hello world").should == Zlib.crc32("hello world")
|
16
7
|
end
|
@@ -18,7 +9,7 @@ describe Taps::Utils do
|
|
18
9
|
it "formats a data hash into one hash that contains an array of headers and an array of array of data" do
|
19
10
|
first_row = { :x => 1, :y => 1 }
|
20
11
|
first_row.stubs(:keys).returns([:x, :y])
|
21
|
-
Taps::Utils.format_data([ first_row, { :x => 2, :y => 2 } ]
|
12
|
+
Taps::Utils.format_data([ first_row, { :x => 2, :y => 2 } ]).should == { :header => [ :x, :y ], :data => [ [1, 1], [2, 2] ] }
|
22
13
|
end
|
23
14
|
|
24
15
|
it "scales chunksize down slowly when the time delta of the block is just over a second" do
|
@@ -53,8 +44,7 @@ describe Taps::Utils do
|
|
53
44
|
end
|
54
45
|
|
55
46
|
it "returns a list of columns that are text fields if the database is mysql" do
|
56
|
-
@db = mock("db")
|
57
|
-
@db.class.stubs(:to_s).returns("Sequel::MySQL::Database")
|
47
|
+
@db = mock("db", :url => "mysql://localhost/mydb")
|
58
48
|
@db.stubs(:schema).with(:mytable).returns([
|
59
49
|
[:id, { :db_type => "int" }],
|
60
50
|
[:mytext, { :db_type => "text" }]
|
metadata
CHANGED
@@ -4,109 +4,101 @@ version: !ruby/object:Gem::Version
|
|
4
4
|
prerelease: false
|
5
5
|
segments:
|
6
6
|
- 0
|
7
|
-
-
|
8
|
-
-
|
9
|
-
version: 0.
|
7
|
+
- 3
|
8
|
+
- 0
|
9
|
+
version: 0.3.0
|
10
10
|
platform: ruby
|
11
11
|
authors:
|
12
12
|
- Ricardo Chimal, Jr.
|
13
|
-
- Adam Wiggins
|
14
13
|
autorequire:
|
15
14
|
bindir: bin
|
16
15
|
cert_chain: []
|
17
16
|
|
18
|
-
date: 2010-
|
17
|
+
date: 2010-04-14 00:00:00 -07:00
|
19
18
|
default_executable:
|
20
19
|
dependencies:
|
21
20
|
- !ruby/object:Gem::Dependency
|
22
|
-
name:
|
21
|
+
name: json_pure
|
23
22
|
prerelease: false
|
24
23
|
requirement: &id001 !ruby/object:Gem::Requirement
|
25
24
|
requirements:
|
26
|
-
- -
|
25
|
+
- - ~>
|
27
26
|
- !ruby/object:Gem::Version
|
28
27
|
segments:
|
29
|
-
-
|
30
|
-
- 9
|
28
|
+
- 1
|
31
29
|
- 2
|
32
|
-
|
30
|
+
- 0
|
31
|
+
version: 1.2.0
|
33
32
|
type: :runtime
|
34
33
|
version_requirements: *id001
|
35
34
|
- !ruby/object:Gem::Dependency
|
36
|
-
name:
|
35
|
+
name: sinatra
|
37
36
|
prerelease: false
|
38
37
|
requirement: &id002 !ruby/object:Gem::Requirement
|
39
38
|
requirements:
|
40
|
-
- -
|
39
|
+
- - ~>
|
41
40
|
- !ruby/object:Gem::Version
|
42
41
|
segments:
|
43
|
-
-
|
44
|
-
-
|
45
|
-
-
|
46
|
-
version:
|
42
|
+
- 1
|
43
|
+
- 0
|
44
|
+
- 0
|
45
|
+
version: 1.0.0
|
47
46
|
type: :runtime
|
48
47
|
version_requirements: *id002
|
49
48
|
- !ruby/object:Gem::Dependency
|
50
|
-
name:
|
49
|
+
name: rest-client
|
51
50
|
prerelease: false
|
52
51
|
requirement: &id003 !ruby/object:Gem::Requirement
|
53
52
|
requirements:
|
54
|
-
- -
|
53
|
+
- - ~>
|
55
54
|
- !ruby/object:Gem::Version
|
56
55
|
segments:
|
56
|
+
- 1
|
57
|
+
- 4
|
57
58
|
- 0
|
58
|
-
|
59
|
-
- 9
|
60
|
-
version: 0.9.9
|
59
|
+
version: 1.4.0
|
61
60
|
type: :runtime
|
62
61
|
version_requirements: *id003
|
63
62
|
- !ruby/object:Gem::Dependency
|
64
|
-
name:
|
63
|
+
name: sequel
|
65
64
|
prerelease: false
|
66
65
|
requirement: &id004 !ruby/object:Gem::Requirement
|
67
66
|
requirements:
|
68
67
|
- - ~>
|
69
68
|
- !ruby/object:Gem::Version
|
70
69
|
segments:
|
71
|
-
- 1
|
72
70
|
- 3
|
71
|
+
- 10
|
73
72
|
- 0
|
74
|
-
version:
|
73
|
+
version: 3.10.0
|
75
74
|
type: :runtime
|
76
75
|
version_requirements: *id004
|
77
76
|
- !ruby/object:Gem::Dependency
|
78
|
-
name:
|
77
|
+
name: sqlite3-ruby
|
79
78
|
prerelease: false
|
80
79
|
requirement: &id005 !ruby/object:Gem::Requirement
|
81
80
|
requirements:
|
82
|
-
- -
|
83
|
-
- !ruby/object:Gem::Version
|
84
|
-
segments:
|
85
|
-
- 3
|
86
|
-
- 0
|
87
|
-
- 0
|
88
|
-
version: 3.0.0
|
89
|
-
- - <
|
81
|
+
- - ~>
|
90
82
|
- !ruby/object:Gem::Version
|
91
83
|
segments:
|
92
|
-
- 3
|
93
84
|
- 1
|
85
|
+
- 2
|
94
86
|
- 0
|
95
|
-
version:
|
87
|
+
version: 1.2.0
|
96
88
|
type: :runtime
|
97
89
|
version_requirements: *id005
|
98
90
|
- !ruby/object:Gem::Dependency
|
99
|
-
name:
|
91
|
+
name: rack
|
100
92
|
prerelease: false
|
101
93
|
requirement: &id006 !ruby/object:Gem::Requirement
|
102
94
|
requirements:
|
103
|
-
- -
|
95
|
+
- - ">="
|
104
96
|
- !ruby/object:Gem::Version
|
105
97
|
segments:
|
106
98
|
- 1
|
107
|
-
- 2
|
108
99
|
- 0
|
109
|
-
|
100
|
+
- 1
|
101
|
+
version: 1.0.1
|
110
102
|
type: :runtime
|
111
103
|
version_requirements: *id006
|
112
104
|
description: A simple database agnostic import/export app to transfer data to/from a remote database.
|
@@ -127,22 +119,21 @@ files:
|
|
127
119
|
- bin/schema
|
128
120
|
- bin/schema.cmd
|
129
121
|
- bin/taps
|
130
|
-
- lib/taps/adapter_hacks.rb
|
131
|
-
- lib/taps/adapter_hacks/invalid_binary_limit.rb
|
132
|
-
- lib/taps/adapter_hacks/invalid_text_limit.rb
|
133
|
-
- lib/taps/adapter_hacks/mysql_invalid_primary_key.rb
|
134
|
-
- lib/taps/adapter_hacks/non_rails_schema_dump.rb
|
135
122
|
- lib/taps/cli.rb
|
136
|
-
- lib/taps/client_session.rb
|
137
123
|
- lib/taps/config.rb
|
124
|
+
- lib/taps/data_stream.rb
|
138
125
|
- lib/taps/db_session.rb
|
126
|
+
- lib/taps/log.rb
|
127
|
+
- lib/taps/monkey.rb
|
128
|
+
- lib/taps/multipart.rb
|
129
|
+
- lib/taps/operation.rb
|
139
130
|
- lib/taps/progress_bar.rb
|
140
131
|
- lib/taps/schema.rb
|
141
132
|
- lib/taps/server.rb
|
142
133
|
- lib/taps/utils.rb
|
143
134
|
- spec/base.rb
|
144
|
-
- spec/
|
145
|
-
- spec/
|
135
|
+
- spec/data_stream_spec.rb
|
136
|
+
- spec/operation_spec.rb
|
146
137
|
- spec/server_spec.rb
|
147
138
|
- spec/utils_spec.rb
|
148
139
|
has_rdoc: true
|
@@ -176,8 +167,8 @@ signing_key:
|
|
176
167
|
specification_version: 3
|
177
168
|
summary: simple database import/export app
|
178
169
|
test_files:
|
170
|
+
- spec/utils_spec.rb
|
171
|
+
- spec/operation_spec.rb
|
172
|
+
- spec/data_stream_spec.rb
|
179
173
|
- spec/base.rb
|
180
|
-
- spec/client_session_spec.rb
|
181
|
-
- spec/schema_spec.rb
|
182
174
|
- spec/server_spec.rb
|
183
|
-
- spec/utils_spec.rb
|
data/lib/taps/adapter_hacks.rb
DELETED
@@ -1,21 +0,0 @@
|
|
1
|
-
module Taps
|
2
|
-
module AdapterHacks
|
3
|
-
extend self
|
4
|
-
|
5
|
-
LIST = {
|
6
|
-
:all => ['non_rails_schema_dump'],
|
7
|
-
:mysql => ['invalid_text_limit', 'mysql_invalid_primary_key'],
|
8
|
-
:postgresql => ['invalid_text_limit', 'invalid_binary_limit']
|
9
|
-
}
|
10
|
-
|
11
|
-
def load(adapter)
|
12
|
-
LIST[:all].each do |r|
|
13
|
-
require File.dirname(__FILE__) + "/adapter_hacks/#{r}"
|
14
|
-
end
|
15
|
-
|
16
|
-
(LIST[adapter.to_sym] || []).each do |r|
|
17
|
-
require File.dirname(__FILE__) + "/adapter_hacks/#{r}"
|
18
|
-
end
|
19
|
-
end
|
20
|
-
end
|
21
|
-
end
|
@@ -1,13 +0,0 @@
|
|
1
|
-
module ActiveRecord
|
2
|
-
module ConnectionAdapters
|
3
|
-
class TableDefinition
|
4
|
-
alias_method :original_binary, :binary
|
5
|
-
def binary(*args)
|
6
|
-
options = args.extract_options!
|
7
|
-
options.delete(:limit)
|
8
|
-
column_names = args
|
9
|
-
column_names.each { |name| column(name, 'binary', options) }
|
10
|
-
end
|
11
|
-
end
|
12
|
-
end
|
13
|
-
end
|
@@ -1,13 +0,0 @@
|
|
1
|
-
module ActiveRecord
|
2
|
-
module ConnectionAdapters
|
3
|
-
class TableDefinition
|
4
|
-
alias_method :original_text, :text
|
5
|
-
def text(*args)
|
6
|
-
options = args.extract_options!
|
7
|
-
options.delete(:limit)
|
8
|
-
column_names = args
|
9
|
-
column_names.each { |name| column(name, 'text', options) }
|
10
|
-
end
|
11
|
-
end
|
12
|
-
end
|
13
|
-
end
|
@@ -1,17 +0,0 @@
|
|
1
|
-
module ActiveRecord
|
2
|
-
module ConnectionAdapters
|
3
|
-
class MysqlAdapter < AbstractAdapter
|
4
|
-
alias_method :orig_pk_and_sequence_for, :pk_and_sequence_for
|
5
|
-
# mysql accepts varchar as a primary key but most others do not.
|
6
|
-
# only say that a field is a primary key if mysql says so
|
7
|
-
# and the field is a kind of integer
|
8
|
-
def pk_and_sequence_for(table)
|
9
|
-
keys = []
|
10
|
-
execute("describe #{quote_table_name(table)}").each_hash do |h|
|
11
|
-
keys << h["Field"] if h["Key"] == "PRI" and !(h["Type"] =~ /int/).nil?
|
12
|
-
end
|
13
|
-
keys.length == 1 ? [keys.first, nil] : nil
|
14
|
-
end
|
15
|
-
end
|
16
|
-
end
|
17
|
-
end
|
data/lib/taps/client_session.rb
DELETED
@@ -1,304 +0,0 @@
|
|
1
|
-
require 'rest_client'
|
2
|
-
require 'sequel'
|
3
|
-
require 'zlib'
|
4
|
-
|
5
|
-
require File.dirname(__FILE__) + '/progress_bar'
|
6
|
-
require File.dirname(__FILE__) + '/config'
|
7
|
-
require File.dirname(__FILE__) + '/utils'
|
8
|
-
|
9
|
-
module Taps
|
10
|
-
class ClientSession
|
11
|
-
attr_reader :database_url, :remote_url, :default_chunksize
|
12
|
-
|
13
|
-
def initialize(database_url, remote_url, default_chunksize)
|
14
|
-
@database_url = database_url
|
15
|
-
@remote_url = remote_url
|
16
|
-
@default_chunksize = default_chunksize
|
17
|
-
end
|
18
|
-
|
19
|
-
def self.start(database_url, remote_url, default_chunksize, &block)
|
20
|
-
s = new(database_url, remote_url, default_chunksize)
|
21
|
-
yield s
|
22
|
-
s.close_session
|
23
|
-
end
|
24
|
-
|
25
|
-
def self.quickstart(&block)
|
26
|
-
start(Taps::Config.database_url, Taps::Config.remote_url, Taps::Config.chunksize) do |s|
|
27
|
-
yield s
|
28
|
-
end
|
29
|
-
end
|
30
|
-
|
31
|
-
def db
|
32
|
-
@db ||= Sequel.connect(database_url)
|
33
|
-
end
|
34
|
-
|
35
|
-
def server
|
36
|
-
@server ||= RestClient::Resource.new(remote_url)
|
37
|
-
end
|
38
|
-
|
39
|
-
def session_resource
|
40
|
-
@session_resource ||= open_session
|
41
|
-
end
|
42
|
-
|
43
|
-
def open_session
|
44
|
-
uri = server['sessions'].post('', http_headers)
|
45
|
-
server[uri]
|
46
|
-
end
|
47
|
-
|
48
|
-
def set_session(uri)
|
49
|
-
@session_resource = server[uri]
|
50
|
-
end
|
51
|
-
|
52
|
-
def close_session
|
53
|
-
@session_resource.delete(http_headers) if @session_resource
|
54
|
-
end
|
55
|
-
|
56
|
-
def safe_url(url)
|
57
|
-
url.sub(/\/\/(.+?)?:(.*?)@/, '//\1:[hidden]@')
|
58
|
-
end
|
59
|
-
|
60
|
-
def safe_remote_url
|
61
|
-
safe_url(remote_url)
|
62
|
-
end
|
63
|
-
|
64
|
-
def safe_database_url
|
65
|
-
safe_url(database_url)
|
66
|
-
end
|
67
|
-
|
68
|
-
def http_headers(extra = {})
|
69
|
-
{ :taps_version => Taps.compatible_version }.merge(extra)
|
70
|
-
end
|
71
|
-
|
72
|
-
def cmd_send
|
73
|
-
begin
|
74
|
-
verify_server
|
75
|
-
cmd_send_schema
|
76
|
-
cmd_send_data
|
77
|
-
cmd_send_indexes
|
78
|
-
cmd_send_reset_sequences
|
79
|
-
rescue RestClient::Exception => e
|
80
|
-
if e.respond_to?(:response)
|
81
|
-
puts "!!! Caught Server Exception"
|
82
|
-
puts "#{e.response}"
|
83
|
-
exit(1)
|
84
|
-
else
|
85
|
-
raise
|
86
|
-
end
|
87
|
-
end
|
88
|
-
end
|
89
|
-
|
90
|
-
def cmd_send_indexes
|
91
|
-
puts "Sending indexes"
|
92
|
-
|
93
|
-
index_data = Taps::Utils.schema_bin(:indexes, database_url)
|
94
|
-
session_resource['indexes'].post(index_data, http_headers)
|
95
|
-
end
|
96
|
-
|
97
|
-
def cmd_send_schema
|
98
|
-
puts "Sending schema"
|
99
|
-
|
100
|
-
schema_data = Taps::Utils.schema_bin(:dump, database_url)
|
101
|
-
session_resource['schema'].post(schema_data, http_headers)
|
102
|
-
end
|
103
|
-
|
104
|
-
def cmd_send_reset_sequences
|
105
|
-
puts "Resetting sequences"
|
106
|
-
|
107
|
-
session_resource["reset_sequences"].post('', http_headers)
|
108
|
-
end
|
109
|
-
|
110
|
-
def cmd_send_data
|
111
|
-
puts "Sending data"
|
112
|
-
|
113
|
-
tables_with_counts, record_count = fetch_tables_info
|
114
|
-
|
115
|
-
puts "#{tables_with_counts.size} tables, #{format_number(record_count)} records"
|
116
|
-
|
117
|
-
tables_with_counts.each do |table_name, count|
|
118
|
-
table = db[table_name]
|
119
|
-
order = Taps::Utils.order_by(db, table_name)
|
120
|
-
chunksize = self.default_chunksize
|
121
|
-
string_columns = Taps::Utils.incorrect_blobs(db, table_name)
|
122
|
-
|
123
|
-
progress = ProgressBar.new(table_name.to_s, count)
|
124
|
-
|
125
|
-
offset = 0
|
126
|
-
loop do
|
127
|
-
row_size = 0
|
128
|
-
chunksize = Taps::Utils.calculate_chunksize(chunksize) do |c|
|
129
|
-
time_skip_start = Time.now
|
130
|
-
rows = Taps::Utils.format_data(table.order(*order).limit(c, offset).all, string_columns)
|
131
|
-
break if rows == { }
|
132
|
-
|
133
|
-
row_size = rows[:data].size
|
134
|
-
gzip_data = Taps::Utils.gzip(Marshal.dump(rows))
|
135
|
-
time_skip = Time.now - time_skip_start
|
136
|
-
|
137
|
-
begin
|
138
|
-
session_resource["tables/#{table_name}"].post(gzip_data, http_headers({
|
139
|
-
:content_type => 'application/octet-stream',
|
140
|
-
:taps_checksum => Taps::Utils.checksum(gzip_data).to_s}))
|
141
|
-
rescue RestClient::RequestFailed => e
|
142
|
-
# retry the same data, it got corrupted somehow.
|
143
|
-
if e.http_code == 412
|
144
|
-
next
|
145
|
-
end
|
146
|
-
raise
|
147
|
-
end
|
148
|
-
time_skip
|
149
|
-
end
|
150
|
-
|
151
|
-
progress.inc(row_size)
|
152
|
-
offset += row_size
|
153
|
-
|
154
|
-
break if row_size == 0
|
155
|
-
end
|
156
|
-
|
157
|
-
progress.finish
|
158
|
-
end
|
159
|
-
end
|
160
|
-
|
161
|
-
def fetch_tables_info
|
162
|
-
tables_with_counts = {}
|
163
|
-
record_count = db.tables.inject(0) do |record_count, table|
|
164
|
-
tables_with_counts[table] = db[table].count
|
165
|
-
record_count += tables_with_counts[table]
|
166
|
-
end
|
167
|
-
|
168
|
-
[ tables_with_counts, record_count ]
|
169
|
-
end
|
170
|
-
|
171
|
-
def cmd_receive
|
172
|
-
begin
|
173
|
-
verify_server
|
174
|
-
cmd_receive_schema
|
175
|
-
cmd_receive_data
|
176
|
-
cmd_receive_indexes
|
177
|
-
cmd_reset_sequences
|
178
|
-
rescue RestClient::Exception => e
|
179
|
-
if e.respond_to?(:response)
|
180
|
-
puts "!!! Caught Server Exception"
|
181
|
-
puts "#{e.response.body}"
|
182
|
-
exit(1)
|
183
|
-
else
|
184
|
-
raise
|
185
|
-
end
|
186
|
-
end
|
187
|
-
end
|
188
|
-
|
189
|
-
def cmd_receive_data
|
190
|
-
puts "Receiving data"
|
191
|
-
|
192
|
-
tables_with_counts, record_count = fetch_remote_tables_info
|
193
|
-
|
194
|
-
puts "#{tables_with_counts.size} tables, #{format_number(record_count)} records"
|
195
|
-
|
196
|
-
tables_with_counts.each do |table_name, count|
|
197
|
-
table = db[table_name.to_sym]
|
198
|
-
chunksize = default_chunksize
|
199
|
-
|
200
|
-
progress = ProgressBar.new(table_name.to_s, count)
|
201
|
-
|
202
|
-
offset = 0
|
203
|
-
loop do
|
204
|
-
begin
|
205
|
-
chunksize, rows = fetch_table_rows(table_name, chunksize, offset)
|
206
|
-
rescue CorruptedData
|
207
|
-
next
|
208
|
-
end
|
209
|
-
break if rows == { }
|
210
|
-
|
211
|
-
table.import(rows[:header], rows[:data])
|
212
|
-
|
213
|
-
progress.inc(rows[:data].size)
|
214
|
-
offset += rows[:data].size
|
215
|
-
end
|
216
|
-
|
217
|
-
progress.finish
|
218
|
-
end
|
219
|
-
end
|
220
|
-
|
221
|
-
class CorruptedData < Exception; end
|
222
|
-
|
223
|
-
def fetch_table_rows(table_name, chunksize, offset)
|
224
|
-
response = nil
|
225
|
-
chunksize = Taps::Utils.calculate_chunksize(chunksize) do |c|
|
226
|
-
response = session_resource["tables/#{table_name}/#{c}?offset=#{offset}"].get(http_headers)
|
227
|
-
end
|
228
|
-
raise CorruptedData unless Taps::Utils.valid_data?(response.to_s, response.headers[:taps_checksum])
|
229
|
-
|
230
|
-
begin
|
231
|
-
rows = Marshal.load(Taps::Utils.gunzip(response.to_s))
|
232
|
-
rescue Object => e
|
233
|
-
puts "Error encountered loading data, wrote the data chunk to dump.#{Process.pid}.gz"
|
234
|
-
File.open("dump.#{Process.pid}.gz", "w") { |f| f.write(response.to_s) }
|
235
|
-
raise
|
236
|
-
end
|
237
|
-
[chunksize, rows]
|
238
|
-
end
|
239
|
-
|
240
|
-
def fetch_remote_tables_info
|
241
|
-
retries = 0
|
242
|
-
max_retries = 1
|
243
|
-
begin
|
244
|
-
tables_with_counts = Marshal.load(session_resource['tables'].get(http_headers))
|
245
|
-
record_count = tables_with_counts.values.inject(0) { |a,c| a += c }
|
246
|
-
rescue RestClient::Exception
|
247
|
-
retries += 1
|
248
|
-
retry if retries <= max_retries
|
249
|
-
puts "Unable to fetch tables information from #{remote_url}. Please check the server log."
|
250
|
-
exit(1)
|
251
|
-
end
|
252
|
-
|
253
|
-
[ tables_with_counts, record_count ]
|
254
|
-
end
|
255
|
-
|
256
|
-
def cmd_receive_schema
|
257
|
-
puts "Receiving schema"
|
258
|
-
|
259
|
-
schema_data = session_resource['schema'].get(http_headers)
|
260
|
-
output = Taps::Utils.load_schema(database_url, schema_data)
|
261
|
-
puts output if output
|
262
|
-
end
|
263
|
-
|
264
|
-
def cmd_receive_indexes
|
265
|
-
puts "Receiving indexes"
|
266
|
-
|
267
|
-
index_data = session_resource['indexes'].get(http_headers)
|
268
|
-
|
269
|
-
output = Taps::Utils.load_indexes(database_url, index_data)
|
270
|
-
puts output if output
|
271
|
-
end
|
272
|
-
|
273
|
-
def cmd_reset_sequences
|
274
|
-
puts "Resetting sequences"
|
275
|
-
|
276
|
-
output = Taps::Utils.schema_bin(:reset_db_sequences, database_url)
|
277
|
-
puts output if output
|
278
|
-
end
|
279
|
-
|
280
|
-
def format_number(num)
|
281
|
-
num.to_s.gsub(/(\d)(?=(\d\d\d)+(?!\d))/, "\\1,")
|
282
|
-
end
|
283
|
-
|
284
|
-
def verify_server
|
285
|
-
begin
|
286
|
-
server['/'].get(http_headers)
|
287
|
-
rescue RestClient::RequestFailed => e
|
288
|
-
if e.http_code == 417
|
289
|
-
puts "#{safe_remote_url} is running a different minor version of taps."
|
290
|
-
puts "#{e.response.body}"
|
291
|
-
exit(1)
|
292
|
-
else
|
293
|
-
raise
|
294
|
-
end
|
295
|
-
rescue RestClient::Unauthorized
|
296
|
-
puts "Bad credentials given for #{safe_remote_url}"
|
297
|
-
exit(1)
|
298
|
-
rescue Errno::ECONNREFUSED
|
299
|
-
puts "Can't connect to #{safe_remote_url}. Please check that it's running"
|
300
|
-
exit(1)
|
301
|
-
end
|
302
|
-
end
|
303
|
-
end
|
304
|
-
end
|