ricardochimal-taps 0.2.0
Sign up to get free protection for your applications and to get access to all the features.
- data/LICENSE +20 -0
- data/README.rdoc +29 -0
- data/Rakefile +52 -0
- data/VERSION.yml +4 -0
- data/bin/schema +38 -0
- data/bin/taps +12 -0
- data/lib/taps/cli.rb +57 -0
- data/lib/taps/client_session.rb +236 -0
- data/lib/taps/config.rb +23 -0
- data/lib/taps/db_session.rb +25 -0
- data/lib/taps/progress_bar.rb +236 -0
- data/lib/taps/schema.rb +82 -0
- data/lib/taps/server.rb +141 -0
- data/lib/taps/utils.rb +76 -0
- data/spec/base.rb +21 -0
- data/spec/client_session_spec.rb +82 -0
- data/spec/schema_spec.rb +38 -0
- data/spec/server_spec.rb +33 -0
- data/spec/utils_spec.rb +45 -0
- metadata +118 -0
data/LICENSE
ADDED
@@ -0,0 +1,20 @@
|
|
1
|
+
Copyright (c) 2008 Ricardo Chimal, Jr
|
2
|
+
|
3
|
+
Permission is hereby granted, free of charge, to any person obtaining
|
4
|
+
a copy of this software and associated documentation files (the
|
5
|
+
"Software"), to deal in the Software without restriction, including
|
6
|
+
without limitation the rights to use, copy, modify, merge, publish,
|
7
|
+
distribute, sublicense, and/or sell copies of the Software, and to
|
8
|
+
permit persons to whom the Software is furnished to do so, subject to
|
9
|
+
the following conditions:
|
10
|
+
|
11
|
+
The above copyright notice and this permission notice shall be
|
12
|
+
included in all copies or substantial portions of the Software.
|
13
|
+
|
14
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
15
|
+
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
16
|
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
17
|
+
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
18
|
+
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
19
|
+
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
20
|
+
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
data/README.rdoc
ADDED
@@ -0,0 +1,29 @@
|
|
1
|
+
= Taps -- simple database import/export app
|
2
|
+
|
3
|
+
A simple database agnostic import/export app to transfer data to/from a remote database.
|
4
|
+
|
5
|
+
== Usage: Server
|
6
|
+
|
7
|
+
$ taps server postgres://localdbuser:localdbpass@localhost/dbname user password
|
8
|
+
|
9
|
+
== Usage: Client
|
10
|
+
|
11
|
+
When you want to pull down a database from a taps server
|
12
|
+
|
13
|
+
$ taps pull postgres://dbuser:dbpassword@localhost/dbname http://user:password@example.com:5000
|
14
|
+
|
15
|
+
or when you want to push a local database to a taps server
|
16
|
+
|
17
|
+
$ taps push postgres://dbuser:dbpassword@localhost/dbname http://user:password@example.com:5000
|
18
|
+
|
19
|
+
== Meta
|
20
|
+
|
21
|
+
Maintained by Ricardo Chimal, Jr. (ricardo at heroku dot com)
|
22
|
+
|
23
|
+
Written by Ricardo Chimal, Jr. (ricardo at heroku dot com) and Adam Wiggins (adam at heroku dot com)
|
24
|
+
|
25
|
+
Early research and inspiration by Blake Mizerany
|
26
|
+
|
27
|
+
Released under the MIT License: http://www.opensource.org/licenses/mit-license.php
|
28
|
+
|
29
|
+
http://github.com/ricardochimal/taps
|
data/Rakefile
ADDED
@@ -0,0 +1,52 @@
|
|
1
|
+
begin
|
2
|
+
require 'jeweler'
|
3
|
+
Jeweler::Tasks.new do |s|
|
4
|
+
s.name = "taps"
|
5
|
+
s.summary = %Q{simple database import/export app}
|
6
|
+
s.email = "ricardo@heroku.com"
|
7
|
+
s.homepage = "http://github.com/ricardochimal/taps"
|
8
|
+
s.description = "A simple database agnostic import/export app to transfer data to/from a remote database."
|
9
|
+
s.authors = ["Ricardo Chimal, Jr.", "Adam Wiggins"]
|
10
|
+
|
11
|
+
s.add_dependency 'sinatra', '~> 0.9.0'
|
12
|
+
s.add_dependency 'activerecord', '= 2.2.2'
|
13
|
+
s.add_dependency 'thor', '= 0.9.9'
|
14
|
+
s.add_dependency 'rest-client', '~> 0.9.0'
|
15
|
+
s.add_dependency 'sequel', '~> 2.10.0'
|
16
|
+
|
17
|
+
s.rubyforge_project = "taps"
|
18
|
+
s.rubygems_version = '1.3.1'
|
19
|
+
|
20
|
+
s.files = FileList['spec/*.rb'] + FileList['lib/**/*.rb'] + ['README.rdoc', 'LICENSE', 'VERSION.yml', 'Rakefile']
|
21
|
+
s.executables = ['taps', 'schema']
|
22
|
+
end
|
23
|
+
rescue LoadError
|
24
|
+
puts "Jeweler not available. Install it with: sudo gem install technicalpickles-jeweler -s http://gems.github.com"
|
25
|
+
end
|
26
|
+
|
27
|
+
require 'rake/rdoctask'
|
28
|
+
Rake::RDocTask.new do |rdoc|
|
29
|
+
rdoc.rdoc_dir = 'rdoc'
|
30
|
+
rdoc.title = 'taps'
|
31
|
+
rdoc.options << '--line-numbers' << '--inline-source'
|
32
|
+
rdoc.rdoc_files.include('README*')
|
33
|
+
rdoc.rdoc_files.include('lib/**/*.rb')
|
34
|
+
end
|
35
|
+
|
36
|
+
begin
|
37
|
+
require 'rcov/rcovtask'
|
38
|
+
Rcov::RcovTask.new do |t|
|
39
|
+
t.libs << 'spec'
|
40
|
+
t.test_files = FileList['spec/*_spec.rb']
|
41
|
+
t.verbose = true
|
42
|
+
end
|
43
|
+
rescue LoadError
|
44
|
+
puts "RCov is not available. In order to run rcov, you must: sudo gem install spicycode-rcov"
|
45
|
+
end
|
46
|
+
|
47
|
+
desc "Run all specs"
|
48
|
+
task :spec do
|
49
|
+
system "bacon #{File.dirname(__FILE__)}/spec/*_spec.rb"
|
50
|
+
end
|
51
|
+
|
52
|
+
task :default => :spec
|
data/VERSION.yml
ADDED
data/bin/schema
ADDED
@@ -0,0 +1,38 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
require 'rubygems'
|
4
|
+
gem 'activerecord', '= 2.2.2'
|
5
|
+
|
6
|
+
require File.dirname(__FILE__) + '/../lib/taps/schema'
|
7
|
+
|
8
|
+
cmd = ARGV.shift.strip rescue ''
|
9
|
+
database_url = ARGV.shift.strip rescue ''
|
10
|
+
|
11
|
+
def show_usage_and_exit
|
12
|
+
puts <<EOTXT
|
13
|
+
schema dump <database_url>
|
14
|
+
schema indexes <database_url>
|
15
|
+
schema reset_db_sequences <database_url>
|
16
|
+
schema load <database_url> <schema_file>
|
17
|
+
schema load_indexes <database_url> <indexes_file>
|
18
|
+
EOTXT
|
19
|
+
exit(1)
|
20
|
+
end
|
21
|
+
|
22
|
+
if cmd == 'dump'
|
23
|
+
puts Taps::Schema.dump_without_indexes(database_url)
|
24
|
+
elsif cmd == 'indexes'
|
25
|
+
puts Taps::Schema.indexes(database_url)
|
26
|
+
elsif cmd == 'load_indexes'
|
27
|
+
filename = ARGV.shift.strip rescue ''
|
28
|
+
indexes = File.read(filename) rescue show_usage_and_exit
|
29
|
+
Taps::Schema.load_indexes(database_url, indexes)
|
30
|
+
elsif cmd == 'load'
|
31
|
+
filename = ARGV.shift.strip rescue ''
|
32
|
+
schema = File.read(filename) rescue show_usage_and_exit
|
33
|
+
Taps::Schema.load(database_url, schema)
|
34
|
+
elsif cmd == 'reset_db_sequences'
|
35
|
+
Taps::Schema.reset_db_sequences(database_url)
|
36
|
+
else
|
37
|
+
show_usage_and_exit
|
38
|
+
end
|
data/bin/taps
ADDED
@@ -0,0 +1,12 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
require 'rubygems'
|
4
|
+
gem 'activerecord', '= 2.2.2'
|
5
|
+
gem 'thor', '= 0.9.9'
|
6
|
+
gem 'rest-client', '~> 0.9.0'
|
7
|
+
gem 'sinatra', '~> 0.9.0'
|
8
|
+
gem 'sequel', '~> 2.10.0'
|
9
|
+
|
10
|
+
require File.dirname(__FILE__) + '/../lib/taps/cli'
|
11
|
+
|
12
|
+
Taps::Cli.start
|
data/lib/taps/cli.rb
ADDED
@@ -0,0 +1,57 @@
|
|
1
|
+
require 'thor'
|
2
|
+
require File.dirname(__FILE__) + '/config'
|
3
|
+
|
4
|
+
Taps::Config.taps_database_url = 'sqlite://taps.db'
|
5
|
+
|
6
|
+
module Taps
|
7
|
+
class Cli < Thor
|
8
|
+
desc "server <local_database_url> <login> <password>", "Start a taps database import/export server"
|
9
|
+
method_options(:port => :numeric)
|
10
|
+
def server(database_url, login, password)
|
11
|
+
Taps::Config.database_url = database_url
|
12
|
+
Taps::Config.login = login
|
13
|
+
Taps::Config.password = password
|
14
|
+
|
15
|
+
port = options[:port] || 5000
|
16
|
+
|
17
|
+
Taps::Config.verify_database_url
|
18
|
+
|
19
|
+
require File.dirname(__FILE__) + '/server'
|
20
|
+
Taps::Server.run!({
|
21
|
+
:port => port,
|
22
|
+
:environment => :production,
|
23
|
+
:logging => true
|
24
|
+
})
|
25
|
+
end
|
26
|
+
|
27
|
+
desc "pull <local_database_url> <remote_url>", "Pull a database from a taps server"
|
28
|
+
method_options(:chunksize => :numeric)
|
29
|
+
def pull(database_url, remote_url)
|
30
|
+
clientxfer(:cmd_receive, database_url, remote_url)
|
31
|
+
end
|
32
|
+
|
33
|
+
desc "push <local_database_url> <remote_url>", "Push a database to a taps server"
|
34
|
+
method_options(:chunksize => :numeric)
|
35
|
+
def push(database_url, remote_url)
|
36
|
+
clientxfer(:cmd_send, database_url, remote_url)
|
37
|
+
end
|
38
|
+
|
39
|
+
def clientxfer(method, database_url, remote_url)
|
40
|
+
if options[:chunksize]
|
41
|
+
Taps::Config.chunksize = options[:chunksize] < 100 ? 100 : options[:chunksize]
|
42
|
+
else
|
43
|
+
Taps::Config.chunksize = 1000
|
44
|
+
end
|
45
|
+
Taps::Config.database_url = database_url
|
46
|
+
Taps::Config.remote_url = remote_url
|
47
|
+
|
48
|
+
Taps::Config.verify_database_url
|
49
|
+
|
50
|
+
require File.dirname(__FILE__) + '/client_session'
|
51
|
+
|
52
|
+
Taps::ClientSession.quickstart do |session|
|
53
|
+
session.send(method)
|
54
|
+
end
|
55
|
+
end
|
56
|
+
end
|
57
|
+
end
|
@@ -0,0 +1,236 @@
|
|
1
|
+
require 'rest_client'
|
2
|
+
require 'sequel'
|
3
|
+
require 'zlib'
|
4
|
+
|
5
|
+
require File.dirname(__FILE__) + '/progress_bar'
|
6
|
+
require File.dirname(__FILE__) + '/config'
|
7
|
+
require File.dirname(__FILE__) + '/utils'
|
8
|
+
|
9
|
+
module Taps
|
10
|
+
class ClientSession
|
11
|
+
attr_reader :database_url, :remote_url, :default_chunksize
|
12
|
+
|
13
|
+
def initialize(database_url, remote_url, default_chunksize)
|
14
|
+
@database_url = database_url
|
15
|
+
@remote_url = remote_url
|
16
|
+
@default_chunksize = default_chunksize
|
17
|
+
end
|
18
|
+
|
19
|
+
def self.start(database_url, remote_url, default_chunksize, &block)
|
20
|
+
s = new(database_url, remote_url, default_chunksize)
|
21
|
+
yield s
|
22
|
+
s.close_session
|
23
|
+
end
|
24
|
+
|
25
|
+
def self.quickstart(&block)
|
26
|
+
start(Taps::Config.database_url, Taps::Config.remote_url, Taps::Config.chunksize) do |s|
|
27
|
+
yield s
|
28
|
+
end
|
29
|
+
end
|
30
|
+
|
31
|
+
def db
|
32
|
+
@db ||= Sequel.connect(database_url)
|
33
|
+
end
|
34
|
+
|
35
|
+
def server
|
36
|
+
@server ||= RestClient::Resource.new(remote_url)
|
37
|
+
end
|
38
|
+
|
39
|
+
def session_resource
|
40
|
+
@session_resource ||= open_session
|
41
|
+
end
|
42
|
+
|
43
|
+
def open_session
|
44
|
+
uri = server['sessions'].post('', :taps_version => Taps::VERSION)
|
45
|
+
server[uri]
|
46
|
+
end
|
47
|
+
|
48
|
+
def close_session
|
49
|
+
@session_resource.delete(:taps_version => Taps::VERSION) if @session_resource
|
50
|
+
end
|
51
|
+
|
52
|
+
def cmd_send
|
53
|
+
verify_server
|
54
|
+
cmd_send_schema
|
55
|
+
cmd_send_data
|
56
|
+
cmd_send_indexes
|
57
|
+
cmd_send_reset_sequences
|
58
|
+
end
|
59
|
+
|
60
|
+
def cmd_send_indexes
|
61
|
+
puts "Sending schema indexes to remote taps server #{remote_url} from local database #{database_url}"
|
62
|
+
|
63
|
+
index_data = `#{File.dirname(__FILE__)}/../../bin/schema indexes #{database_url}`
|
64
|
+
session_resource['indexes'].post(index_data, :taps_version => Taps::VERSION)
|
65
|
+
end
|
66
|
+
|
67
|
+
def cmd_send_schema
|
68
|
+
puts "Sending schema to remote taps server #{remote_url} from local database #{database_url}"
|
69
|
+
|
70
|
+
schema_data = `#{File.dirname(__FILE__)}/../../bin/schema dump #{database_url}`
|
71
|
+
session_resource['schema'].post(schema_data, :taps_version => Taps::VERSION)
|
72
|
+
end
|
73
|
+
|
74
|
+
def cmd_send_reset_sequences
|
75
|
+
puts "Resetting db sequences in remote taps server at #{remote_url}"
|
76
|
+
|
77
|
+
session_resource["reset_sequences"].post('', :taps_version => Taps::VERSION)
|
78
|
+
end
|
79
|
+
|
80
|
+
def cmd_send_data
|
81
|
+
puts "Sending schema and data from local database #{database_url} to remote taps server at #{remote_url}"
|
82
|
+
|
83
|
+
db.tables.each do |table_name|
|
84
|
+
table = db[table_name]
|
85
|
+
count = table.count
|
86
|
+
columns = table.columns
|
87
|
+
order = columns.include?(:id) ? :id : columns.first
|
88
|
+
chunksize = self.default_chunksize
|
89
|
+
|
90
|
+
progress = ProgressBar.new(table_name.to_s, count)
|
91
|
+
|
92
|
+
offset = 0
|
93
|
+
loop do
|
94
|
+
rows = Taps::Utils.format_data(table.order(order).limit(chunksize, offset).all)
|
95
|
+
break if rows == { }
|
96
|
+
|
97
|
+
gzip_data = Taps::Utils.gzip(Marshal.dump(rows))
|
98
|
+
|
99
|
+
chunksize = Taps::Utils.calculate_chunksize(chunksize) do
|
100
|
+
begin
|
101
|
+
session_resource["tables/#{table_name}"].post(gzip_data,
|
102
|
+
:taps_version => Taps::VERSION,
|
103
|
+
:content_type => 'application/octet-stream',
|
104
|
+
:taps_checksum => Taps::Utils.checksum(gzip_data).to_s)
|
105
|
+
rescue RestClient::RequestFailed => e
|
106
|
+
# retry the same data, it got corrupted somehow.
|
107
|
+
if e.http_code == 412
|
108
|
+
next
|
109
|
+
end
|
110
|
+
raise
|
111
|
+
end
|
112
|
+
end
|
113
|
+
|
114
|
+
progress.inc(rows[:data].size)
|
115
|
+
offset += rows[:data].size
|
116
|
+
end
|
117
|
+
|
118
|
+
progress.finish
|
119
|
+
end
|
120
|
+
end
|
121
|
+
|
122
|
+
def cmd_receive
|
123
|
+
verify_server
|
124
|
+
cmd_receive_schema
|
125
|
+
cmd_receive_data
|
126
|
+
cmd_receive_indexes
|
127
|
+
cmd_reset_sequences
|
128
|
+
end
|
129
|
+
|
130
|
+
def cmd_receive_data
|
131
|
+
puts "Receiving data from remote taps server #{remote_url} into local database #{database_url}"
|
132
|
+
|
133
|
+
tables_with_counts, record_count = fetch_tables_info
|
134
|
+
|
135
|
+
puts "#{tables_with_counts.size} tables, #{format_number(record_count)} records"
|
136
|
+
|
137
|
+
tables_with_counts.each do |table_name, count|
|
138
|
+
table = db[table_name.to_sym]
|
139
|
+
chunksize = default_chunksize
|
140
|
+
|
141
|
+
progress = ProgressBar.new(table_name.to_s, count)
|
142
|
+
|
143
|
+
offset = 0
|
144
|
+
loop do
|
145
|
+
begin
|
146
|
+
chunksize, rows = fetch_table_rows(table_name, chunksize, offset)
|
147
|
+
rescue CorruptedData
|
148
|
+
next
|
149
|
+
end
|
150
|
+
break if rows == { }
|
151
|
+
|
152
|
+
table.multi_insert(rows[:header], rows[:data])
|
153
|
+
|
154
|
+
progress.inc(rows[:data].size)
|
155
|
+
offset += rows[:data].size
|
156
|
+
end
|
157
|
+
|
158
|
+
progress.finish
|
159
|
+
end
|
160
|
+
end
|
161
|
+
|
162
|
+
class CorruptedData < Exception; end
|
163
|
+
|
164
|
+
def fetch_table_rows(table_name, chunksize, offset)
|
165
|
+
response = nil
|
166
|
+
chunksize = Taps::Utils.calculate_chunksize(chunksize) do
|
167
|
+
response = session_resource["tables/#{table_name}/#{chunksize}?offset=#{offset}"].get(:taps_version => Taps::VERSION)
|
168
|
+
end
|
169
|
+
raise CorruptedData unless Taps::Utils.valid_data?(response.to_s, response.headers[:taps_checksum])
|
170
|
+
|
171
|
+
rows = Marshal.load(Taps::Utils.gunzip(response.to_s))
|
172
|
+
[chunksize, rows]
|
173
|
+
end
|
174
|
+
|
175
|
+
def fetch_tables_info
|
176
|
+
retries = 0
|
177
|
+
max_retries = 1
|
178
|
+
begin
|
179
|
+
tables_with_counts = Marshal.load(session_resource['tables'].get(:taps_version => Taps::VERSION))
|
180
|
+
record_count = tables_with_counts.values.inject(0) { |a,c| a += c }
|
181
|
+
rescue RestClient::Exception
|
182
|
+
retries += 1
|
183
|
+
retry if retries <= max_retries
|
184
|
+
puts "Unable to fetch tables information from #{remote_url}. Please check the server log."
|
185
|
+
exit(1)
|
186
|
+
end
|
187
|
+
|
188
|
+
[ tables_with_counts, record_count ]
|
189
|
+
end
|
190
|
+
|
191
|
+
def cmd_receive_schema
|
192
|
+
puts "Receiving schema from remote taps server #{remote_url} into local database #{database_url}"
|
193
|
+
|
194
|
+
schema_data = session_resource['schema'].get(:taps_version => Taps::VERSION)
|
195
|
+
puts Taps::Utils.load_schema(database_url, schema_data)
|
196
|
+
end
|
197
|
+
|
198
|
+
def cmd_receive_indexes
|
199
|
+
puts "Receiving schema indexes from remote taps server #{remote_url} into local database #{database_url}"
|
200
|
+
|
201
|
+
index_data = session_resource['indexes'].get(:taps_version => Taps::VERSION)
|
202
|
+
|
203
|
+
puts Taps::Utils.load_indexes(database_url, index_data)
|
204
|
+
end
|
205
|
+
|
206
|
+
def cmd_reset_sequences
|
207
|
+
puts "Resetting db sequences in #{database_url}"
|
208
|
+
|
209
|
+
puts `#{File.dirname(__FILE__)}/../../bin/schema reset_db_sequences #{database_url}`
|
210
|
+
end
|
211
|
+
|
212
|
+
def format_number(num)
|
213
|
+
num.to_s.gsub(/(\d)(?=(\d\d\d)+(?!\d))/, "\\1,")
|
214
|
+
end
|
215
|
+
|
216
|
+
def verify_server
|
217
|
+
begin
|
218
|
+
server['/'].get(:taps_version => Taps::VERSION)
|
219
|
+
rescue RestClient::RequestFailed => e
|
220
|
+
if e.http_code == 417
|
221
|
+
puts "#{remote_url} is running a different version of taps."
|
222
|
+
puts "#{e.response.body}"
|
223
|
+
exit(1)
|
224
|
+
else
|
225
|
+
raise
|
226
|
+
end
|
227
|
+
rescue RestClient::Unauthorized
|
228
|
+
puts "Bad credentials given for #{remote_url}"
|
229
|
+
exit(1)
|
230
|
+
rescue Errno::ECONNREFUSED
|
231
|
+
puts "Can't connect to #{remote_url}. Please check that it's running"
|
232
|
+
exit(1)
|
233
|
+
end
|
234
|
+
end
|
235
|
+
end
|
236
|
+
end
|
data/lib/taps/config.rb
ADDED
@@ -0,0 +1,23 @@
|
|
1
|
+
require 'sequel'
|
2
|
+
|
3
|
+
module Taps
|
4
|
+
|
5
|
+
VERSION = '0.2.0'
|
6
|
+
|
7
|
+
class Config
|
8
|
+
class << self
|
9
|
+
attr_accessor :taps_database_url
|
10
|
+
attr_accessor :login, :password, :database_url, :remote_url
|
11
|
+
attr_accessor :chunksize
|
12
|
+
|
13
|
+
def verify_database_url
|
14
|
+
db = Sequel.connect(self.database_url)
|
15
|
+
db.tables
|
16
|
+
db.disconnect
|
17
|
+
rescue Object => e
|
18
|
+
puts "Failed to connect to database:\n #{e.class} -> #{e}"
|
19
|
+
exit 1
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
23
|
+
end
|
@@ -0,0 +1,25 @@
|
|
1
|
+
Sequel::Model.db = Sequel.connect(Taps::Config.taps_database_url)
|
2
|
+
|
3
|
+
class DbSession < Sequel::Model
|
4
|
+
set_schema do
|
5
|
+
primary_key :id
|
6
|
+
text :key
|
7
|
+
text :database_url
|
8
|
+
timestamp :started_at
|
9
|
+
timestamp :last_access
|
10
|
+
end
|
11
|
+
|
12
|
+
def connection
|
13
|
+
@@connections ||= {}
|
14
|
+
@@connections[key] ||= Sequel.connect(database_url)
|
15
|
+
end
|
16
|
+
|
17
|
+
def disconnect
|
18
|
+
if defined? @@connections and @@connections[key]
|
19
|
+
@@connections[key].disconnect
|
20
|
+
@@connections.delete key
|
21
|
+
end
|
22
|
+
end
|
23
|
+
end
|
24
|
+
|
25
|
+
DbSession.create_table! unless DbSession.table_exists?
|
@@ -0,0 +1,236 @@
|
|
1
|
+
#
|
2
|
+
# Ruby/ProgressBar - a text progress bar library
|
3
|
+
#
|
4
|
+
# Copyright (C) 2001-2005 Satoru Takabayashi <satoru@namazu.org>
|
5
|
+
# All rights reserved.
|
6
|
+
# This is free software with ABSOLUTELY NO WARRANTY.
|
7
|
+
#
|
8
|
+
# You can redistribute it and/or modify it under the terms
|
9
|
+
# of Ruby's license.
|
10
|
+
#
|
11
|
+
|
12
|
+
class ProgressBar
|
13
|
+
VERSION = "0.9"
|
14
|
+
|
15
|
+
def initialize (title, total, out = STDERR)
|
16
|
+
@title = title
|
17
|
+
@total = total
|
18
|
+
@out = out
|
19
|
+
@terminal_width = 80
|
20
|
+
@bar_mark = "o"
|
21
|
+
@current = 0
|
22
|
+
@previous = 0
|
23
|
+
@finished_p = false
|
24
|
+
@start_time = Time.now
|
25
|
+
@previous_time = @start_time
|
26
|
+
@title_width = 14
|
27
|
+
@format = "%-#{@title_width}s %3d%% %s %s"
|
28
|
+
@format_arguments = [:title, :percentage, :bar, :stat]
|
29
|
+
clear
|
30
|
+
show
|
31
|
+
end
|
32
|
+
attr_reader :title
|
33
|
+
attr_reader :current
|
34
|
+
attr_reader :total
|
35
|
+
attr_accessor :start_time
|
36
|
+
|
37
|
+
private
|
38
|
+
def fmt_bar
|
39
|
+
bar_width = do_percentage * @terminal_width / 100
|
40
|
+
sprintf("|%s%s|",
|
41
|
+
@bar_mark * bar_width,
|
42
|
+
" " * (@terminal_width - bar_width))
|
43
|
+
end
|
44
|
+
|
45
|
+
def fmt_percentage
|
46
|
+
do_percentage
|
47
|
+
end
|
48
|
+
|
49
|
+
def fmt_stat
|
50
|
+
if @finished_p then elapsed else eta end
|
51
|
+
end
|
52
|
+
|
53
|
+
def fmt_stat_for_file_transfer
|
54
|
+
if @finished_p then
|
55
|
+
sprintf("%s %s %s", bytes, transfer_rate, elapsed)
|
56
|
+
else
|
57
|
+
sprintf("%s %s %s", bytes, transfer_rate, eta)
|
58
|
+
end
|
59
|
+
end
|
60
|
+
|
61
|
+
def fmt_title
|
62
|
+
@title[0,(@title_width - 1)] + ":"
|
63
|
+
end
|
64
|
+
|
65
|
+
def convert_bytes (bytes)
|
66
|
+
if bytes < 1024
|
67
|
+
sprintf("%6dB", bytes)
|
68
|
+
elsif bytes < 1024 * 1000 # 1000kb
|
69
|
+
sprintf("%5.1fKB", bytes.to_f / 1024)
|
70
|
+
elsif bytes < 1024 * 1024 * 1000 # 1000mb
|
71
|
+
sprintf("%5.1fMB", bytes.to_f / 1024 / 1024)
|
72
|
+
else
|
73
|
+
sprintf("%5.1fGB", bytes.to_f / 1024 / 1024 / 1024)
|
74
|
+
end
|
75
|
+
end
|
76
|
+
|
77
|
+
def transfer_rate
|
78
|
+
bytes_per_second = @current.to_f / (Time.now - @start_time)
|
79
|
+
sprintf("%s/s", convert_bytes(bytes_per_second))
|
80
|
+
end
|
81
|
+
|
82
|
+
def bytes
|
83
|
+
convert_bytes(@current)
|
84
|
+
end
|
85
|
+
|
86
|
+
def format_time (t)
|
87
|
+
t = t.to_i
|
88
|
+
sec = t % 60
|
89
|
+
min = (t / 60) % 60
|
90
|
+
hour = t / 3600
|
91
|
+
sprintf("%02d:%02d:%02d", hour, min, sec);
|
92
|
+
end
|
93
|
+
|
94
|
+
# ETA stands for Estimated Time of Arrival.
|
95
|
+
def eta
|
96
|
+
if @current == 0
|
97
|
+
"ETA: --:--:--"
|
98
|
+
else
|
99
|
+
elapsed = Time.now - @start_time
|
100
|
+
eta = elapsed * @total / @current - elapsed;
|
101
|
+
sprintf("ETA: %s", format_time(eta))
|
102
|
+
end
|
103
|
+
end
|
104
|
+
|
105
|
+
def elapsed
|
106
|
+
elapsed = Time.now - @start_time
|
107
|
+
sprintf("Time: %s", format_time(elapsed))
|
108
|
+
end
|
109
|
+
|
110
|
+
def eol
|
111
|
+
if @finished_p then "\n" else "\r" end
|
112
|
+
end
|
113
|
+
|
114
|
+
def do_percentage
|
115
|
+
if @total.zero?
|
116
|
+
100
|
117
|
+
else
|
118
|
+
@current * 100 / @total
|
119
|
+
end
|
120
|
+
end
|
121
|
+
|
122
|
+
def get_width
|
123
|
+
# FIXME: I don't know how portable it is.
|
124
|
+
default_width = 80
|
125
|
+
begin
|
126
|
+
tiocgwinsz = 0x5413
|
127
|
+
data = [0, 0, 0, 0].pack("SSSS")
|
128
|
+
if @out.ioctl(tiocgwinsz, data) >= 0 then
|
129
|
+
rows, cols, xpixels, ypixels = data.unpack("SSSS")
|
130
|
+
if cols >= 0 then cols else default_width end
|
131
|
+
else
|
132
|
+
default_width
|
133
|
+
end
|
134
|
+
rescue Exception
|
135
|
+
default_width
|
136
|
+
end
|
137
|
+
end
|
138
|
+
|
139
|
+
def show
|
140
|
+
arguments = @format_arguments.map {|method|
|
141
|
+
method = sprintf("fmt_%s", method)
|
142
|
+
send(method)
|
143
|
+
}
|
144
|
+
line = sprintf(@format, *arguments)
|
145
|
+
|
146
|
+
width = get_width
|
147
|
+
if line.length == width - 1
|
148
|
+
@out.print(line + eol)
|
149
|
+
@out.flush
|
150
|
+
elsif line.length >= width
|
151
|
+
@terminal_width = [@terminal_width - (line.length - width + 1), 0].max
|
152
|
+
if @terminal_width == 0 then @out.print(line + eol) else show end
|
153
|
+
else # line.length < width - 1
|
154
|
+
@terminal_width += width - line.length + 1
|
155
|
+
show
|
156
|
+
end
|
157
|
+
@previous_time = Time.now
|
158
|
+
end
|
159
|
+
|
160
|
+
def show_if_needed
|
161
|
+
if @total.zero?
|
162
|
+
cur_percentage = 100
|
163
|
+
prev_percentage = 0
|
164
|
+
else
|
165
|
+
cur_percentage = (@current * 100 / @total).to_i
|
166
|
+
prev_percentage = (@previous * 100 / @total).to_i
|
167
|
+
end
|
168
|
+
|
169
|
+
# Use "!=" instead of ">" to support negative changes
|
170
|
+
if cur_percentage != prev_percentage ||
|
171
|
+
Time.now - @previous_time >= 1 || @finished_p
|
172
|
+
show
|
173
|
+
end
|
174
|
+
end
|
175
|
+
|
176
|
+
public
|
177
|
+
def clear
|
178
|
+
@out.print "\r"
|
179
|
+
@out.print(" " * (get_width - 1))
|
180
|
+
@out.print "\r"
|
181
|
+
end
|
182
|
+
|
183
|
+
def finish
|
184
|
+
@current = @total
|
185
|
+
@finished_p = true
|
186
|
+
show
|
187
|
+
end
|
188
|
+
|
189
|
+
def finished?
|
190
|
+
@finished_p
|
191
|
+
end
|
192
|
+
|
193
|
+
def file_transfer_mode
|
194
|
+
@format_arguments = [:title, :percentage, :bar, :stat_for_file_transfer]
|
195
|
+
end
|
196
|
+
|
197
|
+
def format= (format)
|
198
|
+
@format = format
|
199
|
+
end
|
200
|
+
|
201
|
+
def format_arguments= (arguments)
|
202
|
+
@format_arguments = arguments
|
203
|
+
end
|
204
|
+
|
205
|
+
def halt
|
206
|
+
@finished_p = true
|
207
|
+
show
|
208
|
+
end
|
209
|
+
|
210
|
+
def inc (step = 1)
|
211
|
+
@current += step
|
212
|
+
@current = @total if @current > @total
|
213
|
+
show_if_needed
|
214
|
+
@previous = @current
|
215
|
+
end
|
216
|
+
|
217
|
+
def set (count)
|
218
|
+
if count < 0 || count > @total
|
219
|
+
raise "invalid count: #{count} (total: #{@total})"
|
220
|
+
end
|
221
|
+
@current = count
|
222
|
+
show_if_needed
|
223
|
+
@previous = @current
|
224
|
+
end
|
225
|
+
|
226
|
+
def inspect
|
227
|
+
"#<ProgressBar:#{@current}/#{@total}>"
|
228
|
+
end
|
229
|
+
end
|
230
|
+
|
231
|
+
class ReversedProgressBar < ProgressBar
|
232
|
+
def do_percentage
|
233
|
+
100 - super
|
234
|
+
end
|
235
|
+
end
|
236
|
+
|
data/lib/taps/schema.rb
ADDED
@@ -0,0 +1,82 @@
|
|
1
|
+
require 'active_record'
|
2
|
+
require 'active_support'
|
3
|
+
require 'stringio'
|
4
|
+
require 'uri'
|
5
|
+
|
6
|
+
module Taps
|
7
|
+
module Schema
|
8
|
+
extend self
|
9
|
+
|
10
|
+
def create_config(url)
|
11
|
+
uri = URI.parse(url)
|
12
|
+
adapter = uri.scheme
|
13
|
+
adapter = 'postgresql' if adapter == 'postgres'
|
14
|
+
adapter = 'sqlite3' if adapter == 'sqlite'
|
15
|
+
config = {
|
16
|
+
'adapter' => adapter,
|
17
|
+
'database' => uri.path.blank? ? uri.host : uri.path.split('/')[1],
|
18
|
+
'username' => uri.user,
|
19
|
+
'password' => uri.password,
|
20
|
+
'host' => uri.host,
|
21
|
+
}
|
22
|
+
end
|
23
|
+
|
24
|
+
def connection(database_url)
|
25
|
+
config = create_config(database_url)
|
26
|
+
ActiveRecord::Base.establish_connection(config)
|
27
|
+
end
|
28
|
+
|
29
|
+
def dump(database_url)
|
30
|
+
connection(database_url)
|
31
|
+
|
32
|
+
stream = StringIO.new
|
33
|
+
ActiveRecord::SchemaDumper.ignore_tables = []
|
34
|
+
ActiveRecord::SchemaDumper.dump(ActiveRecord::Base.connection, stream)
|
35
|
+
stream.string
|
36
|
+
end
|
37
|
+
|
38
|
+
def dump_without_indexes(database_url)
|
39
|
+
schema = dump(database_url)
|
40
|
+
schema.split("\n").collect do |line|
|
41
|
+
if line =~ /^\s+add_index/
|
42
|
+
line = "##{line}"
|
43
|
+
end
|
44
|
+
line
|
45
|
+
end.join("\n")
|
46
|
+
end
|
47
|
+
|
48
|
+
def indexes(database_url)
|
49
|
+
schema = dump(database_url)
|
50
|
+
schema.split("\n").collect do |line|
|
51
|
+
line if line =~ /^\s+add_index/
|
52
|
+
end.uniq.join("\n")
|
53
|
+
end
|
54
|
+
|
55
|
+
def load(database_url, schema)
|
56
|
+
connection(database_url)
|
57
|
+
eval(schema)
|
58
|
+
ActiveRecord::Base.connection.execute("DELETE FROM schema_migrations") rescue nil
|
59
|
+
end
|
60
|
+
|
61
|
+
def load_indexes(database_url, indexes)
|
62
|
+
connection(database_url)
|
63
|
+
|
64
|
+
schema =<<EORUBY
|
65
|
+
ActiveRecord::Schema.define do
|
66
|
+
#{indexes}
|
67
|
+
end
|
68
|
+
EORUBY
|
69
|
+
eval(schema)
|
70
|
+
end
|
71
|
+
|
72
|
+
def reset_db_sequences(database_url)
|
73
|
+
connection(database_url)
|
74
|
+
|
75
|
+
if ActiveRecord::Base.connection.respond_to?(:reset_pk_sequence!)
|
76
|
+
ActiveRecord::Base.connection.tables.each do |table|
|
77
|
+
ActiveRecord::Base.connection.reset_pk_sequence!(table)
|
78
|
+
end
|
79
|
+
end
|
80
|
+
end
|
81
|
+
end
|
82
|
+
end
|
data/lib/taps/server.rb
ADDED
@@ -0,0 +1,141 @@
|
|
1
|
+
require 'sinatra/base'
|
2
|
+
require File.dirname(__FILE__) + '/config'
|
3
|
+
require File.dirname(__FILE__) + '/utils'
|
4
|
+
require File.dirname(__FILE__) + '/db_session'
|
5
|
+
|
6
|
+
module Taps
|
7
|
+
class Server < Sinatra::Base
|
8
|
+
use Rack::Auth::Basic do |login, password|
|
9
|
+
login == Taps::Config.login && password == Taps::Config.password
|
10
|
+
end
|
11
|
+
|
12
|
+
error do
|
13
|
+
e = request.env['sinatra.error']
|
14
|
+
puts e.to_s
|
15
|
+
puts e.backtrace.join("\n")
|
16
|
+
"Application error"
|
17
|
+
end
|
18
|
+
|
19
|
+
before do
|
20
|
+
unless request.env['HTTP_TAPS_VERSION'] == Taps::VERSION
|
21
|
+
halt 417, "Taps version #{Taps::VERSION} is required for this server"
|
22
|
+
end
|
23
|
+
end
|
24
|
+
|
25
|
+
get '/' do
|
26
|
+
"hello"
|
27
|
+
end
|
28
|
+
|
29
|
+
post '/sessions' do
|
30
|
+
key = rand(9999999999).to_s
|
31
|
+
database_url = Taps::Config.database_url || request.body.string
|
32
|
+
|
33
|
+
DbSession.create(:key => key, :database_url => database_url, :started_at => Time.now, :last_access => Time.now)
|
34
|
+
|
35
|
+
"/sessions/#{key}"
|
36
|
+
end
|
37
|
+
|
38
|
+
post '/sessions/:key/tables/:table' do
|
39
|
+
session = DbSession.filter(:key => params[:key]).first
|
40
|
+
halt 404 unless session
|
41
|
+
|
42
|
+
gzip_data = request.body.read
|
43
|
+
halt 412 unless Taps::Utils.valid_data?(gzip_data, request.env['HTTP_TAPS_CHECKSUM'])
|
44
|
+
|
45
|
+
rows = Marshal.load(Taps::Utils.gunzip(gzip_data))
|
46
|
+
|
47
|
+
db = session.connection
|
48
|
+
table = db[params[:table].to_sym]
|
49
|
+
table.multi_insert(rows[:header], rows[:data])
|
50
|
+
|
51
|
+
"#{rows[:data].size}"
|
52
|
+
end
|
53
|
+
|
54
|
+
post '/sessions/:key/reset_sequences' do
|
55
|
+
session = DbSession.filter(:key => params[:key]).first
|
56
|
+
halt 404 unless session
|
57
|
+
|
58
|
+
schema_app = File.dirname(__FILE__) + '/../../bin/schema'
|
59
|
+
`#{schema_app} reset_db_sequences #{session.database_url}`
|
60
|
+
end
|
61
|
+
|
62
|
+
post '/sessions/:key/schema' do
|
63
|
+
session = DbSession.filter(:key => params[:key]).first
|
64
|
+
halt 404 unless session
|
65
|
+
|
66
|
+
schema_data = request.body.read
|
67
|
+
Taps::Utils.load_schema(session.database_url, schema_data)
|
68
|
+
end
|
69
|
+
|
70
|
+
post '/sessions/:key/indexes' do
|
71
|
+
session = DbSession.filter(:key => params[:key]).first
|
72
|
+
halt 404 unless session
|
73
|
+
|
74
|
+
index_data = request.body.read
|
75
|
+
Taps::Utils.load_indexes(session.database_url, index_data)
|
76
|
+
end
|
77
|
+
|
78
|
+
get '/sessions/:key/schema' do
|
79
|
+
session = DbSession.filter(:key => params[:key]).first
|
80
|
+
halt 404 unless session
|
81
|
+
|
82
|
+
schema_app = File.dirname(__FILE__) + '/../../bin/schema'
|
83
|
+
`#{schema_app} dump #{session.database_url}`
|
84
|
+
end
|
85
|
+
|
86
|
+
get '/sessions/:key/indexes' do
|
87
|
+
session = DbSession.filter(:key => params[:key]).first
|
88
|
+
halt 404 unless session
|
89
|
+
|
90
|
+
schema_app = File.dirname(__FILE__) + '/../../bin/schema'
|
91
|
+
`#{schema_app} indexes #{session.database_url}`
|
92
|
+
end
|
93
|
+
|
94
|
+
get '/sessions/:key/tables' do
|
95
|
+
session = DbSession.filter(:key => params[:key]).first
|
96
|
+
halt 404 unless session
|
97
|
+
|
98
|
+
db = session.connection
|
99
|
+
tables = db.tables
|
100
|
+
|
101
|
+
tables_with_counts = tables.inject({}) do |accum, table|
|
102
|
+
accum[table] = db[table].count
|
103
|
+
accum
|
104
|
+
end
|
105
|
+
|
106
|
+
Marshal.dump(tables_with_counts)
|
107
|
+
end
|
108
|
+
|
109
|
+
get '/sessions/:key/tables/:table/:chunk' do
|
110
|
+
session = DbSession.filter(:key => params[:key]).first
|
111
|
+
halt 404 unless session
|
112
|
+
|
113
|
+
chunk = params[:chunk].to_i
|
114
|
+
chunk = 500 if chunk < 1
|
115
|
+
|
116
|
+
offset = params[:offset].to_i
|
117
|
+
offset = 0 if offset < 0
|
118
|
+
|
119
|
+
db = session.connection
|
120
|
+
table = db[params[:table].to_sym]
|
121
|
+
columns = table.columns
|
122
|
+
order = columns.include?(:id) ? :id : columns.first
|
123
|
+
raw_data = Marshal.dump(Taps::Utils.format_data(table.order(order).limit(chunk, offset).all))
|
124
|
+
gzip_data = Taps::Utils.gzip(raw_data)
|
125
|
+
response['Taps-Checksum'] = Taps::Utils.checksum(gzip_data).to_s
|
126
|
+
response['Content-Type'] = "application/octet-stream"
|
127
|
+
gzip_data
|
128
|
+
end
|
129
|
+
|
130
|
+
delete '/sessions/:key' do
|
131
|
+
session = DbSession.filter(:key => params[:key]).first
|
132
|
+
halt 404 unless session
|
133
|
+
|
134
|
+
session.disconnect
|
135
|
+
session.destroy
|
136
|
+
|
137
|
+
"ok"
|
138
|
+
end
|
139
|
+
|
140
|
+
end
|
141
|
+
end
|
data/lib/taps/utils.rb
ADDED
@@ -0,0 +1,76 @@
|
|
1
|
+
require 'zlib'
|
2
|
+
require 'stringio'
|
3
|
+
require 'time'
|
4
|
+
require 'tempfile'
|
5
|
+
|
6
|
+
module Taps
|
7
|
+
module Utils
|
8
|
+
extend self
|
9
|
+
|
10
|
+
def checksum(data)
|
11
|
+
Zlib.crc32(data)
|
12
|
+
end
|
13
|
+
|
14
|
+
def valid_data?(data, crc32)
|
15
|
+
Zlib.crc32(data) == crc32.to_i
|
16
|
+
end
|
17
|
+
|
18
|
+
def gzip(data)
|
19
|
+
io = StringIO.new
|
20
|
+
gz = Zlib::GzipWriter.new(io)
|
21
|
+
gz.write data
|
22
|
+
gz.close
|
23
|
+
io.string
|
24
|
+
end
|
25
|
+
|
26
|
+
def gunzip(gzip_data)
|
27
|
+
io = StringIO.new(gzip_data)
|
28
|
+
gz = Zlib::GzipReader.new(io)
|
29
|
+
data = gz.read
|
30
|
+
gz.close
|
31
|
+
data
|
32
|
+
end
|
33
|
+
|
34
|
+
def format_data(data)
|
35
|
+
return {} if data.size == 0
|
36
|
+
header = data[0].keys
|
37
|
+
only_data = data.collect do |row|
|
38
|
+
header.collect { |h| row[h] }
|
39
|
+
end
|
40
|
+
{ :header => header, :data => only_data }
|
41
|
+
end
|
42
|
+
|
43
|
+
def calculate_chunksize(old_chunksize)
|
44
|
+
t1 = Time.now
|
45
|
+
yield
|
46
|
+
t2 = Time.now
|
47
|
+
|
48
|
+
diff = t2 - t1
|
49
|
+
new_chunksize = if diff > 3.0
|
50
|
+
(old_chunksize / 3).ceil
|
51
|
+
elsif diff > 1.1
|
52
|
+
old_chunksize - 100
|
53
|
+
elsif diff < 0.8
|
54
|
+
old_chunksize * 2
|
55
|
+
else
|
56
|
+
old_chunksize + 100
|
57
|
+
end
|
58
|
+
new_chunksize = 100 if new_chunksize < 100
|
59
|
+
new_chunksize
|
60
|
+
end
|
61
|
+
|
62
|
+
def load_schema(database_url, schema_data)
|
63
|
+
Tempfile.open('taps') do |tmp|
|
64
|
+
File.open(tmp.path, 'w') { |f| f.write(schema_data) }
|
65
|
+
`#{File.dirname(__FILE__)}/../../bin/schema load #{database_url} #{tmp.path}`
|
66
|
+
end
|
67
|
+
end
|
68
|
+
|
69
|
+
def load_indexes(database_url, index_data)
|
70
|
+
Tempfile.open('taps') do |tmp|
|
71
|
+
File.open(tmp.path, 'w') { |f| f.write(index_data) }
|
72
|
+
`#{File.dirname(__FILE__)}/../../bin/schema load_indexes #{database_url} #{tmp.path}`
|
73
|
+
end
|
74
|
+
end
|
75
|
+
end
|
76
|
+
end
|
data/spec/base.rb
ADDED
@@ -0,0 +1,21 @@
|
|
1
|
+
require 'rubygems'
|
2
|
+
require 'bacon'
|
3
|
+
require 'mocha'
|
4
|
+
|
5
|
+
class Bacon::Context
|
6
|
+
include Mocha::Standalone
|
7
|
+
|
8
|
+
alias_method :old_it, :it
|
9
|
+
def it(description)
|
10
|
+
old_it(description) do
|
11
|
+
mocha_setup
|
12
|
+
yield
|
13
|
+
mocha_verify
|
14
|
+
mocha_teardown
|
15
|
+
end
|
16
|
+
end
|
17
|
+
end
|
18
|
+
|
19
|
+
require File.dirname(__FILE__) + '/../lib/taps/config'
|
20
|
+
Taps::Config.taps_database_url = 'sqlite://test.db'
|
21
|
+
Sequel.connect(Taps::Config.taps_database_url)
|
@@ -0,0 +1,82 @@
|
|
1
|
+
require File.dirname(__FILE__) + '/base'
|
2
|
+
require File.dirname(__FILE__) + '/../lib/taps/client_session'
|
3
|
+
|
4
|
+
describe Taps::ClientSession do
|
5
|
+
before do
|
6
|
+
@client = Taps::ClientSession.new('sqlite://my.db', 'http://example.com:3000', 1000)
|
7
|
+
@client.stubs(:session_resource).returns(mock('session resource'))
|
8
|
+
end
|
9
|
+
|
10
|
+
it "starts a session and yields the session object to the block" do
|
11
|
+
Taps::ClientSession.start('x', 'y', 1000) do |session|
|
12
|
+
session.database_url.should == 'x'
|
13
|
+
session.remote_url.should == 'y'
|
14
|
+
session.default_chunksize.should == 1000
|
15
|
+
end
|
16
|
+
end
|
17
|
+
|
18
|
+
it "opens the local db connection via sequel and the database url" do
|
19
|
+
Sequel.expects(:connect).with('sqlite://my.db').returns(:con)
|
20
|
+
@client.db.should == :con
|
21
|
+
end
|
22
|
+
|
23
|
+
it "creates a restclient resource to the remote server" do
|
24
|
+
@client.server.url.should == 'http://example.com:3000'
|
25
|
+
end
|
26
|
+
|
27
|
+
it "verifies the db version, receive the schema, data, indexes, then reset the sequences" do
|
28
|
+
@client.expects(:verify_server)
|
29
|
+
@client.expects(:cmd_receive_schema)
|
30
|
+
@client.expects(:cmd_receive_data)
|
31
|
+
@client.expects(:cmd_receive_indexes)
|
32
|
+
@client.expects(:cmd_reset_sequences)
|
33
|
+
@client.cmd_receive.should.be.nil
|
34
|
+
end
|
35
|
+
|
36
|
+
it "checks the version of the server by seeing if it has access" do
|
37
|
+
@client.stubs(:server).returns(mock('server'))
|
38
|
+
@request = mock('request')
|
39
|
+
@client.server.expects(:[]).with('/').returns(@request)
|
40
|
+
@request.expects(:get).with({:taps_version => Taps::VERSION})
|
41
|
+
|
42
|
+
lambda { @client.verify_server }.should.not.raise
|
43
|
+
end
|
44
|
+
|
45
|
+
it "receives data from a remote taps server" do
|
46
|
+
@client.stubs(:puts)
|
47
|
+
@progressbar = mock('progressbar')
|
48
|
+
ProgressBar.stubs(:new).with('mytable', 2).returns(@progressbar)
|
49
|
+
@progressbar.stubs(:inc)
|
50
|
+
@progressbar.stubs(:finish)
|
51
|
+
@mytable = mock('mytable')
|
52
|
+
@client.expects(:fetch_tables_info).returns([ { :mytable => 2 }, 2 ])
|
53
|
+
@client.stubs(:db).returns(mock('db'))
|
54
|
+
@client.db.stubs(:[]).with(:mytable).returns(@mytable)
|
55
|
+
@client.expects(:fetch_table_rows).with(:mytable, 1000, 0).returns([ 1000, { :header => [:x, :y], :data => [[1, 2], [3, 4]] } ])
|
56
|
+
@client.expects(:fetch_table_rows).with(:mytable, 1000, 2).returns([ 1000, { }])
|
57
|
+
@mytable.expects(:multi_insert).with([:x, :y], [[1, 2], [3, 4]])
|
58
|
+
|
59
|
+
lambda { @client.cmd_receive_data }.should.not.raise
|
60
|
+
end
|
61
|
+
|
62
|
+
it "fetches tables info from taps server" do
|
63
|
+
@marshal_data = Marshal.dump({ :mytable => 2 })
|
64
|
+
@client.session_resource.stubs(:[]).with('tables').returns(mock('tables'))
|
65
|
+
@client.session_resource['tables'].stubs(:get).with(:taps_version => Taps::VERSION).returns(@marshal_data)
|
66
|
+
@client.fetch_tables_info.should == [ { :mytable => 2 }, 2 ]
|
67
|
+
end
|
68
|
+
|
69
|
+
it "fetches table rows given a chunksize and offset from taps server" do
|
70
|
+
@data = { :header => [ :x, :y ], :data => [ [1, 2], [3, 4] ] }
|
71
|
+
@gzip_data = Taps::Utils.gzip(Marshal.dump(@data))
|
72
|
+
Taps::Utils.stubs(:calculate_chunksize).with(1000).yields.returns(1000)
|
73
|
+
|
74
|
+
@response = mock('response')
|
75
|
+
@client.session_resource.stubs(:[]).with('tables/mytable/1000?offset=0').returns(mock('table resource'))
|
76
|
+
@client.session_resource['tables/mytable/1000?offset=0'].expects(:get).with(:taps_version => Taps::VERSION).returns(@response)
|
77
|
+
@response.stubs(:to_s).returns(@gzip_data)
|
78
|
+
@response.stubs(:headers).returns({ :taps_checksum => Taps::Utils.checksum(@gzip_data) })
|
79
|
+
@client.fetch_table_rows('mytable', 1000, 0).should == [ 1000, { :header => [:x, :y], :data => [[1, 2], [3, 4]] } ]
|
80
|
+
end
|
81
|
+
end
|
82
|
+
|
data/spec/schema_spec.rb
ADDED
@@ -0,0 +1,38 @@
|
|
1
|
+
require File.dirname(__FILE__) + '/base'
|
2
|
+
require File.dirname(__FILE__) + '/../lib/taps/schema'
|
3
|
+
|
4
|
+
describe Taps::Schema do
|
5
|
+
before do
|
6
|
+
@connection = mock("AR connection")
|
7
|
+
ActiveRecord::Base.stubs(:connection).returns(@connection)
|
8
|
+
end
|
9
|
+
|
10
|
+
it "parses a database url and returns a config hash for activerecord" do
|
11
|
+
Taps::Schema.create_config("postgres://myuser:mypass@localhost/mydb").should == {
|
12
|
+
'adapter' => 'postgresql',
|
13
|
+
'database' => 'mydb',
|
14
|
+
'username' => 'myuser',
|
15
|
+
'password' => 'mypass',
|
16
|
+
'host' => 'localhost'
|
17
|
+
}
|
18
|
+
end
|
19
|
+
|
20
|
+
it "translates sqlite in the database url to sqlite3" do
|
21
|
+
Taps::Schema.create_config("sqlite://myuser:mypass@localhost/mydb")['adapter'].should == 'sqlite3'
|
22
|
+
end
|
23
|
+
|
24
|
+
it "connects activerecord to the database" do
|
25
|
+
Taps::Schema.expects(:create_config).with("postgres://myuser:mypass@localhost/mydb").returns("db config")
|
26
|
+
ActiveRecord::Base.expects(:establish_connection).with("db config").returns(true)
|
27
|
+
Taps::Schema.connection("postgres://myuser:mypass@localhost/mydb").should == true
|
28
|
+
end
|
29
|
+
|
30
|
+
it "resets the db tables' primary keys" do
|
31
|
+
Taps::Schema.stubs(:connection)
|
32
|
+
ActiveRecord::Base.connection.expects(:respond_to?).with(:reset_pk_sequence!).returns(true)
|
33
|
+
ActiveRecord::Base.connection.stubs(:tables).returns(['table1'])
|
34
|
+
ActiveRecord::Base.connection.expects(:reset_pk_sequence!).with('table1')
|
35
|
+
should.not.raise { Taps::Schema.reset_db_sequences("postgres://myuser:mypass@localhost/mydb") }
|
36
|
+
end
|
37
|
+
end
|
38
|
+
|
data/spec/server_spec.rb
ADDED
@@ -0,0 +1,33 @@
|
|
1
|
+
require File.dirname(__FILE__) + '/base'
|
2
|
+
require 'sinatra'
|
3
|
+
require 'sinatra/test/bacon'
|
4
|
+
|
5
|
+
require File.dirname(__FILE__) + '/../lib/taps/server'
|
6
|
+
|
7
|
+
require 'pp'
|
8
|
+
|
9
|
+
describe Taps::Server do
|
10
|
+
before do
|
11
|
+
Taps::Config.login = 'taps'
|
12
|
+
Taps::Config.password = 'tpass'
|
13
|
+
|
14
|
+
@app = Taps::Server
|
15
|
+
@auth_header = "Basic " + ["taps:tpass"].pack("m*")
|
16
|
+
end
|
17
|
+
|
18
|
+
it "asks for http basic authentication" do
|
19
|
+
get '/'
|
20
|
+
status.should == 401
|
21
|
+
end
|
22
|
+
|
23
|
+
it "verifies the client taps version" do
|
24
|
+
get('/', { }, { 'HTTP_AUTHORIZATION' => @auth_header, 'HTTP_TAPS_VERSION' => Taps::VERSION })
|
25
|
+
status.should == 200
|
26
|
+
end
|
27
|
+
|
28
|
+
it "yells loudly if the client taps version doesn't match" do
|
29
|
+
get('/', { }, { 'HTTP_AUTHORIZATION' => @auth_header, 'HTTP_TAPS_VERSION' => '0.0.1' })
|
30
|
+
status.should == 417
|
31
|
+
end
|
32
|
+
end
|
33
|
+
|
data/spec/utils_spec.rb
ADDED
@@ -0,0 +1,45 @@
|
|
1
|
+
require File.dirname(__FILE__) + '/base'
|
2
|
+
require File.dirname(__FILE__) + '/../lib/taps/utils'
|
3
|
+
|
4
|
+
describe Taps::Utils do
|
5
|
+
it "gunzips a string" do
|
6
|
+
@hello_world = "\037\213\b\000R\261\207I\000\003\313H\315\311\311W(\317/\312I\001\000\205\021J\r\v\000\000\000"
|
7
|
+
Taps::Utils.gunzip(@hello_world).should == "hello world"
|
8
|
+
end
|
9
|
+
|
10
|
+
it "gzips and gunzips a string and returns the same string" do
|
11
|
+
Taps::Utils.gunzip(Taps::Utils.gzip("hello world")).should == "hello world"
|
12
|
+
end
|
13
|
+
|
14
|
+
it "generates a checksum using crc32" do
|
15
|
+
Taps::Utils.checksum("hello world").should == Zlib.crc32("hello world")
|
16
|
+
end
|
17
|
+
|
18
|
+
it "formats a data hash into one hash that contains an array of headers and an array of array of data" do
|
19
|
+
Taps::Utils.format_data([ { :x => 1, :y => 1 }, { :x => 2, :y => 2 } ]).should == { :header => [ :x, :y ], :data => [ [1, 1], [2, 2] ] }
|
20
|
+
end
|
21
|
+
|
22
|
+
it "scales chunksize down slowly when the time delta of the block is just over a second" do
|
23
|
+
Time.stubs(:now).returns(10.0).returns(11.5)
|
24
|
+
Taps::Utils.calculate_chunksize(1000) { }.should == 900
|
25
|
+
end
|
26
|
+
|
27
|
+
it "scales chunksize down fast when the time delta of the block is over 3 seconds" do
|
28
|
+
Time.stubs(:now).returns(10.0).returns(15.0)
|
29
|
+
Taps::Utils.calculate_chunksize(3000) { }.should == 1000
|
30
|
+
end
|
31
|
+
|
32
|
+
it "scales up chunksize fast when the time delta of the block is under 0.8 seconds" do
|
33
|
+
Time.stubs(:now).returns(10.0).returns(10.7)
|
34
|
+
Taps::Utils.calculate_chunksize(1000) { }.should == 2000
|
35
|
+
end
|
36
|
+
|
37
|
+
it "scales up chunksize slow when the time delta of the block is between 0.8 and 1.1 seconds" do
|
38
|
+
Time.stubs(:now).returns(10.0).returns(10.8)
|
39
|
+
Taps::Utils.calculate_chunksize(1000) { }.should == 1100
|
40
|
+
|
41
|
+
Time.stubs(:now).returns(10.0).returns(11.1)
|
42
|
+
Taps::Utils.calculate_chunksize(1000) { }.should == 1100
|
43
|
+
end
|
44
|
+
end
|
45
|
+
|
metadata
ADDED
@@ -0,0 +1,118 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: ricardochimal-taps
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.2.0
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Ricardo Chimal, Jr.
|
8
|
+
- Adam Wiggins
|
9
|
+
autorequire:
|
10
|
+
bindir: bin
|
11
|
+
cert_chain: []
|
12
|
+
|
13
|
+
date: 2009-02-06 00:00:00 -08:00
|
14
|
+
default_executable:
|
15
|
+
dependencies:
|
16
|
+
- !ruby/object:Gem::Dependency
|
17
|
+
name: sinatra
|
18
|
+
version_requirement:
|
19
|
+
version_requirements: !ruby/object:Gem::Requirement
|
20
|
+
requirements:
|
21
|
+
- - ~>
|
22
|
+
- !ruby/object:Gem::Version
|
23
|
+
version: 0.9.0
|
24
|
+
version:
|
25
|
+
- !ruby/object:Gem::Dependency
|
26
|
+
name: activerecord
|
27
|
+
version_requirement:
|
28
|
+
version_requirements: !ruby/object:Gem::Requirement
|
29
|
+
requirements:
|
30
|
+
- - "="
|
31
|
+
- !ruby/object:Gem::Version
|
32
|
+
version: 2.2.2
|
33
|
+
version:
|
34
|
+
- !ruby/object:Gem::Dependency
|
35
|
+
name: thor
|
36
|
+
version_requirement:
|
37
|
+
version_requirements: !ruby/object:Gem::Requirement
|
38
|
+
requirements:
|
39
|
+
- - "="
|
40
|
+
- !ruby/object:Gem::Version
|
41
|
+
version: 0.9.9
|
42
|
+
version:
|
43
|
+
- !ruby/object:Gem::Dependency
|
44
|
+
name: rest-client
|
45
|
+
version_requirement:
|
46
|
+
version_requirements: !ruby/object:Gem::Requirement
|
47
|
+
requirements:
|
48
|
+
- - ~>
|
49
|
+
- !ruby/object:Gem::Version
|
50
|
+
version: 0.9.0
|
51
|
+
version:
|
52
|
+
- !ruby/object:Gem::Dependency
|
53
|
+
name: sequel
|
54
|
+
version_requirement:
|
55
|
+
version_requirements: !ruby/object:Gem::Requirement
|
56
|
+
requirements:
|
57
|
+
- - ~>
|
58
|
+
- !ruby/object:Gem::Version
|
59
|
+
version: 2.10.0
|
60
|
+
version:
|
61
|
+
description: A simple database agnostic import/export app to transfer data to/from a remote database.
|
62
|
+
email: ricardo@heroku.com
|
63
|
+
executables:
|
64
|
+
- taps
|
65
|
+
- schema
|
66
|
+
extensions: []
|
67
|
+
|
68
|
+
extra_rdoc_files: []
|
69
|
+
|
70
|
+
files:
|
71
|
+
- spec/base.rb
|
72
|
+
- spec/client_session_spec.rb
|
73
|
+
- spec/schema_spec.rb
|
74
|
+
- spec/server_spec.rb
|
75
|
+
- spec/utils_spec.rb
|
76
|
+
- lib/taps/config.rb
|
77
|
+
- lib/taps/db_session.rb
|
78
|
+
- lib/taps/progress_bar.rb
|
79
|
+
- lib/taps/server.rb
|
80
|
+
- lib/taps/utils.rb
|
81
|
+
- lib/taps/schema.rb
|
82
|
+
- lib/taps/client_session.rb
|
83
|
+
- lib/taps/cli.rb
|
84
|
+
- README.rdoc
|
85
|
+
- LICENSE
|
86
|
+
- VERSION.yml
|
87
|
+
- Rakefile
|
88
|
+
- bin/taps
|
89
|
+
- bin/schema
|
90
|
+
has_rdoc: true
|
91
|
+
homepage: http://github.com/ricardochimal/taps
|
92
|
+
post_install_message:
|
93
|
+
rdoc_options:
|
94
|
+
- --inline-source
|
95
|
+
- --charset=UTF-8
|
96
|
+
require_paths:
|
97
|
+
- lib
|
98
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
99
|
+
requirements:
|
100
|
+
- - ">="
|
101
|
+
- !ruby/object:Gem::Version
|
102
|
+
version: "0"
|
103
|
+
version:
|
104
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
105
|
+
requirements:
|
106
|
+
- - ">="
|
107
|
+
- !ruby/object:Gem::Version
|
108
|
+
version: "0"
|
109
|
+
version:
|
110
|
+
requirements: []
|
111
|
+
|
112
|
+
rubyforge_project: taps
|
113
|
+
rubygems_version: 1.2.0
|
114
|
+
signing_key:
|
115
|
+
specification_version: 2
|
116
|
+
summary: simple database import/export app
|
117
|
+
test_files: []
|
118
|
+
|