taps-jruby 0.3.14
Sign up to get free protection for your applications and to get access to all the features.
- data/LICENSE +20 -0
- data/README.rdoc +51 -0
- data/Rakefile +70 -0
- data/TODO +1 -0
- data/VERSION.yml +5 -0
- data/bin/schema +54 -0
- data/bin/schema.cmd +6 -0
- data/bin/taps +6 -0
- data/lib/taps/cli.rb +193 -0
- data/lib/taps/config.rb +47 -0
- data/lib/taps/data_stream.rb +333 -0
- data/lib/taps/db_session.rb +20 -0
- data/lib/taps/errors.rb +15 -0
- data/lib/taps/log.rb +15 -0
- data/lib/taps/monkey.rb +21 -0
- data/lib/taps/multipart.rb +73 -0
- data/lib/taps/operation.rb +557 -0
- data/lib/taps/progress_bar.rb +236 -0
- data/lib/taps/schema.rb +83 -0
- data/lib/taps/server.rb +188 -0
- data/lib/taps/utils.rb +199 -0
- data/spec/base.rb +26 -0
- data/spec/cli_spec.rb +10 -0
- data/spec/data_stream_spec.rb +23 -0
- data/spec/operation_spec.rb +32 -0
- data/spec/server_spec.rb +35 -0
- data/spec/utils_spec.rb +61 -0
- metadata +194 -0
data/LICENSE
ADDED
@@ -0,0 +1,20 @@
|
|
1
|
+
Copyright (c) 2008 Ricardo Chimal, Jr
|
2
|
+
|
3
|
+
Permission is hereby granted, free of charge, to any person obtaining
|
4
|
+
a copy of this software and associated documentation files (the
|
5
|
+
"Software"), to deal in the Software without restriction, including
|
6
|
+
without limitation the rights to use, copy, modify, merge, publish,
|
7
|
+
distribute, sublicense, and/or sell copies of the Software, and to
|
8
|
+
permit persons to whom the Software is furnished to do so, subject to
|
9
|
+
the following conditions:
|
10
|
+
|
11
|
+
The above copyright notice and this permission notice shall be
|
12
|
+
included in all copies or substantial portions of the Software.
|
13
|
+
|
14
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
15
|
+
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
16
|
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
17
|
+
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
18
|
+
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
19
|
+
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
20
|
+
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
data/README.rdoc
ADDED
@@ -0,0 +1,51 @@
|
|
1
|
+
= Taps -- simple database import/export app
|
2
|
+
|
3
|
+
A simple database agnostic import/export app to transfer data to/from a remote database.
|
4
|
+
|
5
|
+
== Usage: Server
|
6
|
+
|
7
|
+
Here's how you start a taps server
|
8
|
+
|
9
|
+
$ taps server postgres://localdbuser:localdbpass@localhost/dbname httpuser httppassword
|
10
|
+
|
11
|
+
You can also specify an encoding in the database url
|
12
|
+
|
13
|
+
$ taps server mysql://localdbuser:localdbpass@localhost/dbname?encoding=latin1 httpuser httppassword
|
14
|
+
|
15
|
+
== Usage: Client
|
16
|
+
|
17
|
+
When you want to pull down a database from a taps server
|
18
|
+
|
19
|
+
$ taps pull postgres://dbuser:dbpassword@localhost/dbname http://httpuser:httppassword@example.com:5000
|
20
|
+
|
21
|
+
or when you want to push a local database to a taps server
|
22
|
+
|
23
|
+
$ taps push postgres://dbuser:dbpassword@localhost/dbname http://httpuser:httppassword@example.com:5000
|
24
|
+
|
25
|
+
or when you want to transfer a list of tables
|
26
|
+
|
27
|
+
$ taps push postgres://dbuser:dbpassword@localhost/dbname http://httpuser:httppassword@example.com:5000 --tables logs,tags
|
28
|
+
|
29
|
+
or when you want to transfer tables that start with a word
|
30
|
+
|
31
|
+
$ taps push postgres://dbuser:dbpassword@localhost/dbname http://httpuser:httppassword@example.com:5000 --filter '^log_'
|
32
|
+
|
33
|
+
== Known Issues
|
34
|
+
|
35
|
+
* Foreign Keys get lost in the schema transfer
|
36
|
+
* Tables without primary keys will be incredibly slow to transfer. This is due to it being inefficient having large offset values in queries.
|
37
|
+
* Multiple schemas are currently not supported
|
38
|
+
|
39
|
+
== Meta
|
40
|
+
|
41
|
+
Maintained by Ricardo Chimal, Jr. (ricardo at heroku dot com)
|
42
|
+
|
43
|
+
Written by Ricardo Chimal, Jr. (ricardo at heroku dot com) and Adam Wiggins (adam at heroku dot com)
|
44
|
+
|
45
|
+
Early research and inspiration by Blake Mizerany
|
46
|
+
|
47
|
+
Released under the MIT License: http://www.opensource.org/licenses/mit-license.php
|
48
|
+
|
49
|
+
http://github.com/ricardochimal/taps
|
50
|
+
|
51
|
+
Special Thanks to Sequel for making this tool possible http://sequel.rubyforge.org/
|
data/Rakefile
ADDED
@@ -0,0 +1,70 @@
|
|
1
|
+
begin
|
2
|
+
require 'jeweler'
|
3
|
+
Jeweler::Tasks.new do |s|
|
4
|
+
s.name = "taps-jruby"
|
5
|
+
s.summary = %Q{simple database import/export app - jruby version}
|
6
|
+
s.email = "heittman.rob@gmail.com"
|
7
|
+
s.homepage = "http://github.com/rfc2616/taps-jruby"
|
8
|
+
s.description = "A simple database agnostic import/export app to transfer data to/from a remote database. (JRuby version)"
|
9
|
+
s.authors = ["Ricardo Chimal, Jr.", "Rob Heittman"]
|
10
|
+
|
11
|
+
s.rubygems_version = %q{1.3.5}
|
12
|
+
|
13
|
+
s.add_dependency 'json_pure', '>= 1.2.0', '< 1.5.0'
|
14
|
+
s.add_dependency 'sinatra', '~> 1.0.0'
|
15
|
+
s.add_dependency 'rest-client', '>= 1.4.0', '< 1.7.0'
|
16
|
+
s.add_dependency 'sequel', '~> 3.17.0'
|
17
|
+
s.add_dependency 'activerecord-jdbcsqlite3-adapter', '>= 1.0.2'
|
18
|
+
s.add_dependency 'rack', '>= 1.0.1'
|
19
|
+
|
20
|
+
s.files = FileList['spec/*.rb'] + FileList['lib/**/*.rb'] + ['README.rdoc', 'LICENSE', 'VERSION.yml', 'Rakefile'] + FileList['bin/*']
|
21
|
+
s.executables = ['taps', 'schema']
|
22
|
+
end
|
23
|
+
rescue LoadError => e
|
24
|
+
if e.message =~ /jeweler/
|
25
|
+
puts "Jeweler not available. Install it with: sudo gem install technicalpickles-jeweler -s http://gems.github.com"
|
26
|
+
else
|
27
|
+
puts e.message + ' -- while loading jeweler.'
|
28
|
+
end
|
29
|
+
end
|
30
|
+
|
31
|
+
begin
|
32
|
+
require 'rake/rdoctask'
|
33
|
+
Rake::RDocTask.new do |rdoc|
|
34
|
+
rdoc.rdoc_dir = 'rdoc'
|
35
|
+
rdoc.title = 'taps'
|
36
|
+
rdoc.options << '--line-numbers' << '--inline-source'
|
37
|
+
rdoc.rdoc_files.include('README*')
|
38
|
+
rdoc.rdoc_files.include('lib/**/*.rb')
|
39
|
+
end
|
40
|
+
rescue LoadError
|
41
|
+
puts "Rdoc is not available"
|
42
|
+
end
|
43
|
+
|
44
|
+
begin
|
45
|
+
require 'rcov/rcovtask'
|
46
|
+
Rcov::RcovTask.new do |t|
|
47
|
+
t.libs << 'spec'
|
48
|
+
t.test_files = FileList['spec/*_spec.rb']
|
49
|
+
t.verbose = true
|
50
|
+
end
|
51
|
+
rescue LoadError
|
52
|
+
puts "RCov is not available. In order to run rcov, you must: sudo gem install rcov"
|
53
|
+
end
|
54
|
+
|
55
|
+
desc "Run all specs; requires the bacon gem"
|
56
|
+
task :spec do
|
57
|
+
if `which bacon`.empty?
|
58
|
+
puts "bacon is not available. In order to run the specs, you must: sudo gem install bacon."
|
59
|
+
else
|
60
|
+
system "bacon #{File.dirname(__FILE__)}/spec/*_spec.rb"
|
61
|
+
end
|
62
|
+
end
|
63
|
+
|
64
|
+
desc "copy/paste env vars for dev testing"
|
65
|
+
task :env do
|
66
|
+
puts "export RUBYLIB='#{File.dirname(__FILE__) + '/lib'}'"
|
67
|
+
puts "export RUBYOPT='-rrubygems'"
|
68
|
+
end
|
69
|
+
|
70
|
+
task :default => :spec
|
data/VERSION.yml
ADDED
data/bin/schema
ADDED
@@ -0,0 +1,54 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
require 'rubygems'
|
4
|
+
gem 'sequel', '~> 3.17.0'
|
5
|
+
|
6
|
+
$:.unshift File.dirname(__FILE__) + '/../lib'
|
7
|
+
|
8
|
+
require 'taps/schema'
|
9
|
+
|
10
|
+
cmd = ARGV.shift.strip rescue ''
|
11
|
+
database_url = ARGV.shift.strip rescue ''
|
12
|
+
|
13
|
+
def show_usage_and_exit
|
14
|
+
puts <<EOTXT
|
15
|
+
schema console <database_url>
|
16
|
+
schema dump <database_url>
|
17
|
+
schema dump_table <database_url> <table>
|
18
|
+
schema indexes <database_url>
|
19
|
+
schema indexes_individual <database_url>
|
20
|
+
schema reset_db_sequences <database_url>
|
21
|
+
schema load <database_url> <schema_file>
|
22
|
+
schema load_indexes <database_url> <indexes_file>
|
23
|
+
EOTXT
|
24
|
+
exit(1)
|
25
|
+
end
|
26
|
+
|
27
|
+
case cmd
|
28
|
+
when 'dump'
|
29
|
+
puts Taps::Schema.dump(database_url)
|
30
|
+
when 'dump_table'
|
31
|
+
table = ARGV.shift.strip
|
32
|
+
puts Taps::Schema.dump_table(database_url, table)
|
33
|
+
when 'indexes'
|
34
|
+
puts Taps::Schema.indexes(database_url)
|
35
|
+
when 'indexes_individual'
|
36
|
+
puts Taps::Schema.indexes_individual(database_url)
|
37
|
+
when 'load_indexes'
|
38
|
+
filename = ARGV.shift.strip rescue ''
|
39
|
+
indexes = File.read(filename) rescue show_usage_and_exit
|
40
|
+
Taps::Schema.load_indexes(database_url, indexes)
|
41
|
+
when 'load'
|
42
|
+
filename = ARGV.shift.strip rescue ''
|
43
|
+
schema = File.read(filename) rescue show_usage_and_exit
|
44
|
+
Taps::Schema.load(database_url, schema)
|
45
|
+
when 'reset_db_sequences'
|
46
|
+
Taps::Schema.reset_db_sequences(database_url)
|
47
|
+
when 'console'
|
48
|
+
$db = Sequel.connect(database_url)
|
49
|
+
require 'irb'
|
50
|
+
require 'irb/completion'
|
51
|
+
IRB.start
|
52
|
+
else
|
53
|
+
show_usage_and_exit
|
54
|
+
end
|
data/bin/schema.cmd
ADDED
data/bin/taps
ADDED
data/lib/taps/cli.rb
ADDED
@@ -0,0 +1,193 @@
|
|
1
|
+
require 'optparse'
|
2
|
+
require 'tempfile'
|
3
|
+
require 'json/pure'
|
4
|
+
require 'taps/monkey'
|
5
|
+
require 'taps/config'
|
6
|
+
require 'taps/log'
|
7
|
+
|
8
|
+
Taps::Config.taps_database_url = ENV['TAPS_DATABASE_URL'] || begin
|
9
|
+
# this is dirty but it solves a weird problem where the tempfile disappears mid-process
|
10
|
+
$__taps_database = Tempfile.new('taps.db')
|
11
|
+
$__taps_database.open()
|
12
|
+
"jdbc:sqlite://#{$__taps_database.path}"
|
13
|
+
end
|
14
|
+
|
15
|
+
module Taps
|
16
|
+
class Cli
|
17
|
+
attr_accessor :argv
|
18
|
+
|
19
|
+
def initialize(argv)
|
20
|
+
@argv = argv
|
21
|
+
end
|
22
|
+
|
23
|
+
def run
|
24
|
+
method = (argv.shift || 'help').to_sym
|
25
|
+
if [:pull, :push, :server, :version].include? method
|
26
|
+
send(method)
|
27
|
+
else
|
28
|
+
help
|
29
|
+
end
|
30
|
+
end
|
31
|
+
|
32
|
+
def pull
|
33
|
+
opts = clientoptparse(:pull)
|
34
|
+
Taps.log.level = Logger::DEBUG if opts[:debug]
|
35
|
+
if opts[:resume_filename]
|
36
|
+
clientresumexfer(:pull, opts)
|
37
|
+
else
|
38
|
+
clientxfer(:pull, opts)
|
39
|
+
end
|
40
|
+
end
|
41
|
+
|
42
|
+
def push
|
43
|
+
opts = clientoptparse(:push)
|
44
|
+
Taps.log.level = Logger::DEBUG if opts[:debug]
|
45
|
+
if opts[:resume_filename]
|
46
|
+
clientresumexfer(:push, opts)
|
47
|
+
else
|
48
|
+
clientxfer(:push, opts)
|
49
|
+
end
|
50
|
+
end
|
51
|
+
|
52
|
+
def server
|
53
|
+
opts = serveroptparse
|
54
|
+
Taps.log.level = Logger::DEBUG if opts[:debug]
|
55
|
+
Taps::Config.database_url = opts[:database_url]
|
56
|
+
Taps::Config.login = opts[:login]
|
57
|
+
Taps::Config.password = opts[:password]
|
58
|
+
|
59
|
+
Taps::Config.verify_database_url
|
60
|
+
require 'taps/server'
|
61
|
+
Taps::Server.run!({
|
62
|
+
:port => opts[:port],
|
63
|
+
:environment => :production,
|
64
|
+
:logging => true,
|
65
|
+
:dump_errors => true,
|
66
|
+
})
|
67
|
+
end
|
68
|
+
|
69
|
+
def version
|
70
|
+
puts Taps.version
|
71
|
+
end
|
72
|
+
|
73
|
+
def help
|
74
|
+
puts <<EOHELP
|
75
|
+
Options
|
76
|
+
=======
|
77
|
+
server Start a taps database import/export server
|
78
|
+
pull Pull a database from a taps server
|
79
|
+
push Push a database to a taps server
|
80
|
+
version Taps version
|
81
|
+
|
82
|
+
Add '-h' to any command to see their usage
|
83
|
+
EOHELP
|
84
|
+
end
|
85
|
+
|
86
|
+
def serveroptparse
|
87
|
+
opts={:port => 5000, :database_url => nil, :login => nil, :password => nil, :debug => false}
|
88
|
+
OptionParser.new do |o|
|
89
|
+
o.banner = "Usage: #{File.basename($0)} server [OPTIONS] <local_database_url> <login> <password>"
|
90
|
+
o.define_head "Start a taps database import/export server"
|
91
|
+
|
92
|
+
o.on("-p", "--port=N", "Server Port") { |v| opts[:port] = v.to_i if v.to_i > 0 }
|
93
|
+
o.on("-d", "--debug", "Enable Debug Messages") { |v| opts[:debug] = true }
|
94
|
+
o.parse!(argv)
|
95
|
+
|
96
|
+
opts[:database_url] = argv.shift
|
97
|
+
opts[:login] = argv.shift
|
98
|
+
opts[:password] = argv.shift
|
99
|
+
|
100
|
+
if opts[:database_url].nil?
|
101
|
+
$stderr.puts "Missing Database URL"
|
102
|
+
puts o
|
103
|
+
exit 1
|
104
|
+
end
|
105
|
+
if opts[:login].nil?
|
106
|
+
$stderr.puts "Missing Login"
|
107
|
+
puts o
|
108
|
+
exit 1
|
109
|
+
end
|
110
|
+
if opts[:password].nil?
|
111
|
+
$stderr.puts "Missing Password"
|
112
|
+
puts o
|
113
|
+
exit 1
|
114
|
+
end
|
115
|
+
end
|
116
|
+
opts
|
117
|
+
end
|
118
|
+
|
119
|
+
def clientoptparse(cmd)
|
120
|
+
opts={:default_chunksize => 1000, :database_url => nil, :remote_url => nil, :debug => false, :resume_filename => nil, :disable_compresion => false, :indexes_first => false}
|
121
|
+
OptionParser.new do |o|
|
122
|
+
o.banner = "Usage: #{File.basename($0)} #{cmd} [OPTIONS] <local_database_url> <remote_url>"
|
123
|
+
|
124
|
+
case cmd
|
125
|
+
when :pull
|
126
|
+
o.define_head "Pull a database from a taps server"
|
127
|
+
when :push
|
128
|
+
o.define_head "Push a database to a taps server"
|
129
|
+
end
|
130
|
+
|
131
|
+
o.on("-i", "--indexes-first", "Transfer indexes first before data") { |v| opts[:indexes_first] = true }
|
132
|
+
o.on("-r", "--resume=file", "Resume a Taps Session from a stored file") { |v| opts[:resume_filename] = v }
|
133
|
+
o.on("-c", "--chunksize=N", "Initial Chunksize") { |v| opts[:default_chunksize] = (v.to_i < 10 ? 10 : v.to_i) }
|
134
|
+
o.on("-g", "--disable-compression", "Disable Compression") { |v| opts[:disable_compression] = true }
|
135
|
+
o.on("-f", "--filter=regex", "Regex Filter for tables") { |v| opts[:table_filter] = v }
|
136
|
+
o.on("-t", "--tables=A,B,C", Array, "Shortcut to filter on a list of tables") do |v|
|
137
|
+
r_tables = v.collect { |t| "^#{t}$" }.join("|")
|
138
|
+
opts[:table_filter] = "(#{r_tables})"
|
139
|
+
end
|
140
|
+
o.on("-d", "--debug", "Enable Debug Messages") { |v| opts[:debug] = true }
|
141
|
+
o.parse!(argv)
|
142
|
+
|
143
|
+
opts[:database_url] = argv.shift
|
144
|
+
opts[:remote_url] = argv.shift
|
145
|
+
|
146
|
+
if opts[:database_url].nil?
|
147
|
+
$stderr.puts "Missing Database URL"
|
148
|
+
puts o
|
149
|
+
exit 1
|
150
|
+
end
|
151
|
+
if opts[:remote_url].nil?
|
152
|
+
$stderr.puts "Missing Remote Taps URL"
|
153
|
+
puts o
|
154
|
+
exit 1
|
155
|
+
end
|
156
|
+
end
|
157
|
+
|
158
|
+
opts
|
159
|
+
end
|
160
|
+
|
161
|
+
def clientxfer(method, opts)
|
162
|
+
database_url = opts.delete(:database_url)
|
163
|
+
remote_url = opts.delete(:remote_url)
|
164
|
+
|
165
|
+
Taps::Config.verify_database_url(database_url)
|
166
|
+
|
167
|
+
require 'taps/operation'
|
168
|
+
|
169
|
+
Taps::Operation.factory(method, database_url, remote_url, opts).run
|
170
|
+
end
|
171
|
+
|
172
|
+
def clientresumexfer(method, opts)
|
173
|
+
session = JSON.parse(File.read(opts.delete(:resume_filename)))
|
174
|
+
session.symbolize_recursively!
|
175
|
+
|
176
|
+
database_url = opts.delete(:database_url)
|
177
|
+
remote_url = opts.delete(:remote_url) || session.delete(:remote_url)
|
178
|
+
|
179
|
+
Taps::Config.verify_database_url(database_url)
|
180
|
+
|
181
|
+
require 'taps/operation'
|
182
|
+
|
183
|
+
newsession = session.merge({
|
184
|
+
:default_chunksize => opts[:default_chunksize],
|
185
|
+
:disable_compression => opts[:disable_compression],
|
186
|
+
:resume => true,
|
187
|
+
})
|
188
|
+
|
189
|
+
Taps::Operation.factory(method, database_url, remote_url, newsession).run
|
190
|
+
end
|
191
|
+
|
192
|
+
end
|
193
|
+
end
|
data/lib/taps/config.rb
ADDED
@@ -0,0 +1,47 @@
|
|
1
|
+
require 'sequel'
|
2
|
+
#require 'sqlite3'
|
3
|
+
require 'yaml'
|
4
|
+
|
5
|
+
Sequel.datetime_class = DateTime
|
6
|
+
|
7
|
+
module Taps
|
8
|
+
def self.version_yml
|
9
|
+
@@version_yml ||= YAML.load(File.read(File.dirname(__FILE__) + '/../../VERSION.yml'))
|
10
|
+
end
|
11
|
+
|
12
|
+
def self.version
|
13
|
+
version = "#{version_yml[:major]}.#{version_yml[:minor]}.#{version_yml[:patch]}"
|
14
|
+
version += ".#{version_yml[:build]}" if version_yml[:build]
|
15
|
+
version
|
16
|
+
end
|
17
|
+
|
18
|
+
def self.compatible_version
|
19
|
+
"#{version_yml[:major]}.#{version_yml[:minor]}"
|
20
|
+
end
|
21
|
+
|
22
|
+
def self.exiting=(val)
|
23
|
+
@@exiting = val
|
24
|
+
end
|
25
|
+
|
26
|
+
def exiting?
|
27
|
+
(@@exiting ||= false) == true
|
28
|
+
end
|
29
|
+
|
30
|
+
class Config
|
31
|
+
class << self
|
32
|
+
attr_accessor :taps_database_url
|
33
|
+
attr_accessor :login, :password, :database_url, :remote_url
|
34
|
+
attr_accessor :chunksize
|
35
|
+
|
36
|
+
def verify_database_url(db_url=nil)
|
37
|
+
db_url ||= self.database_url
|
38
|
+
db = Sequel.connect(db_url)
|
39
|
+
db.tables
|
40
|
+
db.disconnect
|
41
|
+
rescue Object => e
|
42
|
+
puts "Failed to connect to database:\n #{e.class} -> #{e}"
|
43
|
+
exit 1
|
44
|
+
end
|
45
|
+
end
|
46
|
+
end
|
47
|
+
end
|