taps-jruby 0.3.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/LICENSE +20 -0
- data/README.rdoc +51 -0
- data/Rakefile +70 -0
- data/TODO +1 -0
- data/VERSION.yml +5 -0
- data/bin/schema +54 -0
- data/bin/schema.cmd +6 -0
- data/bin/taps +6 -0
- data/lib/taps/cli.rb +193 -0
- data/lib/taps/config.rb +47 -0
- data/lib/taps/data_stream.rb +333 -0
- data/lib/taps/db_session.rb +20 -0
- data/lib/taps/errors.rb +15 -0
- data/lib/taps/log.rb +15 -0
- data/lib/taps/monkey.rb +21 -0
- data/lib/taps/multipart.rb +73 -0
- data/lib/taps/operation.rb +557 -0
- data/lib/taps/progress_bar.rb +236 -0
- data/lib/taps/schema.rb +83 -0
- data/lib/taps/server.rb +188 -0
- data/lib/taps/utils.rb +199 -0
- data/spec/base.rb +26 -0
- data/spec/cli_spec.rb +10 -0
- data/spec/data_stream_spec.rb +23 -0
- data/spec/operation_spec.rb +32 -0
- data/spec/server_spec.rb +35 -0
- data/spec/utils_spec.rb +61 -0
- metadata +194 -0
data/LICENSE
ADDED
@@ -0,0 +1,20 @@
|
|
1
|
+
Copyright (c) 2008 Ricardo Chimal, Jr
|
2
|
+
|
3
|
+
Permission is hereby granted, free of charge, to any person obtaining
|
4
|
+
a copy of this software and associated documentation files (the
|
5
|
+
"Software"), to deal in the Software without restriction, including
|
6
|
+
without limitation the rights to use, copy, modify, merge, publish,
|
7
|
+
distribute, sublicense, and/or sell copies of the Software, and to
|
8
|
+
permit persons to whom the Software is furnished to do so, subject to
|
9
|
+
the following conditions:
|
10
|
+
|
11
|
+
The above copyright notice and this permission notice shall be
|
12
|
+
included in all copies or substantial portions of the Software.
|
13
|
+
|
14
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
15
|
+
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
16
|
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
17
|
+
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
18
|
+
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
19
|
+
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
20
|
+
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
data/README.rdoc
ADDED
@@ -0,0 +1,51 @@
|
|
1
|
+
= Taps -- simple database import/export app
|
2
|
+
|
3
|
+
A simple database agnostic import/export app to transfer data to/from a remote database.
|
4
|
+
|
5
|
+
== Usage: Server
|
6
|
+
|
7
|
+
Here's how you start a taps server
|
8
|
+
|
9
|
+
$ taps server postgres://localdbuser:localdbpass@localhost/dbname httpuser httppassword
|
10
|
+
|
11
|
+
You can also specify an encoding in the database url
|
12
|
+
|
13
|
+
$ taps server mysql://localdbuser:localdbpass@localhost/dbname?encoding=latin1 httpuser httppassword
|
14
|
+
|
15
|
+
== Usage: Client
|
16
|
+
|
17
|
+
When you want to pull down a database from a taps server
|
18
|
+
|
19
|
+
$ taps pull postgres://dbuser:dbpassword@localhost/dbname http://httpuser:httppassword@example.com:5000
|
20
|
+
|
21
|
+
or when you want to push a local database to a taps server
|
22
|
+
|
23
|
+
$ taps push postgres://dbuser:dbpassword@localhost/dbname http://httpuser:httppassword@example.com:5000
|
24
|
+
|
25
|
+
or when you want to transfer a list of tables
|
26
|
+
|
27
|
+
$ taps push postgres://dbuser:dbpassword@localhost/dbname http://httpuser:httppassword@example.com:5000 --tables logs,tags
|
28
|
+
|
29
|
+
or when you want to transfer tables that start with a word
|
30
|
+
|
31
|
+
$ taps push postgres://dbuser:dbpassword@localhost/dbname http://httpuser:httppassword@example.com:5000 --filter '^log_'
|
32
|
+
|
33
|
+
== Known Issues
|
34
|
+
|
35
|
+
* Foreign Keys get lost in the schema transfer
|
36
|
+
* Tables without primary keys will be incredibly slow to transfer. This is due to it being inefficient having large offset values in queries.
|
37
|
+
* Multiple schemas are currently not supported
|
38
|
+
|
39
|
+
== Meta
|
40
|
+
|
41
|
+
Maintained by Ricardo Chimal, Jr. (ricardo at heroku dot com)
|
42
|
+
|
43
|
+
Written by Ricardo Chimal, Jr. (ricardo at heroku dot com) and Adam Wiggins (adam at heroku dot com)
|
44
|
+
|
45
|
+
Early research and inspiration by Blake Mizerany
|
46
|
+
|
47
|
+
Released under the MIT License: http://www.opensource.org/licenses/mit-license.php
|
48
|
+
|
49
|
+
http://github.com/ricardochimal/taps
|
50
|
+
|
51
|
+
Special Thanks to Sequel for making this tool possible http://sequel.rubyforge.org/
|
data/Rakefile
ADDED
@@ -0,0 +1,70 @@
|
|
1
|
+
begin
|
2
|
+
require 'jeweler'
|
3
|
+
Jeweler::Tasks.new do |s|
|
4
|
+
s.name = "taps-jruby"
|
5
|
+
s.summary = %Q{simple database import/export app - jruby version}
|
6
|
+
s.email = "heittman.rob@gmail.com"
|
7
|
+
s.homepage = "http://github.com/rfc2616/taps-jruby"
|
8
|
+
s.description = "A simple database agnostic import/export app to transfer data to/from a remote database. (JRuby version)"
|
9
|
+
s.authors = ["Ricardo Chimal, Jr.", "Rob Heittman"]
|
10
|
+
|
11
|
+
s.rubygems_version = %q{1.3.5}
|
12
|
+
|
13
|
+
s.add_dependency 'json_pure', '>= 1.2.0', '< 1.5.0'
|
14
|
+
s.add_dependency 'sinatra', '~> 1.0.0'
|
15
|
+
s.add_dependency 'rest-client', '>= 1.4.0', '< 1.7.0'
|
16
|
+
s.add_dependency 'sequel', '~> 3.17.0'
|
17
|
+
s.add_dependency 'activerecord-jdbcsqlite3-adapter', '>= 1.0.2'
|
18
|
+
s.add_dependency 'rack', '>= 1.0.1'
|
19
|
+
|
20
|
+
s.files = FileList['spec/*.rb'] + FileList['lib/**/*.rb'] + ['README.rdoc', 'LICENSE', 'VERSION.yml', 'Rakefile'] + FileList['bin/*']
|
21
|
+
s.executables = ['taps', 'schema']
|
22
|
+
end
|
23
|
+
rescue LoadError => e
|
24
|
+
if e.message =~ /jeweler/
|
25
|
+
puts "Jeweler not available. Install it with: sudo gem install technicalpickles-jeweler -s http://gems.github.com"
|
26
|
+
else
|
27
|
+
puts e.message + ' -- while loading jeweler.'
|
28
|
+
end
|
29
|
+
end
|
30
|
+
|
31
|
+
begin
|
32
|
+
require 'rake/rdoctask'
|
33
|
+
Rake::RDocTask.new do |rdoc|
|
34
|
+
rdoc.rdoc_dir = 'rdoc'
|
35
|
+
rdoc.title = 'taps'
|
36
|
+
rdoc.options << '--line-numbers' << '--inline-source'
|
37
|
+
rdoc.rdoc_files.include('README*')
|
38
|
+
rdoc.rdoc_files.include('lib/**/*.rb')
|
39
|
+
end
|
40
|
+
rescue LoadError
|
41
|
+
puts "Rdoc is not available"
|
42
|
+
end
|
43
|
+
|
44
|
+
begin
|
45
|
+
require 'rcov/rcovtask'
|
46
|
+
Rcov::RcovTask.new do |t|
|
47
|
+
t.libs << 'spec'
|
48
|
+
t.test_files = FileList['spec/*_spec.rb']
|
49
|
+
t.verbose = true
|
50
|
+
end
|
51
|
+
rescue LoadError
|
52
|
+
puts "RCov is not available. In order to run rcov, you must: sudo gem install rcov"
|
53
|
+
end
|
54
|
+
|
55
|
+
desc "Run all specs; requires the bacon gem"
|
56
|
+
task :spec do
|
57
|
+
if `which bacon`.empty?
|
58
|
+
puts "bacon is not available. In order to run the specs, you must: sudo gem install bacon."
|
59
|
+
else
|
60
|
+
system "bacon #{File.dirname(__FILE__)}/spec/*_spec.rb"
|
61
|
+
end
|
62
|
+
end
|
63
|
+
|
64
|
+
desc "copy/paste env vars for dev testing"
|
65
|
+
task :env do
|
66
|
+
puts "export RUBYLIB='#{File.dirname(__FILE__) + '/lib'}'"
|
67
|
+
puts "export RUBYOPT='-rrubygems'"
|
68
|
+
end
|
69
|
+
|
70
|
+
task :default => :spec
|
data/VERSION.yml
ADDED
data/bin/schema
ADDED
@@ -0,0 +1,54 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
require 'rubygems'
|
4
|
+
gem 'sequel', '~> 3.17.0'
|
5
|
+
|
6
|
+
$:.unshift File.dirname(__FILE__) + '/../lib'
|
7
|
+
|
8
|
+
require 'taps/schema'
|
9
|
+
|
10
|
+
cmd = ARGV.shift.strip rescue ''
|
11
|
+
database_url = ARGV.shift.strip rescue ''
|
12
|
+
|
13
|
+
def show_usage_and_exit
|
14
|
+
puts <<EOTXT
|
15
|
+
schema console <database_url>
|
16
|
+
schema dump <database_url>
|
17
|
+
schema dump_table <database_url> <table>
|
18
|
+
schema indexes <database_url>
|
19
|
+
schema indexes_individual <database_url>
|
20
|
+
schema reset_db_sequences <database_url>
|
21
|
+
schema load <database_url> <schema_file>
|
22
|
+
schema load_indexes <database_url> <indexes_file>
|
23
|
+
EOTXT
|
24
|
+
exit(1)
|
25
|
+
end
|
26
|
+
|
27
|
+
case cmd
|
28
|
+
when 'dump'
|
29
|
+
puts Taps::Schema.dump(database_url)
|
30
|
+
when 'dump_table'
|
31
|
+
table = ARGV.shift.strip
|
32
|
+
puts Taps::Schema.dump_table(database_url, table)
|
33
|
+
when 'indexes'
|
34
|
+
puts Taps::Schema.indexes(database_url)
|
35
|
+
when 'indexes_individual'
|
36
|
+
puts Taps::Schema.indexes_individual(database_url)
|
37
|
+
when 'load_indexes'
|
38
|
+
filename = ARGV.shift.strip rescue ''
|
39
|
+
indexes = File.read(filename) rescue show_usage_and_exit
|
40
|
+
Taps::Schema.load_indexes(database_url, indexes)
|
41
|
+
when 'load'
|
42
|
+
filename = ARGV.shift.strip rescue ''
|
43
|
+
schema = File.read(filename) rescue show_usage_and_exit
|
44
|
+
Taps::Schema.load(database_url, schema)
|
45
|
+
when 'reset_db_sequences'
|
46
|
+
Taps::Schema.reset_db_sequences(database_url)
|
47
|
+
when 'console'
|
48
|
+
$db = Sequel.connect(database_url)
|
49
|
+
require 'irb'
|
50
|
+
require 'irb/completion'
|
51
|
+
IRB.start
|
52
|
+
else
|
53
|
+
show_usage_and_exit
|
54
|
+
end
|
data/bin/schema.cmd
ADDED
data/bin/taps
ADDED
data/lib/taps/cli.rb
ADDED
@@ -0,0 +1,193 @@
|
|
1
|
+
require 'optparse'
|
2
|
+
require 'tempfile'
|
3
|
+
require 'json/pure'
|
4
|
+
require 'taps/monkey'
|
5
|
+
require 'taps/config'
|
6
|
+
require 'taps/log'
|
7
|
+
|
8
|
+
Taps::Config.taps_database_url = ENV['TAPS_DATABASE_URL'] || begin
|
9
|
+
# this is dirty but it solves a weird problem where the tempfile disappears mid-process
|
10
|
+
$__taps_database = Tempfile.new('taps.db')
|
11
|
+
$__taps_database.open()
|
12
|
+
"jdbc:sqlite://#{$__taps_database.path}"
|
13
|
+
end
|
14
|
+
|
15
|
+
module Taps
|
16
|
+
class Cli
|
17
|
+
attr_accessor :argv
|
18
|
+
|
19
|
+
def initialize(argv)
|
20
|
+
@argv = argv
|
21
|
+
end
|
22
|
+
|
23
|
+
def run
|
24
|
+
method = (argv.shift || 'help').to_sym
|
25
|
+
if [:pull, :push, :server, :version].include? method
|
26
|
+
send(method)
|
27
|
+
else
|
28
|
+
help
|
29
|
+
end
|
30
|
+
end
|
31
|
+
|
32
|
+
def pull
|
33
|
+
opts = clientoptparse(:pull)
|
34
|
+
Taps.log.level = Logger::DEBUG if opts[:debug]
|
35
|
+
if opts[:resume_filename]
|
36
|
+
clientresumexfer(:pull, opts)
|
37
|
+
else
|
38
|
+
clientxfer(:pull, opts)
|
39
|
+
end
|
40
|
+
end
|
41
|
+
|
42
|
+
def push
|
43
|
+
opts = clientoptparse(:push)
|
44
|
+
Taps.log.level = Logger::DEBUG if opts[:debug]
|
45
|
+
if opts[:resume_filename]
|
46
|
+
clientresumexfer(:push, opts)
|
47
|
+
else
|
48
|
+
clientxfer(:push, opts)
|
49
|
+
end
|
50
|
+
end
|
51
|
+
|
52
|
+
def server
|
53
|
+
opts = serveroptparse
|
54
|
+
Taps.log.level = Logger::DEBUG if opts[:debug]
|
55
|
+
Taps::Config.database_url = opts[:database_url]
|
56
|
+
Taps::Config.login = opts[:login]
|
57
|
+
Taps::Config.password = opts[:password]
|
58
|
+
|
59
|
+
Taps::Config.verify_database_url
|
60
|
+
require 'taps/server'
|
61
|
+
Taps::Server.run!({
|
62
|
+
:port => opts[:port],
|
63
|
+
:environment => :production,
|
64
|
+
:logging => true,
|
65
|
+
:dump_errors => true,
|
66
|
+
})
|
67
|
+
end
|
68
|
+
|
69
|
+
def version
|
70
|
+
puts Taps.version
|
71
|
+
end
|
72
|
+
|
73
|
+
def help
|
74
|
+
puts <<EOHELP
|
75
|
+
Options
|
76
|
+
=======
|
77
|
+
server Start a taps database import/export server
|
78
|
+
pull Pull a database from a taps server
|
79
|
+
push Push a database to a taps server
|
80
|
+
version Taps version
|
81
|
+
|
82
|
+
Add '-h' to any command to see their usage
|
83
|
+
EOHELP
|
84
|
+
end
|
85
|
+
|
86
|
+
def serveroptparse
|
87
|
+
opts={:port => 5000, :database_url => nil, :login => nil, :password => nil, :debug => false}
|
88
|
+
OptionParser.new do |o|
|
89
|
+
o.banner = "Usage: #{File.basename($0)} server [OPTIONS] <local_database_url> <login> <password>"
|
90
|
+
o.define_head "Start a taps database import/export server"
|
91
|
+
|
92
|
+
o.on("-p", "--port=N", "Server Port") { |v| opts[:port] = v.to_i if v.to_i > 0 }
|
93
|
+
o.on("-d", "--debug", "Enable Debug Messages") { |v| opts[:debug] = true }
|
94
|
+
o.parse!(argv)
|
95
|
+
|
96
|
+
opts[:database_url] = argv.shift
|
97
|
+
opts[:login] = argv.shift
|
98
|
+
opts[:password] = argv.shift
|
99
|
+
|
100
|
+
if opts[:database_url].nil?
|
101
|
+
$stderr.puts "Missing Database URL"
|
102
|
+
puts o
|
103
|
+
exit 1
|
104
|
+
end
|
105
|
+
if opts[:login].nil?
|
106
|
+
$stderr.puts "Missing Login"
|
107
|
+
puts o
|
108
|
+
exit 1
|
109
|
+
end
|
110
|
+
if opts[:password].nil?
|
111
|
+
$stderr.puts "Missing Password"
|
112
|
+
puts o
|
113
|
+
exit 1
|
114
|
+
end
|
115
|
+
end
|
116
|
+
opts
|
117
|
+
end
|
118
|
+
|
119
|
+
def clientoptparse(cmd)
|
120
|
+
opts={:default_chunksize => 1000, :database_url => nil, :remote_url => nil, :debug => false, :resume_filename => nil, :disable_compresion => false, :indexes_first => false}
|
121
|
+
OptionParser.new do |o|
|
122
|
+
o.banner = "Usage: #{File.basename($0)} #{cmd} [OPTIONS] <local_database_url> <remote_url>"
|
123
|
+
|
124
|
+
case cmd
|
125
|
+
when :pull
|
126
|
+
o.define_head "Pull a database from a taps server"
|
127
|
+
when :push
|
128
|
+
o.define_head "Push a database to a taps server"
|
129
|
+
end
|
130
|
+
|
131
|
+
o.on("-i", "--indexes-first", "Transfer indexes first before data") { |v| opts[:indexes_first] = true }
|
132
|
+
o.on("-r", "--resume=file", "Resume a Taps Session from a stored file") { |v| opts[:resume_filename] = v }
|
133
|
+
o.on("-c", "--chunksize=N", "Initial Chunksize") { |v| opts[:default_chunksize] = (v.to_i < 10 ? 10 : v.to_i) }
|
134
|
+
o.on("-g", "--disable-compression", "Disable Compression") { |v| opts[:disable_compression] = true }
|
135
|
+
o.on("-f", "--filter=regex", "Regex Filter for tables") { |v| opts[:table_filter] = v }
|
136
|
+
o.on("-t", "--tables=A,B,C", Array, "Shortcut to filter on a list of tables") do |v|
|
137
|
+
r_tables = v.collect { |t| "^#{t}$" }.join("|")
|
138
|
+
opts[:table_filter] = "(#{r_tables})"
|
139
|
+
end
|
140
|
+
o.on("-d", "--debug", "Enable Debug Messages") { |v| opts[:debug] = true }
|
141
|
+
o.parse!(argv)
|
142
|
+
|
143
|
+
opts[:database_url] = argv.shift
|
144
|
+
opts[:remote_url] = argv.shift
|
145
|
+
|
146
|
+
if opts[:database_url].nil?
|
147
|
+
$stderr.puts "Missing Database URL"
|
148
|
+
puts o
|
149
|
+
exit 1
|
150
|
+
end
|
151
|
+
if opts[:remote_url].nil?
|
152
|
+
$stderr.puts "Missing Remote Taps URL"
|
153
|
+
puts o
|
154
|
+
exit 1
|
155
|
+
end
|
156
|
+
end
|
157
|
+
|
158
|
+
opts
|
159
|
+
end
|
160
|
+
|
161
|
+
def clientxfer(method, opts)
|
162
|
+
database_url = opts.delete(:database_url)
|
163
|
+
remote_url = opts.delete(:remote_url)
|
164
|
+
|
165
|
+
Taps::Config.verify_database_url(database_url)
|
166
|
+
|
167
|
+
require 'taps/operation'
|
168
|
+
|
169
|
+
Taps::Operation.factory(method, database_url, remote_url, opts).run
|
170
|
+
end
|
171
|
+
|
172
|
+
def clientresumexfer(method, opts)
|
173
|
+
session = JSON.parse(File.read(opts.delete(:resume_filename)))
|
174
|
+
session.symbolize_recursively!
|
175
|
+
|
176
|
+
database_url = opts.delete(:database_url)
|
177
|
+
remote_url = opts.delete(:remote_url) || session.delete(:remote_url)
|
178
|
+
|
179
|
+
Taps::Config.verify_database_url(database_url)
|
180
|
+
|
181
|
+
require 'taps/operation'
|
182
|
+
|
183
|
+
newsession = session.merge({
|
184
|
+
:default_chunksize => opts[:default_chunksize],
|
185
|
+
:disable_compression => opts[:disable_compression],
|
186
|
+
:resume => true,
|
187
|
+
})
|
188
|
+
|
189
|
+
Taps::Operation.factory(method, database_url, remote_url, newsession).run
|
190
|
+
end
|
191
|
+
|
192
|
+
end
|
193
|
+
end
|
data/lib/taps/config.rb
ADDED
@@ -0,0 +1,47 @@
|
|
1
|
+
require 'sequel'
|
2
|
+
#require 'sqlite3'
|
3
|
+
require 'yaml'
|
4
|
+
|
5
|
+
Sequel.datetime_class = DateTime
|
6
|
+
|
7
|
+
module Taps
|
8
|
+
def self.version_yml
|
9
|
+
@@version_yml ||= YAML.load(File.read(File.dirname(__FILE__) + '/../../VERSION.yml'))
|
10
|
+
end
|
11
|
+
|
12
|
+
def self.version
|
13
|
+
version = "#{version_yml[:major]}.#{version_yml[:minor]}.#{version_yml[:patch]}"
|
14
|
+
version += ".#{version_yml[:build]}" if version_yml[:build]
|
15
|
+
version
|
16
|
+
end
|
17
|
+
|
18
|
+
def self.compatible_version
|
19
|
+
"#{version_yml[:major]}.#{version_yml[:minor]}"
|
20
|
+
end
|
21
|
+
|
22
|
+
def self.exiting=(val)
|
23
|
+
@@exiting = val
|
24
|
+
end
|
25
|
+
|
26
|
+
def exiting?
|
27
|
+
(@@exiting ||= false) == true
|
28
|
+
end
|
29
|
+
|
30
|
+
class Config
|
31
|
+
class << self
|
32
|
+
attr_accessor :taps_database_url
|
33
|
+
attr_accessor :login, :password, :database_url, :remote_url
|
34
|
+
attr_accessor :chunksize
|
35
|
+
|
36
|
+
def verify_database_url(db_url=nil)
|
37
|
+
db_url ||= self.database_url
|
38
|
+
db = Sequel.connect(db_url)
|
39
|
+
db.tables
|
40
|
+
db.disconnect
|
41
|
+
rescue Object => e
|
42
|
+
puts "Failed to connect to database:\n #{e.class} -> #{e}"
|
43
|
+
exit 1
|
44
|
+
end
|
45
|
+
end
|
46
|
+
end
|
47
|
+
end
|