taps-jruby 0.3.14

Sign up to get free protection for your applications and to get access to all the features.
data/LICENSE ADDED
@@ -0,0 +1,20 @@
1
+ Copyright (c) 2008 Ricardo Chimal, Jr
2
+
3
+ Permission is hereby granted, free of charge, to any person obtaining
4
+ a copy of this software and associated documentation files (the
5
+ "Software"), to deal in the Software without restriction, including
6
+ without limitation the rights to use, copy, modify, merge, publish,
7
+ distribute, sublicense, and/or sell copies of the Software, and to
8
+ permit persons to whom the Software is furnished to do so, subject to
9
+ the following conditions:
10
+
11
+ The above copyright notice and this permission notice shall be
12
+ included in all copies or substantial portions of the Software.
13
+
14
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
15
+ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
16
+ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
17
+ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
18
+ LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
19
+ OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
20
+ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
@@ -0,0 +1,51 @@
1
+ = Taps -- simple database import/export app
2
+
3
+ A simple database agnostic import/export app to transfer data to/from a remote database.
4
+
5
+ == Usage: Server
6
+
7
+ Here's how you start a taps server
8
+
9
+ $ taps server postgres://localdbuser:localdbpass@localhost/dbname httpuser httppassword
10
+
11
+ You can also specify an encoding in the database url
12
+
13
+ $ taps server mysql://localdbuser:localdbpass@localhost/dbname?encoding=latin1 httpuser httppassword
14
+
15
+ == Usage: Client
16
+
17
+ When you want to pull down a database from a taps server
18
+
19
+ $ taps pull postgres://dbuser:dbpassword@localhost/dbname http://httpuser:httppassword@example.com:5000
20
+
21
+ or when you want to push a local database to a taps server
22
+
23
+ $ taps push postgres://dbuser:dbpassword@localhost/dbname http://httpuser:httppassword@example.com:5000
24
+
25
+ or when you want to transfer a list of tables
26
+
27
+ $ taps push postgres://dbuser:dbpassword@localhost/dbname http://httpuser:httppassword@example.com:5000 --tables logs,tags
28
+
29
+ or when you want to transfer tables that start with a word
30
+
31
+ $ taps push postgres://dbuser:dbpassword@localhost/dbname http://httpuser:httppassword@example.com:5000 --filter '^log_'
32
+
33
+ == Known Issues
34
+
35
+ * Foreign Keys get lost in the schema transfer
36
+ * Tables without primary keys will be incredibly slow to transfer. This is due to it being inefficient having large offset values in queries.
37
+ * Multiple schemas are currently not supported
38
+
39
+ == Meta
40
+
41
+ Maintained by Ricardo Chimal, Jr. (ricardo at heroku dot com)
42
+
43
+ Written by Ricardo Chimal, Jr. (ricardo at heroku dot com) and Adam Wiggins (adam at heroku dot com)
44
+
45
+ Early research and inspiration by Blake Mizerany
46
+
47
+ Released under the MIT License: http://www.opensource.org/licenses/mit-license.php
48
+
49
+ http://github.com/ricardochimal/taps
50
+
51
+ Special Thanks to Sequel for making this tool possible http://sequel.rubyforge.org/
@@ -0,0 +1,70 @@
1
+ begin
2
+ require 'jeweler'
3
+ Jeweler::Tasks.new do |s|
4
+ s.name = "taps-jruby"
5
+ s.summary = %Q{simple database import/export app - jruby version}
6
+ s.email = "heittman.rob@gmail.com"
7
+ s.homepage = "http://github.com/rfc2616/taps-jruby"
8
+ s.description = "A simple database agnostic import/export app to transfer data to/from a remote database. (JRuby version)"
9
+ s.authors = ["Ricardo Chimal, Jr.", "Rob Heittman"]
10
+
11
+ s.rubygems_version = %q{1.3.5}
12
+
13
+ s.add_dependency 'json_pure', '>= 1.2.0', '< 1.5.0'
14
+ s.add_dependency 'sinatra', '~> 1.0.0'
15
+ s.add_dependency 'rest-client', '>= 1.4.0', '< 1.7.0'
16
+ s.add_dependency 'sequel', '~> 3.17.0'
17
+ s.add_dependency 'activerecord-jdbcsqlite3-adapter', '>= 1.0.2'
18
+ s.add_dependency 'rack', '>= 1.0.1'
19
+
20
+ s.files = FileList['spec/*.rb'] + FileList['lib/**/*.rb'] + ['README.rdoc', 'LICENSE', 'VERSION.yml', 'Rakefile'] + FileList['bin/*']
21
+ s.executables = ['taps', 'schema']
22
+ end
23
+ rescue LoadError => e
24
+ if e.message =~ /jeweler/
25
+ puts "Jeweler not available. Install it with: sudo gem install technicalpickles-jeweler -s http://gems.github.com"
26
+ else
27
+ puts e.message + ' -- while loading jeweler.'
28
+ end
29
+ end
30
+
31
+ begin
32
+ require 'rake/rdoctask'
33
+ Rake::RDocTask.new do |rdoc|
34
+ rdoc.rdoc_dir = 'rdoc'
35
+ rdoc.title = 'taps'
36
+ rdoc.options << '--line-numbers' << '--inline-source'
37
+ rdoc.rdoc_files.include('README*')
38
+ rdoc.rdoc_files.include('lib/**/*.rb')
39
+ end
40
+ rescue LoadError
41
+ puts "Rdoc is not available"
42
+ end
43
+
44
+ begin
45
+ require 'rcov/rcovtask'
46
+ Rcov::RcovTask.new do |t|
47
+ t.libs << 'spec'
48
+ t.test_files = FileList['spec/*_spec.rb']
49
+ t.verbose = true
50
+ end
51
+ rescue LoadError
52
+ puts "RCov is not available. In order to run rcov, you must: sudo gem install rcov"
53
+ end
54
+
55
+ desc "Run all specs; requires the bacon gem"
56
+ task :spec do
57
+ if `which bacon`.empty?
58
+ puts "bacon is not available. In order to run the specs, you must: sudo gem install bacon."
59
+ else
60
+ system "bacon #{File.dirname(__FILE__)}/spec/*_spec.rb"
61
+ end
62
+ end
63
+
64
+ desc "copy/paste env vars for dev testing"
65
+ task :env do
66
+ puts "export RUBYLIB='#{File.dirname(__FILE__) + '/lib'}'"
67
+ puts "export RUBYOPT='-rrubygems'"
68
+ end
69
+
70
+ task :default => :spec
data/TODO ADDED
@@ -0,0 +1 @@
1
+ * Detect incompatible Marshal versions
@@ -0,0 +1,5 @@
1
+ ---
2
+ :build:
3
+ :minor: 3
4
+ :patch: 14
5
+ :major: 0
@@ -0,0 +1,54 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ require 'rubygems'
4
+ gem 'sequel', '~> 3.17.0'
5
+
6
+ $:.unshift File.dirname(__FILE__) + '/../lib'
7
+
8
+ require 'taps/schema'
9
+
10
+ cmd = ARGV.shift.strip rescue ''
11
+ database_url = ARGV.shift.strip rescue ''
12
+
13
+ def show_usage_and_exit
14
+ puts <<EOTXT
15
+ schema console <database_url>
16
+ schema dump <database_url>
17
+ schema dump_table <database_url> <table>
18
+ schema indexes <database_url>
19
+ schema indexes_individual <database_url>
20
+ schema reset_db_sequences <database_url>
21
+ schema load <database_url> <schema_file>
22
+ schema load_indexes <database_url> <indexes_file>
23
+ EOTXT
24
+ exit(1)
25
+ end
26
+
27
+ case cmd
28
+ when 'dump'
29
+ puts Taps::Schema.dump(database_url)
30
+ when 'dump_table'
31
+ table = ARGV.shift.strip
32
+ puts Taps::Schema.dump_table(database_url, table)
33
+ when 'indexes'
34
+ puts Taps::Schema.indexes(database_url)
35
+ when 'indexes_individual'
36
+ puts Taps::Schema.indexes_individual(database_url)
37
+ when 'load_indexes'
38
+ filename = ARGV.shift.strip rescue ''
39
+ indexes = File.read(filename) rescue show_usage_and_exit
40
+ Taps::Schema.load_indexes(database_url, indexes)
41
+ when 'load'
42
+ filename = ARGV.shift.strip rescue ''
43
+ schema = File.read(filename) rescue show_usage_and_exit
44
+ Taps::Schema.load(database_url, schema)
45
+ when 'reset_db_sequences'
46
+ Taps::Schema.reset_db_sequences(database_url)
47
+ when 'console'
48
+ $db = Sequel.connect(database_url)
49
+ require 'irb'
50
+ require 'irb/completion'
51
+ IRB.start
52
+ else
53
+ show_usage_and_exit
54
+ end
@@ -0,0 +1,6 @@
1
+ @ECHO OFF
2
+ IF NOT "%~f0" == "~f0" GOTO :WinNT
3
+ @"ruby.exe" "./schema" %1 %2 %3 %4 %5 %6 %7 %8 %9
4
+ GOTO :EOF
5
+ :WinNT
6
+ @"ruby.exe" "%~dpn0" %*
@@ -0,0 +1,6 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ $:.unshift File.dirname(__FILE__) + '/../lib'
4
+ require 'taps/cli'
5
+
6
+ Taps::Cli.new(ARGV.dup).run
@@ -0,0 +1,193 @@
1
+ require 'optparse'
2
+ require 'tempfile'
3
+ require 'json/pure'
4
+ require 'taps/monkey'
5
+ require 'taps/config'
6
+ require 'taps/log'
7
+
8
+ Taps::Config.taps_database_url = ENV['TAPS_DATABASE_URL'] || begin
9
+ # this is dirty but it solves a weird problem where the tempfile disappears mid-process
10
+ $__taps_database = Tempfile.new('taps.db')
11
+ $__taps_database.open()
12
+ "jdbc:sqlite://#{$__taps_database.path}"
13
+ end
14
+
15
+ module Taps
16
+ class Cli
17
+ attr_accessor :argv
18
+
19
+ def initialize(argv)
20
+ @argv = argv
21
+ end
22
+
23
+ def run
24
+ method = (argv.shift || 'help').to_sym
25
+ if [:pull, :push, :server, :version].include? method
26
+ send(method)
27
+ else
28
+ help
29
+ end
30
+ end
31
+
32
+ def pull
33
+ opts = clientoptparse(:pull)
34
+ Taps.log.level = Logger::DEBUG if opts[:debug]
35
+ if opts[:resume_filename]
36
+ clientresumexfer(:pull, opts)
37
+ else
38
+ clientxfer(:pull, opts)
39
+ end
40
+ end
41
+
42
+ def push
43
+ opts = clientoptparse(:push)
44
+ Taps.log.level = Logger::DEBUG if opts[:debug]
45
+ if opts[:resume_filename]
46
+ clientresumexfer(:push, opts)
47
+ else
48
+ clientxfer(:push, opts)
49
+ end
50
+ end
51
+
52
+ def server
53
+ opts = serveroptparse
54
+ Taps.log.level = Logger::DEBUG if opts[:debug]
55
+ Taps::Config.database_url = opts[:database_url]
56
+ Taps::Config.login = opts[:login]
57
+ Taps::Config.password = opts[:password]
58
+
59
+ Taps::Config.verify_database_url
60
+ require 'taps/server'
61
+ Taps::Server.run!({
62
+ :port => opts[:port],
63
+ :environment => :production,
64
+ :logging => true,
65
+ :dump_errors => true,
66
+ })
67
+ end
68
+
69
+ def version
70
+ puts Taps.version
71
+ end
72
+
73
+ def help
74
+ puts <<EOHELP
75
+ Options
76
+ =======
77
+ server Start a taps database import/export server
78
+ pull Pull a database from a taps server
79
+ push Push a database to a taps server
80
+ version Taps version
81
+
82
+ Add '-h' to any command to see their usage
83
+ EOHELP
84
+ end
85
+
86
+ def serveroptparse
87
+ opts={:port => 5000, :database_url => nil, :login => nil, :password => nil, :debug => false}
88
+ OptionParser.new do |o|
89
+ o.banner = "Usage: #{File.basename($0)} server [OPTIONS] <local_database_url> <login> <password>"
90
+ o.define_head "Start a taps database import/export server"
91
+
92
+ o.on("-p", "--port=N", "Server Port") { |v| opts[:port] = v.to_i if v.to_i > 0 }
93
+ o.on("-d", "--debug", "Enable Debug Messages") { |v| opts[:debug] = true }
94
+ o.parse!(argv)
95
+
96
+ opts[:database_url] = argv.shift
97
+ opts[:login] = argv.shift
98
+ opts[:password] = argv.shift
99
+
100
+ if opts[:database_url].nil?
101
+ $stderr.puts "Missing Database URL"
102
+ puts o
103
+ exit 1
104
+ end
105
+ if opts[:login].nil?
106
+ $stderr.puts "Missing Login"
107
+ puts o
108
+ exit 1
109
+ end
110
+ if opts[:password].nil?
111
+ $stderr.puts "Missing Password"
112
+ puts o
113
+ exit 1
114
+ end
115
+ end
116
+ opts
117
+ end
118
+
119
+ def clientoptparse(cmd)
120
+ opts={:default_chunksize => 1000, :database_url => nil, :remote_url => nil, :debug => false, :resume_filename => nil, :disable_compresion => false, :indexes_first => false}
121
+ OptionParser.new do |o|
122
+ o.banner = "Usage: #{File.basename($0)} #{cmd} [OPTIONS] <local_database_url> <remote_url>"
123
+
124
+ case cmd
125
+ when :pull
126
+ o.define_head "Pull a database from a taps server"
127
+ when :push
128
+ o.define_head "Push a database to a taps server"
129
+ end
130
+
131
+ o.on("-i", "--indexes-first", "Transfer indexes first before data") { |v| opts[:indexes_first] = true }
132
+ o.on("-r", "--resume=file", "Resume a Taps Session from a stored file") { |v| opts[:resume_filename] = v }
133
+ o.on("-c", "--chunksize=N", "Initial Chunksize") { |v| opts[:default_chunksize] = (v.to_i < 10 ? 10 : v.to_i) }
134
+ o.on("-g", "--disable-compression", "Disable Compression") { |v| opts[:disable_compression] = true }
135
+ o.on("-f", "--filter=regex", "Regex Filter for tables") { |v| opts[:table_filter] = v }
136
+ o.on("-t", "--tables=A,B,C", Array, "Shortcut to filter on a list of tables") do |v|
137
+ r_tables = v.collect { |t| "^#{t}$" }.join("|")
138
+ opts[:table_filter] = "(#{r_tables})"
139
+ end
140
+ o.on("-d", "--debug", "Enable Debug Messages") { |v| opts[:debug] = true }
141
+ o.parse!(argv)
142
+
143
+ opts[:database_url] = argv.shift
144
+ opts[:remote_url] = argv.shift
145
+
146
+ if opts[:database_url].nil?
147
+ $stderr.puts "Missing Database URL"
148
+ puts o
149
+ exit 1
150
+ end
151
+ if opts[:remote_url].nil?
152
+ $stderr.puts "Missing Remote Taps URL"
153
+ puts o
154
+ exit 1
155
+ end
156
+ end
157
+
158
+ opts
159
+ end
160
+
161
+ def clientxfer(method, opts)
162
+ database_url = opts.delete(:database_url)
163
+ remote_url = opts.delete(:remote_url)
164
+
165
+ Taps::Config.verify_database_url(database_url)
166
+
167
+ require 'taps/operation'
168
+
169
+ Taps::Operation.factory(method, database_url, remote_url, opts).run
170
+ end
171
+
172
+ def clientresumexfer(method, opts)
173
+ session = JSON.parse(File.read(opts.delete(:resume_filename)))
174
+ session.symbolize_recursively!
175
+
176
+ database_url = opts.delete(:database_url)
177
+ remote_url = opts.delete(:remote_url) || session.delete(:remote_url)
178
+
179
+ Taps::Config.verify_database_url(database_url)
180
+
181
+ require 'taps/operation'
182
+
183
+ newsession = session.merge({
184
+ :default_chunksize => opts[:default_chunksize],
185
+ :disable_compression => opts[:disable_compression],
186
+ :resume => true,
187
+ })
188
+
189
+ Taps::Operation.factory(method, database_url, remote_url, newsession).run
190
+ end
191
+
192
+ end
193
+ end
@@ -0,0 +1,47 @@
1
+ require 'sequel'
2
+ #require 'sqlite3'
3
+ require 'yaml'
4
+
5
+ Sequel.datetime_class = DateTime
6
+
7
+ module Taps
8
+ def self.version_yml
9
+ @@version_yml ||= YAML.load(File.read(File.dirname(__FILE__) + '/../../VERSION.yml'))
10
+ end
11
+
12
+ def self.version
13
+ version = "#{version_yml[:major]}.#{version_yml[:minor]}.#{version_yml[:patch]}"
14
+ version += ".#{version_yml[:build]}" if version_yml[:build]
15
+ version
16
+ end
17
+
18
+ def self.compatible_version
19
+ "#{version_yml[:major]}.#{version_yml[:minor]}"
20
+ end
21
+
22
+ def self.exiting=(val)
23
+ @@exiting = val
24
+ end
25
+
26
+ def exiting?
27
+ (@@exiting ||= false) == true
28
+ end
29
+
30
+ class Config
31
+ class << self
32
+ attr_accessor :taps_database_url
33
+ attr_accessor :login, :password, :database_url, :remote_url
34
+ attr_accessor :chunksize
35
+
36
+ def verify_database_url(db_url=nil)
37
+ db_url ||= self.database_url
38
+ db = Sequel.connect(db_url)
39
+ db.tables
40
+ db.disconnect
41
+ rescue Object => e
42
+ puts "Failed to connect to database:\n #{e.class} -> #{e}"
43
+ exit 1
44
+ end
45
+ end
46
+ end
47
+ end