taps-jruby 0.3.14

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
data/LICENSE ADDED
@@ -0,0 +1,20 @@
1
+ Copyright (c) 2008 Ricardo Chimal, Jr
2
+
3
+ Permission is hereby granted, free of charge, to any person obtaining
4
+ a copy of this software and associated documentation files (the
5
+ "Software"), to deal in the Software without restriction, including
6
+ without limitation the rights to use, copy, modify, merge, publish,
7
+ distribute, sublicense, and/or sell copies of the Software, and to
8
+ permit persons to whom the Software is furnished to do so, subject to
9
+ the following conditions:
10
+
11
+ The above copyright notice and this permission notice shall be
12
+ included in all copies or substantial portions of the Software.
13
+
14
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
15
+ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
16
+ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
17
+ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
18
+ LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
19
+ OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
20
+ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
@@ -0,0 +1,51 @@
1
+ = Taps -- simple database import/export app
2
+
3
+ A simple database agnostic import/export app to transfer data to/from a remote database.
4
+
5
+ == Usage: Server
6
+
7
+ Here's how you start a taps server
8
+
9
+ $ taps server postgres://localdbuser:localdbpass@localhost/dbname httpuser httppassword
10
+
11
+ You can also specify an encoding in the database url
12
+
13
+ $ taps server mysql://localdbuser:localdbpass@localhost/dbname?encoding=latin1 httpuser httppassword
14
+
15
+ == Usage: Client
16
+
17
+ When you want to pull down a database from a taps server
18
+
19
+ $ taps pull postgres://dbuser:dbpassword@localhost/dbname http://httpuser:httppassword@example.com:5000
20
+
21
+ or when you want to push a local database to a taps server
22
+
23
+ $ taps push postgres://dbuser:dbpassword@localhost/dbname http://httpuser:httppassword@example.com:5000
24
+
25
+ or when you want to transfer a list of tables
26
+
27
+ $ taps push postgres://dbuser:dbpassword@localhost/dbname http://httpuser:httppassword@example.com:5000 --tables logs,tags
28
+
29
+ or when you want to transfer tables that start with a word
30
+
31
+ $ taps push postgres://dbuser:dbpassword@localhost/dbname http://httpuser:httppassword@example.com:5000 --filter '^log_'
32
+
33
+ == Known Issues
34
+
35
+ * Foreign Keys get lost in the schema transfer
36
+ * Tables without primary keys will be incredibly slow to transfer. This is due to it being inefficient having large offset values in queries.
37
+ * Multiple schemas are currently not supported
38
+
39
+ == Meta
40
+
41
+ Maintained by Ricardo Chimal, Jr. (ricardo at heroku dot com)
42
+
43
+ Written by Ricardo Chimal, Jr. (ricardo at heroku dot com) and Adam Wiggins (adam at heroku dot com)
44
+
45
+ Early research and inspiration by Blake Mizerany
46
+
47
+ Released under the MIT License: http://www.opensource.org/licenses/mit-license.php
48
+
49
+ http://github.com/ricardochimal/taps
50
+
51
+ Special Thanks to Sequel for making this tool possible http://sequel.rubyforge.org/
@@ -0,0 +1,70 @@
1
+ begin
2
+ require 'jeweler'
3
+ Jeweler::Tasks.new do |s|
4
+ s.name = "taps-jruby"
5
+ s.summary = %Q{simple database import/export app - jruby version}
6
+ s.email = "heittman.rob@gmail.com"
7
+ s.homepage = "http://github.com/rfc2616/taps-jruby"
8
+ s.description = "A simple database agnostic import/export app to transfer data to/from a remote database. (JRuby version)"
9
+ s.authors = ["Ricardo Chimal, Jr.", "Rob Heittman"]
10
+
11
+ s.rubygems_version = %q{1.3.5}
12
+
13
+ s.add_dependency 'json_pure', '>= 1.2.0', '< 1.5.0'
14
+ s.add_dependency 'sinatra', '~> 1.0.0'
15
+ s.add_dependency 'rest-client', '>= 1.4.0', '< 1.7.0'
16
+ s.add_dependency 'sequel', '~> 3.17.0'
17
+ s.add_dependency 'activerecord-jdbcsqlite3-adapter', '>= 1.0.2'
18
+ s.add_dependency 'rack', '>= 1.0.1'
19
+
20
+ s.files = FileList['spec/*.rb'] + FileList['lib/**/*.rb'] + ['README.rdoc', 'LICENSE', 'VERSION.yml', 'Rakefile'] + FileList['bin/*']
21
+ s.executables = ['taps', 'schema']
22
+ end
23
+ rescue LoadError => e
24
+ if e.message =~ /jeweler/
25
+ puts "Jeweler not available. Install it with: sudo gem install technicalpickles-jeweler -s http://gems.github.com"
26
+ else
27
+ puts e.message + ' -- while loading jeweler.'
28
+ end
29
+ end
30
+
31
+ begin
32
+ require 'rake/rdoctask'
33
+ Rake::RDocTask.new do |rdoc|
34
+ rdoc.rdoc_dir = 'rdoc'
35
+ rdoc.title = 'taps'
36
+ rdoc.options << '--line-numbers' << '--inline-source'
37
+ rdoc.rdoc_files.include('README*')
38
+ rdoc.rdoc_files.include('lib/**/*.rb')
39
+ end
40
+ rescue LoadError
41
+ puts "Rdoc is not available"
42
+ end
43
+
44
+ begin
45
+ require 'rcov/rcovtask'
46
+ Rcov::RcovTask.new do |t|
47
+ t.libs << 'spec'
48
+ t.test_files = FileList['spec/*_spec.rb']
49
+ t.verbose = true
50
+ end
51
+ rescue LoadError
52
+ puts "RCov is not available. In order to run rcov, you must: sudo gem install rcov"
53
+ end
54
+
55
+ desc "Run all specs; requires the bacon gem"
56
+ task :spec do
57
+ if `which bacon`.empty?
58
+ puts "bacon is not available. In order to run the specs, you must: sudo gem install bacon."
59
+ else
60
+ system "bacon #{File.dirname(__FILE__)}/spec/*_spec.rb"
61
+ end
62
+ end
63
+
64
+ desc "copy/paste env vars for dev testing"
65
+ task :env do
66
+ puts "export RUBYLIB='#{File.dirname(__FILE__) + '/lib'}'"
67
+ puts "export RUBYOPT='-rrubygems'"
68
+ end
69
+
70
+ task :default => :spec
data/TODO ADDED
@@ -0,0 +1 @@
1
+ * Detect incompatible Marshal versions
@@ -0,0 +1,5 @@
1
+ ---
2
+ :build:
3
+ :minor: 3
4
+ :patch: 14
5
+ :major: 0
@@ -0,0 +1,54 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ require 'rubygems'
4
+ gem 'sequel', '~> 3.17.0'
5
+
6
+ $:.unshift File.dirname(__FILE__) + '/../lib'
7
+
8
+ require 'taps/schema'
9
+
10
+ cmd = ARGV.shift.strip rescue ''
11
+ database_url = ARGV.shift.strip rescue ''
12
+
13
+ def show_usage_and_exit
14
+ puts <<EOTXT
15
+ schema console <database_url>
16
+ schema dump <database_url>
17
+ schema dump_table <database_url> <table>
18
+ schema indexes <database_url>
19
+ schema indexes_individual <database_url>
20
+ schema reset_db_sequences <database_url>
21
+ schema load <database_url> <schema_file>
22
+ schema load_indexes <database_url> <indexes_file>
23
+ EOTXT
24
+ exit(1)
25
+ end
26
+
27
+ case cmd
28
+ when 'dump'
29
+ puts Taps::Schema.dump(database_url)
30
+ when 'dump_table'
31
+ table = ARGV.shift.strip
32
+ puts Taps::Schema.dump_table(database_url, table)
33
+ when 'indexes'
34
+ puts Taps::Schema.indexes(database_url)
35
+ when 'indexes_individual'
36
+ puts Taps::Schema.indexes_individual(database_url)
37
+ when 'load_indexes'
38
+ filename = ARGV.shift.strip rescue ''
39
+ indexes = File.read(filename) rescue show_usage_and_exit
40
+ Taps::Schema.load_indexes(database_url, indexes)
41
+ when 'load'
42
+ filename = ARGV.shift.strip rescue ''
43
+ schema = File.read(filename) rescue show_usage_and_exit
44
+ Taps::Schema.load(database_url, schema)
45
+ when 'reset_db_sequences'
46
+ Taps::Schema.reset_db_sequences(database_url)
47
+ when 'console'
48
+ $db = Sequel.connect(database_url)
49
+ require 'irb'
50
+ require 'irb/completion'
51
+ IRB.start
52
+ else
53
+ show_usage_and_exit
54
+ end
@@ -0,0 +1,6 @@
1
+ @ECHO OFF
2
+ IF NOT "%~f0" == "~f0" GOTO :WinNT
3
+ @"ruby.exe" "./schema" %1 %2 %3 %4 %5 %6 %7 %8 %9
4
+ GOTO :EOF
5
+ :WinNT
6
+ @"ruby.exe" "%~dpn0" %*
@@ -0,0 +1,6 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ $:.unshift File.dirname(__FILE__) + '/../lib'
4
+ require 'taps/cli'
5
+
6
+ Taps::Cli.new(ARGV.dup).run
@@ -0,0 +1,193 @@
1
+ require 'optparse'
2
+ require 'tempfile'
3
+ require 'json/pure'
4
+ require 'taps/monkey'
5
+ require 'taps/config'
6
+ require 'taps/log'
7
+
8
+ Taps::Config.taps_database_url = ENV['TAPS_DATABASE_URL'] || begin
9
+ # this is dirty but it solves a weird problem where the tempfile disappears mid-process
10
+ $__taps_database = Tempfile.new('taps.db')
11
+ $__taps_database.open()
12
+ "jdbc:sqlite://#{$__taps_database.path}"
13
+ end
14
+
15
+ module Taps
16
+ class Cli
17
+ attr_accessor :argv
18
+
19
+ def initialize(argv)
20
+ @argv = argv
21
+ end
22
+
23
+ def run
24
+ method = (argv.shift || 'help').to_sym
25
+ if [:pull, :push, :server, :version].include? method
26
+ send(method)
27
+ else
28
+ help
29
+ end
30
+ end
31
+
32
+ def pull
33
+ opts = clientoptparse(:pull)
34
+ Taps.log.level = Logger::DEBUG if opts[:debug]
35
+ if opts[:resume_filename]
36
+ clientresumexfer(:pull, opts)
37
+ else
38
+ clientxfer(:pull, opts)
39
+ end
40
+ end
41
+
42
+ def push
43
+ opts = clientoptparse(:push)
44
+ Taps.log.level = Logger::DEBUG if opts[:debug]
45
+ if opts[:resume_filename]
46
+ clientresumexfer(:push, opts)
47
+ else
48
+ clientxfer(:push, opts)
49
+ end
50
+ end
51
+
52
+ def server
53
+ opts = serveroptparse
54
+ Taps.log.level = Logger::DEBUG if opts[:debug]
55
+ Taps::Config.database_url = opts[:database_url]
56
+ Taps::Config.login = opts[:login]
57
+ Taps::Config.password = opts[:password]
58
+
59
+ Taps::Config.verify_database_url
60
+ require 'taps/server'
61
+ Taps::Server.run!({
62
+ :port => opts[:port],
63
+ :environment => :production,
64
+ :logging => true,
65
+ :dump_errors => true,
66
+ })
67
+ end
68
+
69
+ def version
70
+ puts Taps.version
71
+ end
72
+
73
+ def help
74
+ puts <<EOHELP
75
+ Options
76
+ =======
77
+ server Start a taps database import/export server
78
+ pull Pull a database from a taps server
79
+ push Push a database to a taps server
80
+ version Taps version
81
+
82
+ Add '-h' to any command to see their usage
83
+ EOHELP
84
+ end
85
+
86
+ def serveroptparse
87
+ opts={:port => 5000, :database_url => nil, :login => nil, :password => nil, :debug => false}
88
+ OptionParser.new do |o|
89
+ o.banner = "Usage: #{File.basename($0)} server [OPTIONS] <local_database_url> <login> <password>"
90
+ o.define_head "Start a taps database import/export server"
91
+
92
+ o.on("-p", "--port=N", "Server Port") { |v| opts[:port] = v.to_i if v.to_i > 0 }
93
+ o.on("-d", "--debug", "Enable Debug Messages") { |v| opts[:debug] = true }
94
+ o.parse!(argv)
95
+
96
+ opts[:database_url] = argv.shift
97
+ opts[:login] = argv.shift
98
+ opts[:password] = argv.shift
99
+
100
+ if opts[:database_url].nil?
101
+ $stderr.puts "Missing Database URL"
102
+ puts o
103
+ exit 1
104
+ end
105
+ if opts[:login].nil?
106
+ $stderr.puts "Missing Login"
107
+ puts o
108
+ exit 1
109
+ end
110
+ if opts[:password].nil?
111
+ $stderr.puts "Missing Password"
112
+ puts o
113
+ exit 1
114
+ end
115
+ end
116
+ opts
117
+ end
118
+
119
+ def clientoptparse(cmd)
120
+ opts={:default_chunksize => 1000, :database_url => nil, :remote_url => nil, :debug => false, :resume_filename => nil, :disable_compresion => false, :indexes_first => false}
121
+ OptionParser.new do |o|
122
+ o.banner = "Usage: #{File.basename($0)} #{cmd} [OPTIONS] <local_database_url> <remote_url>"
123
+
124
+ case cmd
125
+ when :pull
126
+ o.define_head "Pull a database from a taps server"
127
+ when :push
128
+ o.define_head "Push a database to a taps server"
129
+ end
130
+
131
+ o.on("-i", "--indexes-first", "Transfer indexes first before data") { |v| opts[:indexes_first] = true }
132
+ o.on("-r", "--resume=file", "Resume a Taps Session from a stored file") { |v| opts[:resume_filename] = v }
133
+ o.on("-c", "--chunksize=N", "Initial Chunksize") { |v| opts[:default_chunksize] = (v.to_i < 10 ? 10 : v.to_i) }
134
+ o.on("-g", "--disable-compression", "Disable Compression") { |v| opts[:disable_compression] = true }
135
+ o.on("-f", "--filter=regex", "Regex Filter for tables") { |v| opts[:table_filter] = v }
136
+ o.on("-t", "--tables=A,B,C", Array, "Shortcut to filter on a list of tables") do |v|
137
+ r_tables = v.collect { |t| "^#{t}$" }.join("|")
138
+ opts[:table_filter] = "(#{r_tables})"
139
+ end
140
+ o.on("-d", "--debug", "Enable Debug Messages") { |v| opts[:debug] = true }
141
+ o.parse!(argv)
142
+
143
+ opts[:database_url] = argv.shift
144
+ opts[:remote_url] = argv.shift
145
+
146
+ if opts[:database_url].nil?
147
+ $stderr.puts "Missing Database URL"
148
+ puts o
149
+ exit 1
150
+ end
151
+ if opts[:remote_url].nil?
152
+ $stderr.puts "Missing Remote Taps URL"
153
+ puts o
154
+ exit 1
155
+ end
156
+ end
157
+
158
+ opts
159
+ end
160
+
161
+ def clientxfer(method, opts)
162
+ database_url = opts.delete(:database_url)
163
+ remote_url = opts.delete(:remote_url)
164
+
165
+ Taps::Config.verify_database_url(database_url)
166
+
167
+ require 'taps/operation'
168
+
169
+ Taps::Operation.factory(method, database_url, remote_url, opts).run
170
+ end
171
+
172
+ def clientresumexfer(method, opts)
173
+ session = JSON.parse(File.read(opts.delete(:resume_filename)))
174
+ session.symbolize_recursively!
175
+
176
+ database_url = opts.delete(:database_url)
177
+ remote_url = opts.delete(:remote_url) || session.delete(:remote_url)
178
+
179
+ Taps::Config.verify_database_url(database_url)
180
+
181
+ require 'taps/operation'
182
+
183
+ newsession = session.merge({
184
+ :default_chunksize => opts[:default_chunksize],
185
+ :disable_compression => opts[:disable_compression],
186
+ :resume => true,
187
+ })
188
+
189
+ Taps::Operation.factory(method, database_url, remote_url, newsession).run
190
+ end
191
+
192
+ end
193
+ end
@@ -0,0 +1,47 @@
1
+ require 'sequel'
2
+ #require 'sqlite3'
3
+ require 'yaml'
4
+
5
+ Sequel.datetime_class = DateTime
6
+
7
+ module Taps
8
+ def self.version_yml
9
+ @@version_yml ||= YAML.load(File.read(File.dirname(__FILE__) + '/../../VERSION.yml'))
10
+ end
11
+
12
+ def self.version
13
+ version = "#{version_yml[:major]}.#{version_yml[:minor]}.#{version_yml[:patch]}"
14
+ version += ".#{version_yml[:build]}" if version_yml[:build]
15
+ version
16
+ end
17
+
18
+ def self.compatible_version
19
+ "#{version_yml[:major]}.#{version_yml[:minor]}"
20
+ end
21
+
22
+ def self.exiting=(val)
23
+ @@exiting = val
24
+ end
25
+
26
+ def exiting?
27
+ (@@exiting ||= false) == true
28
+ end
29
+
30
+ class Config
31
+ class << self
32
+ attr_accessor :taps_database_url
33
+ attr_accessor :login, :password, :database_url, :remote_url
34
+ attr_accessor :chunksize
35
+
36
+ def verify_database_url(db_url=nil)
37
+ db_url ||= self.database_url
38
+ db = Sequel.connect(db_url)
39
+ db.tables
40
+ db.disconnect
41
+ rescue Object => e
42
+ puts "Failed to connect to database:\n #{e.class} -> #{e}"
43
+ exit 1
44
+ end
45
+ end
46
+ end
47
+ end