taps 0.3.11 → 0.3.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/Rakefile +3 -3
- data/VERSION.yml +1 -1
- data/bin/schema +28 -28
- data/lib/taps/cli.rb +171 -166
- data/lib/taps/config.rb +39 -39
- data/lib/taps/data_stream.rb +291 -291
- data/lib/taps/db_session.rb +13 -13
- data/lib/taps/log.rb +12 -12
- data/lib/taps/monkey.rb +17 -17
- data/lib/taps/multipart.rb +51 -51
- data/lib/taps/operation.rb +525 -525
- data/lib/taps/schema.rb +58 -58
- data/lib/taps/server.rb +154 -154
- data/lib/taps/utils.rb +145 -145
- data/spec/base.rb +11 -11
- data/spec/cli_spec.rb +5 -5
- data/spec/data_stream_spec.rb +16 -16
- data/spec/operation_spec.rb +21 -21
- data/spec/server_spec.rb +26 -26
- data/spec/utils_spec.rb +49 -49
- metadata +5 -5
data/Rakefile
CHANGED
@@ -13,7 +13,7 @@ begin
|
|
13
13
|
s.add_dependency 'json_pure', '>= 1.2.0', '< 1.5.0'
|
14
14
|
s.add_dependency 'sinatra', '~> 1.0.0'
|
15
15
|
s.add_dependency 'rest-client', '~> 1.4.0'
|
16
|
-
s.add_dependency 'sequel', '~> 3.
|
16
|
+
s.add_dependency 'sequel', '~> 3.15.0'
|
17
17
|
s.add_dependency 'sqlite3-ruby', '~> 1.2'
|
18
18
|
s.add_dependency 'rack', '>= 1.0.1'
|
19
19
|
|
@@ -65,8 +65,8 @@ end
|
|
65
65
|
|
66
66
|
desc "copy/paste env vars for dev testing"
|
67
67
|
task :env do
|
68
|
-
|
69
|
-
|
68
|
+
puts "export RUBYLIB='#{File.dirname(__FILE__) + '/lib'}'"
|
69
|
+
puts "export RUBYOPT='-rrubygems'"
|
70
70
|
end
|
71
71
|
|
72
72
|
task :default => :spec
|
data/VERSION.yml
CHANGED
data/bin/schema
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
#!/usr/bin/env ruby
|
2
2
|
|
3
3
|
require 'rubygems'
|
4
|
-
gem 'sequel', '~> 3.
|
4
|
+
gem 'sequel', '~> 3.15.0'
|
5
5
|
|
6
6
|
$:.unshift File.dirname(__FILE__) + '/../lib'
|
7
7
|
|
@@ -11,44 +11,44 @@ cmd = ARGV.shift.strip rescue ''
|
|
11
11
|
database_url = ARGV.shift.strip rescue ''
|
12
12
|
|
13
13
|
def show_usage_and_exit
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
14
|
+
puts <<EOTXT
|
15
|
+
schema console <database_url>
|
16
|
+
schema dump <database_url>
|
17
|
+
schema dump_table <database_url> <table>
|
18
|
+
schema indexes <database_url>
|
19
|
+
schema indexes_individual <database_url>
|
20
|
+
schema reset_db_sequences <database_url>
|
21
|
+
schema load <database_url> <schema_file>
|
22
|
+
schema load_indexes <database_url> <indexes_file>
|
23
23
|
EOTXT
|
24
|
-
|
24
|
+
exit(1)
|
25
25
|
end
|
26
26
|
|
27
27
|
case cmd
|
28
28
|
when 'dump'
|
29
|
-
|
29
|
+
puts Taps::Schema.dump(database_url)
|
30
30
|
when 'dump_table'
|
31
|
-
|
32
|
-
|
31
|
+
table = ARGV.shift.strip
|
32
|
+
puts Taps::Schema.dump_table(database_url, table)
|
33
33
|
when 'indexes'
|
34
|
-
|
34
|
+
puts Taps::Schema.indexes(database_url)
|
35
35
|
when 'indexes_individual'
|
36
|
-
|
36
|
+
puts Taps::Schema.indexes_individual(database_url)
|
37
37
|
when 'load_indexes'
|
38
|
-
|
39
|
-
|
40
|
-
|
38
|
+
filename = ARGV.shift.strip rescue ''
|
39
|
+
indexes = File.read(filename) rescue show_usage_and_exit
|
40
|
+
Taps::Schema.load_indexes(database_url, indexes)
|
41
41
|
when 'load'
|
42
|
-
|
43
|
-
|
44
|
-
|
42
|
+
filename = ARGV.shift.strip rescue ''
|
43
|
+
schema = File.read(filename) rescue show_usage_and_exit
|
44
|
+
Taps::Schema.load(database_url, schema)
|
45
45
|
when 'reset_db_sequences'
|
46
|
-
|
46
|
+
Taps::Schema.reset_db_sequences(database_url)
|
47
47
|
when 'console'
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
|
48
|
+
$db = Sequel.connect(database_url)
|
49
|
+
require 'irb'
|
50
|
+
require 'irb/completion'
|
51
|
+
IRB.start
|
52
52
|
else
|
53
|
-
|
53
|
+
show_usage_and_exit
|
54
54
|
end
|
data/lib/taps/cli.rb
CHANGED
@@ -5,68 +5,73 @@ require 'taps/monkey'
|
|
5
5
|
require 'taps/config'
|
6
6
|
require 'taps/log'
|
7
7
|
|
8
|
-
Taps::Config.taps_database_url = ENV['TAPS_DATABASE_URL'] ||
|
8
|
+
Taps::Config.taps_database_url = ENV['TAPS_DATABASE_URL'] || begin
|
9
|
+
# this is dirty but it solves a weird problem where the tempfile disappears mid-process
|
10
|
+
$__taps_database = Tempfile.new('taps.db')
|
11
|
+
$__taps_database.open()
|
12
|
+
"sqlite://#{$__taps_database.path}"
|
13
|
+
end
|
9
14
|
|
10
15
|
module Taps
|
11
16
|
class Cli
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
|
17
|
+
attr_accessor :argv
|
18
|
+
|
19
|
+
def initialize(argv)
|
20
|
+
@argv = argv
|
21
|
+
end
|
22
|
+
|
23
|
+
def run
|
24
|
+
method = (argv.shift || 'help').to_sym
|
25
|
+
if [:pull, :push, :server, :version].include? method
|
26
|
+
send(method)
|
27
|
+
else
|
28
|
+
help
|
29
|
+
end
|
30
|
+
end
|
31
|
+
|
32
|
+
def pull
|
33
|
+
opts = clientoptparse(:pull)
|
34
|
+
Taps.log.level = Logger::DEBUG if opts[:debug]
|
35
|
+
if opts[:resume_filename]
|
36
|
+
clientresumexfer(:pull, opts)
|
37
|
+
else
|
38
|
+
clientxfer(:pull, opts)
|
39
|
+
end
|
40
|
+
end
|
41
|
+
|
42
|
+
def push
|
43
|
+
opts = clientoptparse(:push)
|
44
|
+
Taps.log.level = Logger::DEBUG if opts[:debug]
|
45
|
+
if opts[:resume_filename]
|
46
|
+
clientresumexfer(:push, opts)
|
47
|
+
else
|
48
|
+
clientxfer(:push, opts)
|
49
|
+
end
|
50
|
+
end
|
51
|
+
|
52
|
+
def server
|
53
|
+
opts = serveroptparse
|
54
|
+
Taps.log.level = Logger::DEBUG if opts[:debug]
|
55
|
+
Taps::Config.database_url = opts[:database_url]
|
56
|
+
Taps::Config.login = opts[:login]
|
57
|
+
Taps::Config.password = opts[:password]
|
58
|
+
|
59
|
+
Taps::Config.verify_database_url
|
60
|
+
require 'taps/server'
|
61
|
+
Taps::Server.run!({
|
62
|
+
:port => opts[:port],
|
63
|
+
:environment => :production,
|
64
|
+
:logging => true,
|
65
|
+
:dump_errors => true,
|
66
|
+
})
|
67
|
+
end
|
68
|
+
|
69
|
+
def version
|
70
|
+
puts Taps.version
|
71
|
+
end
|
72
|
+
|
73
|
+
def help
|
74
|
+
puts <<EOHELP
|
70
75
|
Options
|
71
76
|
=======
|
72
77
|
server Start a taps database import/export server
|
@@ -76,113 +81,113 @@ version Taps version
|
|
76
81
|
|
77
82
|
Add '-h' to any command to see their usage
|
78
83
|
EOHELP
|
79
|
-
|
80
|
-
|
81
|
-
|
82
|
-
|
83
|
-
|
84
|
-
|
85
|
-
|
86
|
-
|
87
|
-
|
88
|
-
|
89
|
-
|
90
|
-
|
91
|
-
|
92
|
-
|
93
|
-
|
94
|
-
|
95
|
-
|
96
|
-
|
97
|
-
|
98
|
-
|
99
|
-
|
100
|
-
|
101
|
-
|
102
|
-
|
103
|
-
|
104
|
-
|
105
|
-
|
106
|
-
|
107
|
-
|
108
|
-
|
109
|
-
|
110
|
-
|
111
|
-
|
112
|
-
|
113
|
-
|
114
|
-
|
115
|
-
|
116
|
-
|
117
|
-
|
118
|
-
|
119
|
-
|
120
|
-
|
121
|
-
|
122
|
-
|
123
|
-
|
124
|
-
|
125
|
-
|
126
|
-
|
127
|
-
|
128
|
-
|
129
|
-
|
130
|
-
|
131
|
-
|
132
|
-
|
133
|
-
|
134
|
-
|
135
|
-
|
136
|
-
|
137
|
-
|
138
|
-
|
139
|
-
|
140
|
-
|
141
|
-
|
142
|
-
|
143
|
-
|
144
|
-
|
145
|
-
|
146
|
-
|
147
|
-
|
148
|
-
|
149
|
-
|
150
|
-
|
151
|
-
|
152
|
-
|
153
|
-
|
154
|
-
|
155
|
-
|
156
|
-
|
157
|
-
|
158
|
-
|
159
|
-
|
160
|
-
|
161
|
-
|
162
|
-
|
163
|
-
|
164
|
-
|
165
|
-
|
166
|
-
|
167
|
-
|
168
|
-
|
169
|
-
|
170
|
-
|
171
|
-
|
172
|
-
|
173
|
-
|
174
|
-
|
175
|
-
|
176
|
-
|
177
|
-
|
178
|
-
|
179
|
-
|
180
|
-
|
181
|
-
|
182
|
-
|
183
|
-
|
184
|
-
|
185
|
-
|
84
|
+
end
|
85
|
+
|
86
|
+
def serveroptparse
|
87
|
+
opts={:port => 5000, :database_url => nil, :login => nil, :password => nil, :debug => false}
|
88
|
+
OptionParser.new do |o|
|
89
|
+
o.banner = "Usage: #{File.basename($0)} server [OPTIONS] <local_database_url> <login> <password>"
|
90
|
+
o.define_head "Start a taps database import/export server"
|
91
|
+
|
92
|
+
o.on("-p", "--port=N", "Server Port") { |v| opts[:port] = v.to_i if v.to_i > 0 }
|
93
|
+
o.on("-d", "--debug", "Enable Debug Messages") { |v| opts[:debug] = true }
|
94
|
+
o.parse!(argv)
|
95
|
+
|
96
|
+
opts[:database_url] = argv.shift
|
97
|
+
opts[:login] = argv.shift
|
98
|
+
opts[:password] = argv.shift
|
99
|
+
|
100
|
+
if opts[:database_url].nil?
|
101
|
+
$stderr.puts "Missing Database URL"
|
102
|
+
puts o
|
103
|
+
exit 1
|
104
|
+
end
|
105
|
+
if opts[:login].nil?
|
106
|
+
$stderr.puts "Missing Login"
|
107
|
+
puts o
|
108
|
+
exit 1
|
109
|
+
end
|
110
|
+
if opts[:password].nil?
|
111
|
+
$stderr.puts "Missing Password"
|
112
|
+
puts o
|
113
|
+
exit 1
|
114
|
+
end
|
115
|
+
end
|
116
|
+
opts
|
117
|
+
end
|
118
|
+
|
119
|
+
def clientoptparse(cmd)
|
120
|
+
opts={:default_chunksize => 1000, :database_url => nil, :remote_url => nil, :debug => false, :resume_filename => nil, :disable_compresion => false, :indexes_first => false}
|
121
|
+
OptionParser.new do |o|
|
122
|
+
o.banner = "Usage: #{File.basename($0)} #{cmd} [OPTIONS] <local_database_url> <remote_url>"
|
123
|
+
|
124
|
+
case cmd
|
125
|
+
when :pull
|
126
|
+
o.define_head "Pull a database from a taps server"
|
127
|
+
when :push
|
128
|
+
o.define_head "Push a database to a taps server"
|
129
|
+
end
|
130
|
+
|
131
|
+
o.on("-i", "--indexes-first", "Transfer indexes first before data") { |v| opts[:indexes_first] = true }
|
132
|
+
o.on("-r", "--resume=file", "Resume a Taps Session from a stored file") { |v| opts[:resume_filename] = v }
|
133
|
+
o.on("-c", "--chunksize=N", "Initial Chunksize") { |v| opts[:default_chunksize] = (v.to_i < 10 ? 10 : v.to_i) }
|
134
|
+
o.on("-g", "--disable-compression", "Disable Compression") { |v| opts[:disable_compression] = true }
|
135
|
+
o.on("-f", "--filter=regex", "Regex Filter for tables") { |v| opts[:table_filter] = v }
|
136
|
+
o.on("-t", "--tables=A,B,C", Array, "Shortcut to filter on a list of tables") do |v|
|
137
|
+
r_tables = v.collect { |t| "^#{t}$" }.join("|")
|
138
|
+
opts[:table_filter] = "(#{r_tables})"
|
139
|
+
end
|
140
|
+
o.on("-d", "--debug", "Enable Debug Messages") { |v| opts[:debug] = true }
|
141
|
+
o.parse!(argv)
|
142
|
+
|
143
|
+
opts[:database_url] = argv.shift
|
144
|
+
opts[:remote_url] = argv.shift
|
145
|
+
|
146
|
+
if opts[:database_url].nil?
|
147
|
+
$stderr.puts "Missing Database URL"
|
148
|
+
puts o
|
149
|
+
exit 1
|
150
|
+
end
|
151
|
+
if opts[:remote_url].nil?
|
152
|
+
$stderr.puts "Missing Remote Taps URL"
|
153
|
+
puts o
|
154
|
+
exit 1
|
155
|
+
end
|
156
|
+
end
|
157
|
+
|
158
|
+
opts
|
159
|
+
end
|
160
|
+
|
161
|
+
def clientxfer(method, opts)
|
162
|
+
database_url = opts.delete(:database_url)
|
163
|
+
remote_url = opts.delete(:remote_url)
|
164
|
+
|
165
|
+
Taps::Config.verify_database_url(database_url)
|
166
|
+
|
167
|
+
require 'taps/operation'
|
168
|
+
|
169
|
+
Taps::Operation.factory(method, database_url, remote_url, opts).run
|
170
|
+
end
|
171
|
+
|
172
|
+
def clientresumexfer(method, opts)
|
173
|
+
session = JSON.parse(File.read(opts.delete(:resume_filename)))
|
174
|
+
session.symbolize_recursively!
|
175
|
+
|
176
|
+
database_url = opts.delete(:database_url)
|
177
|
+
remote_url = opts.delete(:remote_url) || session.delete(:remote_url)
|
178
|
+
|
179
|
+
Taps::Config.verify_database_url(database_url)
|
180
|
+
|
181
|
+
require 'taps/operation'
|
182
|
+
|
183
|
+
newsession = session.merge({
|
184
|
+
:default_chunksize => opts[:default_chunksize],
|
185
|
+
:disable_compression => opts[:disable_compression],
|
186
|
+
:resume => true,
|
187
|
+
})
|
188
|
+
|
189
|
+
Taps::Operation.factory(method, database_url, remote_url, newsession).run
|
190
|
+
end
|
186
191
|
|
187
192
|
end
|
188
193
|
end
|