taps2 0.5.5 → 0.6.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/{README.rdoc → README.md} +29 -22
- data/bin/{schema → schema2} +17 -8
- data/bin/{schema.cmd → schema2.cmd} +0 -0
- data/bin/{taps → taps2} +1 -1
- data/lib/taps/chunksize.rb +16 -12
- data/lib/taps/cli.rb +33 -38
- data/lib/taps/config.rb +2 -2
- data/lib/taps/data_stream.rb +256 -262
- data/lib/taps/errors.rb +1 -1
- data/lib/taps/log.rb +1 -1
- data/lib/taps/monkey.rb +10 -9
- data/lib/taps/multipart.rb +1 -2
- data/lib/taps/operation.rb +81 -86
- data/lib/taps/progress_bar.rb +42 -45
- data/lib/taps/schema.rb +2 -2
- data/lib/taps/server.rb +37 -38
- data/lib/taps/utils.rb +31 -34
- data/lib/taps/version.rb +13 -13
- data/lib/vendor/okjson.rb +115 -161
- data/spec/base.rb +2 -2
- data/spec/chunksize_spec.rb +6 -6
- data/spec/cli_spec.rb +6 -6
- data/spec/data_stream_spec.rb +4 -4
- data/spec/operation_spec.rb +17 -18
- data/spec/server_spec.rb +7 -7
- data/spec/utils_spec.rb +20 -20
- metadata +38 -39
- data/VERSION.yml +0 -5
data/lib/taps/errors.rb
CHANGED
data/lib/taps/log.rb
CHANGED
data/lib/taps/monkey.rb
CHANGED
@@ -1,21 +1,22 @@
|
|
1
1
|
class Hash
|
2
2
|
def symbolize_keys
|
3
|
-
|
4
|
-
options[(
|
5
|
-
|
3
|
+
each_with_object({}) do |(key, value), options|
|
4
|
+
options[(begin
|
5
|
+
key.to_sym
|
6
|
+
rescue
|
7
|
+
key
|
8
|
+
end) || key] = value
|
6
9
|
end
|
7
10
|
end
|
8
11
|
|
9
12
|
def symbolize_keys!
|
10
|
-
|
13
|
+
replace(symbolize_keys)
|
11
14
|
end
|
12
15
|
|
13
16
|
def symbolize_recursively!
|
14
|
-
|
15
|
-
|
16
|
-
if v.
|
17
|
-
v.symbolize_keys!
|
18
|
-
end
|
17
|
+
replace(symbolize_keys)
|
18
|
+
each do |_k, v|
|
19
|
+
v.symbolize_keys! if v.is_a?(Hash)
|
19
20
|
end
|
20
21
|
end
|
21
22
|
end
|
data/lib/taps/multipart.rb
CHANGED
data/lib/taps/operation.rb
CHANGED
@@ -16,7 +16,7 @@ module Taps
|
|
16
16
|
attr_reader :database_url, :remote_url, :opts
|
17
17
|
attr_reader :session_uri
|
18
18
|
|
19
|
-
def initialize(database_url, remote_url, opts={})
|
19
|
+
def initialize(database_url, remote_url, opts = {})
|
20
20
|
@database_url = database_url
|
21
21
|
@remote_url = remote_url
|
22
22
|
@opts = opts
|
@@ -25,7 +25,7 @@ module Taps
|
|
25
25
|
end
|
26
26
|
|
27
27
|
def file_prefix
|
28
|
-
|
28
|
+
'op'
|
29
29
|
end
|
30
30
|
|
31
31
|
def skip_schema?
|
@@ -48,7 +48,7 @@ module Taps
|
|
48
48
|
return tables unless table_filter || exclude_tables
|
49
49
|
|
50
50
|
re = table_filter ? Regexp.new(table_filter) : nil
|
51
|
-
if tables.
|
51
|
+
if tables.is_a?(Hash)
|
52
52
|
ntables = {}
|
53
53
|
tables.each do |t, d|
|
54
54
|
if !exclude_tables.include?(t.to_s) && (!re || !re.match(t.to_s).nil?)
|
@@ -66,7 +66,7 @@ module Taps
|
|
66
66
|
end
|
67
67
|
|
68
68
|
def store_session
|
69
|
-
file = "#{file_prefix}_#{Time.now.strftime(
|
69
|
+
file = "#{file_prefix}_#{Time.now.strftime('%Y%m%d%H%M')}.dat"
|
70
70
|
puts "\nSaving session to #{file}.."
|
71
71
|
File.open(file, 'w') do |f|
|
72
72
|
f.write(::OkJson.encode(to_hash))
|
@@ -75,13 +75,13 @@ module Taps
|
|
75
75
|
|
76
76
|
def to_hash
|
77
77
|
{
|
78
|
-
:
|
79
|
-
:
|
80
|
-
:
|
81
|
-
:
|
82
|
-
:
|
83
|
-
:
|
84
|
-
:
|
78
|
+
klass: self.class.to_s,
|
79
|
+
database_url: database_url,
|
80
|
+
remote_url: remote_url,
|
81
|
+
session_uri: session_uri,
|
82
|
+
stream_state: stream_state,
|
83
|
+
completed_tables: completed_tables,
|
84
|
+
table_filter: table_filter
|
85
85
|
}
|
86
86
|
end
|
87
87
|
|
@@ -90,15 +90,15 @@ module Taps
|
|
90
90
|
end
|
91
91
|
|
92
92
|
def setup_signal_trap
|
93
|
-
trap(
|
93
|
+
trap('INT') do
|
94
94
|
puts "\nCompleting current action..."
|
95
95
|
@exiting = true
|
96
|
-
|
96
|
+
end
|
97
97
|
|
98
|
-
trap(
|
98
|
+
trap('TERM') do
|
99
99
|
puts "\nCompleting current action..."
|
100
100
|
@exiting = true
|
101
|
-
|
101
|
+
end
|
102
102
|
end
|
103
103
|
|
104
104
|
def resuming?
|
@@ -162,56 +162,54 @@ module Taps
|
|
162
162
|
end
|
163
163
|
|
164
164
|
def http_headers(extra = {})
|
165
|
-
base = { :
|
166
|
-
if compression_disabled?
|
167
|
-
|
168
|
-
|
169
|
-
|
170
|
-
|
165
|
+
base = { taps_version: Taps.version }
|
166
|
+
base[:accept_encoding] = if compression_disabled?
|
167
|
+
''
|
168
|
+
else
|
169
|
+
'gzip, deflate'
|
170
|
+
end
|
171
171
|
base.merge(extra)
|
172
172
|
end
|
173
173
|
|
174
174
|
def format_number(num)
|
175
|
-
num.to_s.gsub(/(\d)(?=(\d\d\d)+(?!\d))/,
|
175
|
+
num.to_s.gsub(/(\d)(?=(\d\d\d)+(?!\d))/, '\\1,')
|
176
176
|
end
|
177
177
|
|
178
178
|
def verify_server
|
179
|
-
|
180
|
-
|
181
|
-
|
182
|
-
|
183
|
-
|
184
|
-
puts "#{e.response.to_s}"
|
185
|
-
exit(1)
|
186
|
-
else
|
187
|
-
raise
|
188
|
-
end
|
189
|
-
rescue RestClient::Unauthorized
|
190
|
-
puts "Bad credentials given for #{safe_remote_url}"
|
191
|
-
exit(1)
|
192
|
-
rescue Errno::ECONNREFUSED
|
193
|
-
puts "Can't connect to #{safe_remote_url}. Please check that it's running"
|
179
|
+
server['/'].get(http_headers)
|
180
|
+
rescue RestClient::RequestFailed => e
|
181
|
+
if e.http_code == 417
|
182
|
+
puts "#{safe_remote_url} is running a different minor version of taps."
|
183
|
+
puts e.response.to_s
|
194
184
|
exit(1)
|
185
|
+
else
|
186
|
+
raise
|
195
187
|
end
|
188
|
+
rescue RestClient::Unauthorized
|
189
|
+
puts "Bad credentials given for #{safe_remote_url}"
|
190
|
+
exit(1)
|
191
|
+
rescue Errno::ECONNREFUSED
|
192
|
+
puts "Can't connect to #{safe_remote_url}. Please check that it's running"
|
193
|
+
exit(1)
|
196
194
|
end
|
197
195
|
|
198
|
-
def catch_errors
|
196
|
+
def catch_errors
|
199
197
|
verify_server
|
200
198
|
|
201
199
|
begin
|
202
|
-
|
200
|
+
yield
|
203
201
|
close_session
|
204
202
|
rescue RestClient::Exception, Taps::BaseError => e
|
205
203
|
store_session
|
206
|
-
if e.
|
207
|
-
puts
|
204
|
+
if e.is_a?(Taps::BaseError)
|
205
|
+
puts '!!! Caught Server Exception'
|
208
206
|
puts "#{e.class}: #{e.message}"
|
209
207
|
puts "\n#{e.original_backtrace}" if e.original_backtrace
|
210
208
|
exit(1)
|
211
209
|
elsif e.respond_to?(:response)
|
212
|
-
puts
|
210
|
+
puts '!!! Caught Server Exception'
|
213
211
|
puts "HTTP CODE: #{e.http_code}"
|
214
|
-
puts
|
212
|
+
puts e.response.to_s
|
215
213
|
exit(1)
|
216
214
|
else
|
217
215
|
raise
|
@@ -222,10 +220,10 @@ module Taps
|
|
222
220
|
def self.factory(type, database_url, remote_url, opts)
|
223
221
|
type = :resume if opts[:resume]
|
224
222
|
klass = case type
|
225
|
-
|
226
|
-
|
227
|
-
|
228
|
-
|
223
|
+
when :pull then Taps::Pull
|
224
|
+
when :push then Taps::Push
|
225
|
+
when :resume then eval(opts[:klass])
|
226
|
+
else raise "Unknown Operation Type -> #{type}"
|
229
227
|
end
|
230
228
|
|
231
229
|
klass.new(database_url, remote_url, opts)
|
@@ -234,17 +232,17 @@ module Taps
|
|
234
232
|
|
235
233
|
class Pull < Operation
|
236
234
|
def file_prefix
|
237
|
-
|
235
|
+
'pull'
|
238
236
|
end
|
239
237
|
|
240
238
|
def to_hash
|
241
|
-
super.merge(:
|
239
|
+
super.merge(remote_tables_info: remote_tables_info)
|
242
240
|
end
|
243
241
|
|
244
242
|
def run
|
245
243
|
catch_errors do
|
246
244
|
unless resuming?
|
247
|
-
pull_schema
|
245
|
+
pull_schema unless skip_schema?
|
248
246
|
pull_indexes if indexes_first? && !skip_schema?
|
249
247
|
end
|
250
248
|
setup_signal_trap
|
@@ -256,11 +254,11 @@ module Taps
|
|
256
254
|
end
|
257
255
|
|
258
256
|
def pull_schema
|
259
|
-
puts
|
257
|
+
puts 'Receiving schema'
|
260
258
|
|
261
259
|
progress = ProgressBar.new('Schema', tables.size)
|
262
|
-
tables.each do |table_name,
|
263
|
-
schema_data = session_resource['pull/schema'].post({:table_name
|
260
|
+
tables.each do |table_name, _count|
|
261
|
+
schema_data = session_resource['pull/schema'].post({ table_name: table_name }, http_headers).to_s
|
264
262
|
log.debug "Table: #{table_name}\n#{schema_data}\n"
|
265
263
|
output = Taps::Utils.load_schema(database_url, schema_data)
|
266
264
|
output = output.to_s.strip
|
@@ -271,16 +269,14 @@ module Taps
|
|
271
269
|
end
|
272
270
|
|
273
271
|
def pull_data
|
274
|
-
puts
|
272
|
+
puts 'Receiving data'
|
275
273
|
|
276
274
|
puts "#{tables.size} tables, #{format_number(record_count)} records"
|
277
275
|
|
278
276
|
tables.each do |table_name, count|
|
279
277
|
progress = ProgressBar.new(table_name.to_s, count)
|
280
|
-
stream = Taps::DataStream.factory(db,
|
281
|
-
|
282
|
-
:table_name => table_name
|
283
|
-
})
|
278
|
+
stream = Taps::DataStream.factory(db, chunksize: default_chunksize,
|
279
|
+
table_name: table_name)
|
284
280
|
pull_data_from_table(stream, progress)
|
285
281
|
end
|
286
282
|
end
|
@@ -332,7 +328,7 @@ module Taps
|
|
332
328
|
end
|
333
329
|
|
334
330
|
def record_count
|
335
|
-
@record_count ||= remote_tables_info.values.inject(0) { |a,c| a += c }
|
331
|
+
@record_count ||= remote_tables_info.values.inject(0) { |a, c| a += c }
|
336
332
|
end
|
337
333
|
|
338
334
|
def remote_tables_info
|
@@ -355,7 +351,7 @@ module Taps
|
|
355
351
|
apply_table_filter(tables).each do |table_name|
|
356
352
|
retries = 0
|
357
353
|
begin
|
358
|
-
count = Integer(session_resource['pull/table_count'].post({:
|
354
|
+
count = Integer(session_resource['pull/table_count'].post({ table: table_name }, http_headers).to_s)
|
359
355
|
data[table_name] = count
|
360
356
|
rescue RestClient::Exception
|
361
357
|
retries += 1
|
@@ -368,12 +364,12 @@ module Taps
|
|
368
364
|
end
|
369
365
|
|
370
366
|
def pull_indexes
|
371
|
-
puts
|
367
|
+
puts 'Receiving indexes'
|
372
368
|
|
373
369
|
idxs = ::OkJson.decode(session_resource['pull/indexes'].get(http_headers).to_s)
|
374
370
|
|
375
371
|
apply_table_filter(idxs).each do |table, indexes|
|
376
|
-
next
|
372
|
+
next if indexes.empty?
|
377
373
|
progress = ProgressBar.new(table, indexes.size)
|
378
374
|
indexes.each do |idx|
|
379
375
|
output = Taps::Utils.load_indexes(database_url, idx)
|
@@ -386,7 +382,7 @@ module Taps
|
|
386
382
|
end
|
387
383
|
|
388
384
|
def pull_reset_sequences
|
389
|
-
puts
|
385
|
+
puts 'Resetting sequences'
|
390
386
|
|
391
387
|
output = Taps::Utils.schema_bin(:reset_db_sequences, database_url)
|
392
388
|
output = output.to_s.strip
|
@@ -396,17 +392,17 @@ module Taps
|
|
396
392
|
|
397
393
|
class Push < Operation
|
398
394
|
def file_prefix
|
399
|
-
|
395
|
+
'push'
|
400
396
|
end
|
401
397
|
|
402
398
|
def to_hash
|
403
|
-
super.merge(:
|
399
|
+
super.merge(local_tables_info: local_tables_info)
|
404
400
|
end
|
405
401
|
|
406
402
|
def run
|
407
403
|
catch_errors do
|
408
404
|
unless resuming?
|
409
|
-
push_schema
|
405
|
+
push_schema unless skip_schema?
|
410
406
|
push_indexes if indexes_first? && !skip_schema?
|
411
407
|
end
|
412
408
|
setup_signal_trap
|
@@ -420,12 +416,12 @@ module Taps
|
|
420
416
|
def push_indexes
|
421
417
|
idxs = ::OkJson.decode(Taps::Utils.schema_bin(:indexes_individual, database_url))
|
422
418
|
|
423
|
-
return
|
419
|
+
return if idxs.empty?
|
424
420
|
|
425
|
-
puts
|
421
|
+
puts 'Sending indexes'
|
426
422
|
|
427
423
|
apply_table_filter(idxs).each do |table, indexes|
|
428
|
-
next
|
424
|
+
next if indexes.empty?
|
429
425
|
progress = ProgressBar.new(table, indexes.size)
|
430
426
|
indexes.each do |idx|
|
431
427
|
session_resource['push/indexes'].post(idx, http_headers)
|
@@ -436,10 +432,10 @@ module Taps
|
|
436
432
|
end
|
437
433
|
|
438
434
|
def push_schema
|
439
|
-
puts
|
435
|
+
puts 'Sending schema'
|
440
436
|
|
441
437
|
progress = ProgressBar.new('Schema', tables.size)
|
442
|
-
tables.each do |table,
|
438
|
+
tables.each do |table, _count|
|
443
439
|
schema_data = Taps::Utils.schema_bin(:dump_table, database_url, table)
|
444
440
|
log.debug "Table: #{table}\n#{schema_data}\n"
|
445
441
|
session_resource['push/schema'].post(schema_data, http_headers)
|
@@ -449,7 +445,7 @@ module Taps
|
|
449
445
|
end
|
450
446
|
|
451
447
|
def push_reset_sequences
|
452
|
-
puts
|
448
|
+
puts 'Resetting sequences'
|
453
449
|
|
454
450
|
session_resource['push/reset_sequences'].post('', http_headers)
|
455
451
|
end
|
@@ -466,14 +462,14 @@ module Taps
|
|
466
462
|
end
|
467
463
|
|
468
464
|
def push_data
|
469
|
-
puts
|
465
|
+
puts 'Sending data'
|
470
466
|
|
471
467
|
puts "#{tables.size} tables, #{format_number(record_count)} records"
|
472
468
|
|
473
469
|
tables.each do |table_name, count|
|
474
470
|
stream = Taps::DataStream.factory(db,
|
475
|
-
|
476
|
-
|
471
|
+
table_name: table_name,
|
472
|
+
chunksize: default_chunksize)
|
477
473
|
progress = ProgressBar.new(table_name.to_s, count)
|
478
474
|
push_data_from_table(stream, progress)
|
479
475
|
end
|
@@ -501,8 +497,8 @@ module Taps
|
|
501
497
|
data = nil
|
502
498
|
d2 = c.time_delta do
|
503
499
|
data = {
|
504
|
-
:
|
505
|
-
:
|
500
|
+
state: stream.to_hash,
|
501
|
+
checksum: Taps::Utils.checksum(encoded_data).to_s
|
506
502
|
}
|
507
503
|
end
|
508
504
|
|
@@ -510,15 +506,15 @@ module Taps
|
|
510
506
|
content, content_type = nil
|
511
507
|
d3 = c.time_delta do
|
512
508
|
content, content_type = Taps::Multipart.create do |r|
|
513
|
-
r.attach :
|
514
|
-
|
515
|
-
|
516
|
-
r.attach :
|
517
|
-
|
518
|
-
|
509
|
+
r.attach name: :encoded_data,
|
510
|
+
payload: encoded_data,
|
511
|
+
content_type: 'application/octet-stream'
|
512
|
+
r.attach name: :json,
|
513
|
+
payload: ::OkJson.encode(data),
|
514
|
+
content_type: 'application/json'
|
519
515
|
end
|
520
516
|
end
|
521
|
-
session_resource['push/table'].post(content, http_headers(:
|
517
|
+
session_resource['push/table'].post(content, http_headers(content_type: content_type))
|
522
518
|
self.stream_state = stream.to_hash
|
523
519
|
rescue => e
|
524
520
|
Taps::Utils.reraise_server_exception(e)
|
@@ -563,7 +559,7 @@ module Taps
|
|
563
559
|
end
|
564
560
|
|
565
561
|
def record_count
|
566
|
-
@record_count ||= local_tables_info.values.inject(0) { |a,c| a += c }
|
562
|
+
@record_count ||= local_tables_info.values.inject(0) { |a, c| a += c }
|
567
563
|
end
|
568
564
|
|
569
565
|
def fetch_local_tables_info
|
@@ -573,6 +569,5 @@ module Taps
|
|
573
569
|
end
|
574
570
|
apply_table_filter(tables_with_counts)
|
575
571
|
end
|
576
|
-
|
577
572
|
end
|
578
573
|
end
|