taps2 0.5.5 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/{README.rdoc → README.md} +29 -22
- data/bin/{schema → schema2} +17 -8
- data/bin/{schema.cmd → schema2.cmd} +0 -0
- data/bin/{taps → taps2} +1 -1
- data/lib/taps/chunksize.rb +16 -12
- data/lib/taps/cli.rb +33 -38
- data/lib/taps/config.rb +2 -2
- data/lib/taps/data_stream.rb +256 -262
- data/lib/taps/errors.rb +1 -1
- data/lib/taps/log.rb +1 -1
- data/lib/taps/monkey.rb +10 -9
- data/lib/taps/multipart.rb +1 -2
- data/lib/taps/operation.rb +81 -86
- data/lib/taps/progress_bar.rb +42 -45
- data/lib/taps/schema.rb +2 -2
- data/lib/taps/server.rb +37 -38
- data/lib/taps/utils.rb +31 -34
- data/lib/taps/version.rb +13 -13
- data/lib/vendor/okjson.rb +115 -161
- data/spec/base.rb +2 -2
- data/spec/chunksize_spec.rb +6 -6
- data/spec/cli_spec.rb +6 -6
- data/spec/data_stream_spec.rb +4 -4
- data/spec/operation_spec.rb +17 -18
- data/spec/server_spec.rb +7 -7
- data/spec/utils_spec.rb +20 -20
- metadata +38 -39
- data/VERSION.yml +0 -5
data/lib/taps/errors.rb
CHANGED
data/lib/taps/log.rb
CHANGED
data/lib/taps/monkey.rb
CHANGED
@@ -1,21 +1,22 @@
|
|
1
1
|
class Hash
|
2
2
|
def symbolize_keys
|
3
|
-
|
4
|
-
options[(
|
5
|
-
|
3
|
+
each_with_object({}) do |(key, value), options|
|
4
|
+
options[(begin
|
5
|
+
key.to_sym
|
6
|
+
rescue
|
7
|
+
key
|
8
|
+
end) || key] = value
|
6
9
|
end
|
7
10
|
end
|
8
11
|
|
9
12
|
def symbolize_keys!
|
10
|
-
|
13
|
+
replace(symbolize_keys)
|
11
14
|
end
|
12
15
|
|
13
16
|
def symbolize_recursively!
|
14
|
-
|
15
|
-
|
16
|
-
if v.
|
17
|
-
v.symbolize_keys!
|
18
|
-
end
|
17
|
+
replace(symbolize_keys)
|
18
|
+
each do |_k, v|
|
19
|
+
v.symbolize_keys! if v.is_a?(Hash)
|
19
20
|
end
|
20
21
|
end
|
21
22
|
end
|
data/lib/taps/multipart.rb
CHANGED
data/lib/taps/operation.rb
CHANGED
@@ -16,7 +16,7 @@ module Taps
|
|
16
16
|
attr_reader :database_url, :remote_url, :opts
|
17
17
|
attr_reader :session_uri
|
18
18
|
|
19
|
-
def initialize(database_url, remote_url, opts={})
|
19
|
+
def initialize(database_url, remote_url, opts = {})
|
20
20
|
@database_url = database_url
|
21
21
|
@remote_url = remote_url
|
22
22
|
@opts = opts
|
@@ -25,7 +25,7 @@ module Taps
|
|
25
25
|
end
|
26
26
|
|
27
27
|
def file_prefix
|
28
|
-
|
28
|
+
'op'
|
29
29
|
end
|
30
30
|
|
31
31
|
def skip_schema?
|
@@ -48,7 +48,7 @@ module Taps
|
|
48
48
|
return tables unless table_filter || exclude_tables
|
49
49
|
|
50
50
|
re = table_filter ? Regexp.new(table_filter) : nil
|
51
|
-
if tables.
|
51
|
+
if tables.is_a?(Hash)
|
52
52
|
ntables = {}
|
53
53
|
tables.each do |t, d|
|
54
54
|
if !exclude_tables.include?(t.to_s) && (!re || !re.match(t.to_s).nil?)
|
@@ -66,7 +66,7 @@ module Taps
|
|
66
66
|
end
|
67
67
|
|
68
68
|
def store_session
|
69
|
-
file = "#{file_prefix}_#{Time.now.strftime(
|
69
|
+
file = "#{file_prefix}_#{Time.now.strftime('%Y%m%d%H%M')}.dat"
|
70
70
|
puts "\nSaving session to #{file}.."
|
71
71
|
File.open(file, 'w') do |f|
|
72
72
|
f.write(::OkJson.encode(to_hash))
|
@@ -75,13 +75,13 @@ module Taps
|
|
75
75
|
|
76
76
|
def to_hash
|
77
77
|
{
|
78
|
-
:
|
79
|
-
:
|
80
|
-
:
|
81
|
-
:
|
82
|
-
:
|
83
|
-
:
|
84
|
-
:
|
78
|
+
klass: self.class.to_s,
|
79
|
+
database_url: database_url,
|
80
|
+
remote_url: remote_url,
|
81
|
+
session_uri: session_uri,
|
82
|
+
stream_state: stream_state,
|
83
|
+
completed_tables: completed_tables,
|
84
|
+
table_filter: table_filter
|
85
85
|
}
|
86
86
|
end
|
87
87
|
|
@@ -90,15 +90,15 @@ module Taps
|
|
90
90
|
end
|
91
91
|
|
92
92
|
def setup_signal_trap
|
93
|
-
trap(
|
93
|
+
trap('INT') do
|
94
94
|
puts "\nCompleting current action..."
|
95
95
|
@exiting = true
|
96
|
-
|
96
|
+
end
|
97
97
|
|
98
|
-
trap(
|
98
|
+
trap('TERM') do
|
99
99
|
puts "\nCompleting current action..."
|
100
100
|
@exiting = true
|
101
|
-
|
101
|
+
end
|
102
102
|
end
|
103
103
|
|
104
104
|
def resuming?
|
@@ -162,56 +162,54 @@ module Taps
|
|
162
162
|
end
|
163
163
|
|
164
164
|
def http_headers(extra = {})
|
165
|
-
base = { :
|
166
|
-
if compression_disabled?
|
167
|
-
|
168
|
-
|
169
|
-
|
170
|
-
|
165
|
+
base = { taps_version: Taps.version }
|
166
|
+
base[:accept_encoding] = if compression_disabled?
|
167
|
+
''
|
168
|
+
else
|
169
|
+
'gzip, deflate'
|
170
|
+
end
|
171
171
|
base.merge(extra)
|
172
172
|
end
|
173
173
|
|
174
174
|
def format_number(num)
|
175
|
-
num.to_s.gsub(/(\d)(?=(\d\d\d)+(?!\d))/,
|
175
|
+
num.to_s.gsub(/(\d)(?=(\d\d\d)+(?!\d))/, '\\1,')
|
176
176
|
end
|
177
177
|
|
178
178
|
def verify_server
|
179
|
-
|
180
|
-
|
181
|
-
|
182
|
-
|
183
|
-
|
184
|
-
puts "#{e.response.to_s}"
|
185
|
-
exit(1)
|
186
|
-
else
|
187
|
-
raise
|
188
|
-
end
|
189
|
-
rescue RestClient::Unauthorized
|
190
|
-
puts "Bad credentials given for #{safe_remote_url}"
|
191
|
-
exit(1)
|
192
|
-
rescue Errno::ECONNREFUSED
|
193
|
-
puts "Can't connect to #{safe_remote_url}. Please check that it's running"
|
179
|
+
server['/'].get(http_headers)
|
180
|
+
rescue RestClient::RequestFailed => e
|
181
|
+
if e.http_code == 417
|
182
|
+
puts "#{safe_remote_url} is running a different minor version of taps."
|
183
|
+
puts e.response.to_s
|
194
184
|
exit(1)
|
185
|
+
else
|
186
|
+
raise
|
195
187
|
end
|
188
|
+
rescue RestClient::Unauthorized
|
189
|
+
puts "Bad credentials given for #{safe_remote_url}"
|
190
|
+
exit(1)
|
191
|
+
rescue Errno::ECONNREFUSED
|
192
|
+
puts "Can't connect to #{safe_remote_url}. Please check that it's running"
|
193
|
+
exit(1)
|
196
194
|
end
|
197
195
|
|
198
|
-
def catch_errors
|
196
|
+
def catch_errors
|
199
197
|
verify_server
|
200
198
|
|
201
199
|
begin
|
202
|
-
|
200
|
+
yield
|
203
201
|
close_session
|
204
202
|
rescue RestClient::Exception, Taps::BaseError => e
|
205
203
|
store_session
|
206
|
-
if e.
|
207
|
-
puts
|
204
|
+
if e.is_a?(Taps::BaseError)
|
205
|
+
puts '!!! Caught Server Exception'
|
208
206
|
puts "#{e.class}: #{e.message}"
|
209
207
|
puts "\n#{e.original_backtrace}" if e.original_backtrace
|
210
208
|
exit(1)
|
211
209
|
elsif e.respond_to?(:response)
|
212
|
-
puts
|
210
|
+
puts '!!! Caught Server Exception'
|
213
211
|
puts "HTTP CODE: #{e.http_code}"
|
214
|
-
puts
|
212
|
+
puts e.response.to_s
|
215
213
|
exit(1)
|
216
214
|
else
|
217
215
|
raise
|
@@ -222,10 +220,10 @@ module Taps
|
|
222
220
|
def self.factory(type, database_url, remote_url, opts)
|
223
221
|
type = :resume if opts[:resume]
|
224
222
|
klass = case type
|
225
|
-
|
226
|
-
|
227
|
-
|
228
|
-
|
223
|
+
when :pull then Taps::Pull
|
224
|
+
when :push then Taps::Push
|
225
|
+
when :resume then eval(opts[:klass])
|
226
|
+
else raise "Unknown Operation Type -> #{type}"
|
229
227
|
end
|
230
228
|
|
231
229
|
klass.new(database_url, remote_url, opts)
|
@@ -234,17 +232,17 @@ module Taps
|
|
234
232
|
|
235
233
|
class Pull < Operation
|
236
234
|
def file_prefix
|
237
|
-
|
235
|
+
'pull'
|
238
236
|
end
|
239
237
|
|
240
238
|
def to_hash
|
241
|
-
super.merge(:
|
239
|
+
super.merge(remote_tables_info: remote_tables_info)
|
242
240
|
end
|
243
241
|
|
244
242
|
def run
|
245
243
|
catch_errors do
|
246
244
|
unless resuming?
|
247
|
-
pull_schema
|
245
|
+
pull_schema unless skip_schema?
|
248
246
|
pull_indexes if indexes_first? && !skip_schema?
|
249
247
|
end
|
250
248
|
setup_signal_trap
|
@@ -256,11 +254,11 @@ module Taps
|
|
256
254
|
end
|
257
255
|
|
258
256
|
def pull_schema
|
259
|
-
puts
|
257
|
+
puts 'Receiving schema'
|
260
258
|
|
261
259
|
progress = ProgressBar.new('Schema', tables.size)
|
262
|
-
tables.each do |table_name,
|
263
|
-
schema_data = session_resource['pull/schema'].post({:table_name
|
260
|
+
tables.each do |table_name, _count|
|
261
|
+
schema_data = session_resource['pull/schema'].post({ table_name: table_name }, http_headers).to_s
|
264
262
|
log.debug "Table: #{table_name}\n#{schema_data}\n"
|
265
263
|
output = Taps::Utils.load_schema(database_url, schema_data)
|
266
264
|
output = output.to_s.strip
|
@@ -271,16 +269,14 @@ module Taps
|
|
271
269
|
end
|
272
270
|
|
273
271
|
def pull_data
|
274
|
-
puts
|
272
|
+
puts 'Receiving data'
|
275
273
|
|
276
274
|
puts "#{tables.size} tables, #{format_number(record_count)} records"
|
277
275
|
|
278
276
|
tables.each do |table_name, count|
|
279
277
|
progress = ProgressBar.new(table_name.to_s, count)
|
280
|
-
stream = Taps::DataStream.factory(db,
|
281
|
-
|
282
|
-
:table_name => table_name
|
283
|
-
})
|
278
|
+
stream = Taps::DataStream.factory(db, chunksize: default_chunksize,
|
279
|
+
table_name: table_name)
|
284
280
|
pull_data_from_table(stream, progress)
|
285
281
|
end
|
286
282
|
end
|
@@ -332,7 +328,7 @@ module Taps
|
|
332
328
|
end
|
333
329
|
|
334
330
|
def record_count
|
335
|
-
@record_count ||= remote_tables_info.values.inject(0) { |a,c| a += c }
|
331
|
+
@record_count ||= remote_tables_info.values.inject(0) { |a, c| a += c }
|
336
332
|
end
|
337
333
|
|
338
334
|
def remote_tables_info
|
@@ -355,7 +351,7 @@ module Taps
|
|
355
351
|
apply_table_filter(tables).each do |table_name|
|
356
352
|
retries = 0
|
357
353
|
begin
|
358
|
-
count = Integer(session_resource['pull/table_count'].post({:
|
354
|
+
count = Integer(session_resource['pull/table_count'].post({ table: table_name }, http_headers).to_s)
|
359
355
|
data[table_name] = count
|
360
356
|
rescue RestClient::Exception
|
361
357
|
retries += 1
|
@@ -368,12 +364,12 @@ module Taps
|
|
368
364
|
end
|
369
365
|
|
370
366
|
def pull_indexes
|
371
|
-
puts
|
367
|
+
puts 'Receiving indexes'
|
372
368
|
|
373
369
|
idxs = ::OkJson.decode(session_resource['pull/indexes'].get(http_headers).to_s)
|
374
370
|
|
375
371
|
apply_table_filter(idxs).each do |table, indexes|
|
376
|
-
next
|
372
|
+
next if indexes.empty?
|
377
373
|
progress = ProgressBar.new(table, indexes.size)
|
378
374
|
indexes.each do |idx|
|
379
375
|
output = Taps::Utils.load_indexes(database_url, idx)
|
@@ -386,7 +382,7 @@ module Taps
|
|
386
382
|
end
|
387
383
|
|
388
384
|
def pull_reset_sequences
|
389
|
-
puts
|
385
|
+
puts 'Resetting sequences'
|
390
386
|
|
391
387
|
output = Taps::Utils.schema_bin(:reset_db_sequences, database_url)
|
392
388
|
output = output.to_s.strip
|
@@ -396,17 +392,17 @@ module Taps
|
|
396
392
|
|
397
393
|
class Push < Operation
|
398
394
|
def file_prefix
|
399
|
-
|
395
|
+
'push'
|
400
396
|
end
|
401
397
|
|
402
398
|
def to_hash
|
403
|
-
super.merge(:
|
399
|
+
super.merge(local_tables_info: local_tables_info)
|
404
400
|
end
|
405
401
|
|
406
402
|
def run
|
407
403
|
catch_errors do
|
408
404
|
unless resuming?
|
409
|
-
push_schema
|
405
|
+
push_schema unless skip_schema?
|
410
406
|
push_indexes if indexes_first? && !skip_schema?
|
411
407
|
end
|
412
408
|
setup_signal_trap
|
@@ -420,12 +416,12 @@ module Taps
|
|
420
416
|
def push_indexes
|
421
417
|
idxs = ::OkJson.decode(Taps::Utils.schema_bin(:indexes_individual, database_url))
|
422
418
|
|
423
|
-
return
|
419
|
+
return if idxs.empty?
|
424
420
|
|
425
|
-
puts
|
421
|
+
puts 'Sending indexes'
|
426
422
|
|
427
423
|
apply_table_filter(idxs).each do |table, indexes|
|
428
|
-
next
|
424
|
+
next if indexes.empty?
|
429
425
|
progress = ProgressBar.new(table, indexes.size)
|
430
426
|
indexes.each do |idx|
|
431
427
|
session_resource['push/indexes'].post(idx, http_headers)
|
@@ -436,10 +432,10 @@ module Taps
|
|
436
432
|
end
|
437
433
|
|
438
434
|
def push_schema
|
439
|
-
puts
|
435
|
+
puts 'Sending schema'
|
440
436
|
|
441
437
|
progress = ProgressBar.new('Schema', tables.size)
|
442
|
-
tables.each do |table,
|
438
|
+
tables.each do |table, _count|
|
443
439
|
schema_data = Taps::Utils.schema_bin(:dump_table, database_url, table)
|
444
440
|
log.debug "Table: #{table}\n#{schema_data}\n"
|
445
441
|
session_resource['push/schema'].post(schema_data, http_headers)
|
@@ -449,7 +445,7 @@ module Taps
|
|
449
445
|
end
|
450
446
|
|
451
447
|
def push_reset_sequences
|
452
|
-
puts
|
448
|
+
puts 'Resetting sequences'
|
453
449
|
|
454
450
|
session_resource['push/reset_sequences'].post('', http_headers)
|
455
451
|
end
|
@@ -466,14 +462,14 @@ module Taps
|
|
466
462
|
end
|
467
463
|
|
468
464
|
def push_data
|
469
|
-
puts
|
465
|
+
puts 'Sending data'
|
470
466
|
|
471
467
|
puts "#{tables.size} tables, #{format_number(record_count)} records"
|
472
468
|
|
473
469
|
tables.each do |table_name, count|
|
474
470
|
stream = Taps::DataStream.factory(db,
|
475
|
-
|
476
|
-
|
471
|
+
table_name: table_name,
|
472
|
+
chunksize: default_chunksize)
|
477
473
|
progress = ProgressBar.new(table_name.to_s, count)
|
478
474
|
push_data_from_table(stream, progress)
|
479
475
|
end
|
@@ -501,8 +497,8 @@ module Taps
|
|
501
497
|
data = nil
|
502
498
|
d2 = c.time_delta do
|
503
499
|
data = {
|
504
|
-
:
|
505
|
-
:
|
500
|
+
state: stream.to_hash,
|
501
|
+
checksum: Taps::Utils.checksum(encoded_data).to_s
|
506
502
|
}
|
507
503
|
end
|
508
504
|
|
@@ -510,15 +506,15 @@ module Taps
|
|
510
506
|
content, content_type = nil
|
511
507
|
d3 = c.time_delta do
|
512
508
|
content, content_type = Taps::Multipart.create do |r|
|
513
|
-
r.attach :
|
514
|
-
|
515
|
-
|
516
|
-
r.attach :
|
517
|
-
|
518
|
-
|
509
|
+
r.attach name: :encoded_data,
|
510
|
+
payload: encoded_data,
|
511
|
+
content_type: 'application/octet-stream'
|
512
|
+
r.attach name: :json,
|
513
|
+
payload: ::OkJson.encode(data),
|
514
|
+
content_type: 'application/json'
|
519
515
|
end
|
520
516
|
end
|
521
|
-
session_resource['push/table'].post(content, http_headers(:
|
517
|
+
session_resource['push/table'].post(content, http_headers(content_type: content_type))
|
522
518
|
self.stream_state = stream.to_hash
|
523
519
|
rescue => e
|
524
520
|
Taps::Utils.reraise_server_exception(e)
|
@@ -563,7 +559,7 @@ module Taps
|
|
563
559
|
end
|
564
560
|
|
565
561
|
def record_count
|
566
|
-
@record_count ||= local_tables_info.values.inject(0) { |a,c| a += c }
|
562
|
+
@record_count ||= local_tables_info.values.inject(0) { |a, c| a += c }
|
567
563
|
end
|
568
564
|
|
569
565
|
def fetch_local_tables_info
|
@@ -573,6 +569,5 @@ module Taps
|
|
573
569
|
end
|
574
570
|
apply_table_filter(tables_with_counts)
|
575
571
|
end
|
576
|
-
|
577
572
|
end
|
578
573
|
end
|