tapsoob 0.4.26-java → 0.5.4-java
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/tapsoob/cli/data_stream.rb +1 -1
- data/lib/tapsoob/cli/root.rb +11 -5
- data/lib/tapsoob/data_stream.rb +14 -50
- data/lib/tapsoob/operation.rb +64 -30
- data/lib/tapsoob/version.rb +1 -1
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 1b6104bbadc92d741435a9773c00b8645c3d6272a7a341a23fd93d0a7c0f205d
|
4
|
+
data.tar.gz: 459b8833771b33ca771d2813501f7b9dcbf89446a994d398076b9bf9fe90b9ea
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: decc3c7ca389bab2014d244d3c25ab9feaded6847bdbb74781872d6bfdce47f2ea851630f81ec349977dcb98d1b0f2a2d2f4e9898170665726631231a9b7eabc
|
7
|
+
data.tar.gz: 412420cdf70186f69d67e4d822afae86c5c5b97789688d1b801800846a078e297a130d5c73196a7ad77a3aa46727d896648f8605a8932d41217b720b3983f112
|
@@ -26,7 +26,7 @@ module Tapsoob
|
|
26
26
|
option :tables, desc: "Shortcut to filter on a list of tables", type: :array, aliases: "-t"
|
27
27
|
option :"exclude-tables", desc: "Shortcut to exclude a list of tables", type: :array, aliases: "-e"
|
28
28
|
option :progress, desc: "Show progress", default: true, type: :boolean, aliases: "-p"
|
29
|
-
option :purge, desc: "Purge data in tables prior to performing the import", default: false, type: :boolean
|
29
|
+
option :purge, desc: "Purge data in tables prior to performing the import", default: false, type: :boolean
|
30
30
|
option :"skip-duplicates", desc: "Remove duplicates when loading data", default: false, type: :boolean
|
31
31
|
option :"discard-identity", desc: "Remove identity when pushing data (may result in creating duplicates)", default: false, type: :boolean
|
32
32
|
option :debug, desc: "Enable debug messages", default: false, type: :boolean, aliases: "-d"
|
data/lib/tapsoob/cli/root.rb
CHANGED
@@ -10,14 +10,16 @@ module Tapsoob
|
|
10
10
|
module CLI
|
11
11
|
class Root < Thor
|
12
12
|
desc "pull DUMP_PATH DATABASE_URL", "Pull a dump from a database to a folder"
|
13
|
-
option :
|
13
|
+
option :data, desc: "Pull the data to the database", default: true, type: :boolean, aliases: '-d'
|
14
|
+
option :schema, desc: "Pull the schema to the database", default: true, type: :boolean, aliases: "-s"
|
14
15
|
option :"indexes-first", desc: "Transfer indexes first before data", default: false, type: :boolean, aliases: "-i"
|
15
16
|
option :resume, desc: "Resume a Tapsoob Session from a stored file", type: :string, aliases: "-r"
|
16
17
|
option :chunksize, desc: "Initial chunksize", default: 1000, type: :numeric, aliases: "-c"
|
17
18
|
option :"disable-compression", desc: "Disable Compression", default: false, type: :boolean, aliases: "-g"
|
18
19
|
option :tables, desc: "Shortcut to filter on a list of tables", type: :array, aliases: "-t"
|
19
20
|
option :"exclude-tables", desc: "Shortcut to exclude a list of tables", type: :array, aliases: "-e"
|
20
|
-
option :
|
21
|
+
option :progress, desc: "Show progress", default: true, type: :boolean
|
22
|
+
option :debug, desc: "Enable debug messages", default: false, type: :boolean
|
21
23
|
def pull(dump_path, database_url)
|
22
24
|
opts = parse_opts(options)
|
23
25
|
Tapsoob.log.level = Logger::DEBUG if opts[:debug]
|
@@ -29,7 +31,8 @@ module Tapsoob
|
|
29
31
|
end
|
30
32
|
|
31
33
|
desc "push DUMP_PATH DATABASE_URL", "Push a previously tapsoob dump to a database"
|
32
|
-
option :
|
34
|
+
option :data, desc: "Push the data to the database", default: true, type: :boolean, aliases: '-d'
|
35
|
+
option :schema, desc: "Push the schema to the database", default: true, type: :boolean, aliases: "-s"
|
33
36
|
option :"indexes-first", desc: "Transfer indexes first before data", default: false, type: :boolean, aliases: "-i"
|
34
37
|
option :resume, desc: "Resume a Tapsoob Session from a stored file", type: :string, aliases: "-r"
|
35
38
|
option :chunksize, desc: "Initial chunksize", default: 1000, type: :numeric, aliases: "-c"
|
@@ -39,7 +42,8 @@ module Tapsoob
|
|
39
42
|
option :purge, desc: "Purge data in tables prior to performing the import", default: false, type: :boolean, aliases: "-p"
|
40
43
|
option :"skip-duplicates", desc: "Remove duplicates when loading data", default: false, type: :boolean
|
41
44
|
option :"discard-identity", desc: "Remove identity when pushing data (may result in creating duplicates)", default: false, type: :boolean
|
42
|
-
option :
|
45
|
+
option :progress, desc: "Show progress", default: true, type: :boolean
|
46
|
+
option :debug, desc: "Enable debug messages", default: false, type: :boolean
|
43
47
|
def push(dump_path, database_url)
|
44
48
|
opts = parse_opts(options)
|
45
49
|
Tapsoob.log.level = Logger::DEBUG if opts[:debug]
|
@@ -65,10 +69,12 @@ module Tapsoob
|
|
65
69
|
def parse_opts(options)
|
66
70
|
# Default options
|
67
71
|
opts = {
|
68
|
-
|
72
|
+
data: options[:data],
|
73
|
+
schema: options[:schema],
|
69
74
|
indexes_first: options[:"indexes_first"],
|
70
75
|
disable_compression: options[:"disable-compression"],
|
71
76
|
tables: options[:tables],
|
77
|
+
progress: options[:progress],
|
72
78
|
debug: options[:debug]
|
73
79
|
}
|
74
80
|
|
data/lib/tapsoob/data_stream.rb
CHANGED
@@ -74,6 +74,7 @@ module Tapsoob
|
|
74
74
|
def fetch_rows
|
75
75
|
state[:chunksize] = fetch_chunksize
|
76
76
|
ds = table.order(*order_by).limit(state[:chunksize], state[:offset])
|
77
|
+
state[:size] = table.count
|
77
78
|
log.debug "DataStream#fetch_rows SQL -> #{ds.sql}"
|
78
79
|
rows = Tapsoob::Utils.format_data(db, ds.all,
|
79
80
|
:string_columns => string_columns,
|
@@ -87,6 +88,7 @@ module Tapsoob
|
|
87
88
|
def fetch_file(dump_path)
|
88
89
|
state[:chunksize] = fetch_chunksize
|
89
90
|
ds = JSON.parse(File.read(File.join(dump_path, "data", "#{table_name}.json")))
|
91
|
+
state[:size] = ds["data"].size
|
90
92
|
log.debug "DataStream#fetch_file"
|
91
93
|
rows = {
|
92
94
|
:table_name => ds["table_name"],
|
@@ -132,71 +134,34 @@ module Tapsoob
|
|
132
134
|
t2 = Time.now
|
133
135
|
elapsed_time = t2 - t1
|
134
136
|
|
135
|
-
|
136
|
-
@complete = rows[:data] == [ ]
|
137
|
-
else
|
138
|
-
@complete = rows == { }
|
139
|
-
end
|
137
|
+
state[:offset] += (rows == {} ? 0 : rows[:data].size)
|
140
138
|
|
141
|
-
[encoded_data, (
|
139
|
+
[encoded_data, (rows == {} ? 0 : rows[:data].size), elapsed_time]
|
142
140
|
end
|
143
141
|
|
144
142
|
def complete?
|
145
|
-
|
143
|
+
state[:offset] >= state[:size]
|
146
144
|
end
|
147
145
|
|
148
|
-
def
|
149
|
-
params = fetch_from_database
|
146
|
+
def fetch_data_from_database(params)
|
150
147
|
encoded_data = params[:encoded_data]
|
151
|
-
json = params[:json]
|
152
148
|
|
153
|
-
rows = parse_encoded_data(encoded_data,
|
154
|
-
|
155
|
-
@complete = rows == { }
|
149
|
+
rows = parse_encoded_data(encoded_data, params[:checksum])
|
156
150
|
|
157
151
|
# update local state
|
158
|
-
state.merge!(
|
159
|
-
|
160
|
-
unless @complete
|
161
|
-
yield rows if block_given?
|
162
|
-
state[:offset] += rows[:data].size
|
163
|
-
rows[:data].size
|
164
|
-
else
|
165
|
-
0
|
166
|
-
end
|
167
|
-
end
|
152
|
+
state.merge!(params[:state].merge(:chunksize => state[:chunksize]))
|
168
153
|
|
169
|
-
|
170
|
-
|
171
|
-
log.debug "DataStream#fetch_from_database state -> #{state.inspect}"
|
172
|
-
state[:chunksize] = Tapsoob::Utils.calculate_chunksize(state[:chunksize]) do |c|
|
173
|
-
state[:chunksize] = c.to_i
|
174
|
-
encoded_data = fetch.first
|
175
|
-
|
176
|
-
checksum = Tapsoob::Utils.checksum(encoded_data).to_s
|
177
|
-
|
178
|
-
res = {
|
179
|
-
:json => { :checksum => checksum, :state => to_hash },
|
180
|
-
:encoded_data => encoded_data
|
181
|
-
}
|
182
|
-
end
|
183
|
-
|
184
|
-
res
|
154
|
+
yield rows if block_given?
|
155
|
+
(rows == {} ? 0 : rows[:data].size)
|
185
156
|
end
|
186
157
|
|
187
|
-
def
|
158
|
+
def fetch_data_to_database(params)
|
188
159
|
encoded_data = params[:encoded_data]
|
189
160
|
|
190
161
|
rows = parse_encoded_data(encoded_data, params[:checksum])
|
191
|
-
|
192
|
-
|
193
|
-
|
194
|
-
unless @complete
|
195
|
-
import_rows(rows)
|
196
|
-
rows[:data].size
|
197
|
-
else
|
198
|
-
0
|
199
|
-
end
|
162
|
+
|
163
|
+
import_rows(rows)
|
164
|
+
(rows == {} ? 0 : rows[:data].size)
|
200
165
|
end
|
201
166
|
|
202
167
|
def self.parse_json(json)
|
@@ -266,7 +231,6 @@ module Tapsoob
|
|
266
231
|
end
|
267
232
|
|
268
233
|
table.import(columns, data, :commit_every => 100)
|
269
|
-
state[:offset] += rows[:data].size
|
270
234
|
rescue Exception => ex
|
271
235
|
case ex.message
|
272
236
|
when /integer out of range/ then
|
data/lib/tapsoob/operation.rb
CHANGED
@@ -21,8 +21,12 @@ module Tapsoob
|
|
21
21
|
"op"
|
22
22
|
end
|
23
23
|
|
24
|
-
def
|
25
|
-
|
24
|
+
def data?
|
25
|
+
opts[:data]
|
26
|
+
end
|
27
|
+
|
28
|
+
def schema?
|
29
|
+
opts[:schema]
|
26
30
|
end
|
27
31
|
|
28
32
|
def indexes_first?
|
@@ -162,13 +166,13 @@ module Tapsoob
|
|
162
166
|
def run
|
163
167
|
catch_errors do
|
164
168
|
unless resuming?
|
165
|
-
pull_schema if
|
166
|
-
pull_indexes if indexes_first? &&
|
169
|
+
pull_schema if schema?
|
170
|
+
pull_indexes if indexes_first? && schema?
|
167
171
|
end
|
168
172
|
setup_signal_trap
|
169
|
-
pull_partial_data if resuming?
|
170
|
-
pull_data
|
171
|
-
pull_indexes if !indexes_first? &&
|
173
|
+
pull_partial_data if data? && resuming?
|
174
|
+
pull_data if data?
|
175
|
+
pull_indexes if !indexes_first? && schema?
|
172
176
|
pull_reset_sequences
|
173
177
|
end
|
174
178
|
end
|
@@ -216,25 +220,56 @@ module Tapsoob
|
|
216
220
|
|
217
221
|
def pull_data_from_table(stream, progress)
|
218
222
|
loop do
|
219
|
-
|
220
|
-
|
223
|
+
if exiting?
|
224
|
+
store_session
|
225
|
+
exit 0
|
226
|
+
end
|
221
227
|
|
222
|
-
|
223
|
-
|
224
|
-
|
225
|
-
|
226
|
-
|
228
|
+
row_size = 0
|
229
|
+
chunksize = stream.state[:chunksize]
|
230
|
+
|
231
|
+
begin
|
232
|
+
chunksize = Tapsoob::Utils.calculate_chunksize(chunksize) do |c|
|
233
|
+
stream.state[:chunksize] = c.to_i
|
234
|
+
encoded_data, row_size, elapsed_time = nil
|
235
|
+
d1 = c.time_delta do
|
236
|
+
encoded_data, row_size, elapsed_time = stream.fetch
|
237
|
+
end
|
238
|
+
|
239
|
+
data = nil
|
240
|
+
d2 = c.time_delta do
|
241
|
+
data = {
|
242
|
+
:state => stream.to_hash,
|
243
|
+
:checksum => Tapsoob::Utils.checksum(encoded_data).to_s,
|
244
|
+
:encoded_data => encoded_data
|
245
|
+
}
|
246
|
+
end
|
247
|
+
|
248
|
+
stream.fetch_data_from_database(data) do |rows|
|
249
|
+
next if rows == {}
|
250
|
+
|
251
|
+
if dump_path.nil?
|
252
|
+
puts JSON.generate(rows)
|
253
|
+
else
|
254
|
+
Tapsoob::Utils.export_rows(dump_path, stream.table_name, rows)
|
255
|
+
end
|
227
256
|
end
|
257
|
+
log.debug "row size: #{row_size}"
|
258
|
+
stream.error = false
|
259
|
+
self.stream_state = stream.to_hash
|
260
|
+
|
261
|
+
c.idle_secs = (d1 + d2)
|
262
|
+
|
263
|
+
elapsed_time
|
228
264
|
end
|
229
|
-
stream.error = false
|
230
|
-
self.stream_state = stream.to_hash
|
231
265
|
rescue Tapsoob::CorruptedData => e
|
232
266
|
log.info "Corrupted Data Received #{e.message}, retrying..."
|
233
267
|
stream.error = true
|
234
268
|
next
|
235
269
|
end
|
236
270
|
|
237
|
-
progress.inc(
|
271
|
+
progress.inc(row_size) if progress
|
272
|
+
|
238
273
|
break if stream.complete?
|
239
274
|
end
|
240
275
|
|
@@ -324,13 +359,13 @@ module Tapsoob
|
|
324
359
|
def run
|
325
360
|
catch_errors do
|
326
361
|
unless resuming?
|
327
|
-
push_schema if
|
328
|
-
push_indexes if indexes_first? &&
|
362
|
+
push_schema if schema?
|
363
|
+
push_indexes if indexes_first? && schema?
|
329
364
|
end
|
330
365
|
setup_signal_trap
|
331
|
-
push_partial_data if resuming?
|
332
|
-
push_data
|
333
|
-
push_indexes if !indexes_first? &&
|
366
|
+
push_partial_data if data? && resuming?
|
367
|
+
push_data if data?
|
368
|
+
push_indexes if !indexes_first? && schema?
|
334
369
|
push_reset_sequences
|
335
370
|
end
|
336
371
|
end
|
@@ -403,7 +438,7 @@ module Tapsoob
|
|
403
438
|
:purge => opts[:purge] || false,
|
404
439
|
:debug => opts[:debug]
|
405
440
|
})
|
406
|
-
progress = ProgressBar.new(table_name.to_s, count)
|
441
|
+
progress = (opts[:progress] ? ProgressBar.new(table_name.to_s, count) : nil)
|
407
442
|
push_data_from_file(stream, progress)
|
408
443
|
end
|
409
444
|
end
|
@@ -425,17 +460,17 @@ module Tapsoob
|
|
425
460
|
d1 = c.time_delta do
|
426
461
|
encoded_data, row_size, elapsed_time = stream.fetch({ :type => "file", :source => dump_path })
|
427
462
|
end
|
428
|
-
break if stream.complete?
|
429
463
|
|
430
464
|
data = nil
|
431
465
|
d2 = c.time_delta do
|
432
466
|
data = {
|
433
|
-
:state
|
434
|
-
:checksum
|
467
|
+
:state => stream.to_hash,
|
468
|
+
:checksum => Tapsoob::Utils.checksum(encoded_data).to_s,
|
469
|
+
:encoded_data => encoded_data
|
435
470
|
}
|
436
471
|
end
|
437
472
|
|
438
|
-
|
473
|
+
stream.fetch_data_to_database(data)
|
439
474
|
log.debug "row size: #{row_size}"
|
440
475
|
self.stream_state = stream.to_hash
|
441
476
|
|
@@ -454,13 +489,12 @@ module Tapsoob
|
|
454
489
|
end
|
455
490
|
stream.state[:chunksize] = chunksize
|
456
491
|
|
457
|
-
progress.inc(row_size)
|
492
|
+
progress.inc(row_size) if progress
|
458
493
|
|
459
|
-
stream.increment(row_size)
|
460
494
|
break if stream.complete?
|
461
495
|
end
|
462
496
|
|
463
|
-
progress.finish
|
497
|
+
progress.finish if progress
|
464
498
|
completed_tables << stream.table_name.to_s
|
465
499
|
self.stream_state = {}
|
466
500
|
end
|
data/lib/tapsoob/version.rb
CHANGED
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: tapsoob
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.4
|
4
|
+
version: 0.5.4
|
5
5
|
platform: java
|
6
6
|
authors:
|
7
7
|
- Félix Bellanger
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2022-
|
12
|
+
date: 2022-02-15 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
requirement: !ruby/object:Gem::Requirement
|