tapsoob 0.4.27 → 0.5.2
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/tapsoob/cli/root.rb +3 -0
- data/lib/tapsoob/data_stream.rb +14 -50
- data/lib/tapsoob/operation.rb +48 -18
- data/lib/tapsoob/version.rb +1 -1
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 10ae20ca73c6821c188d7345388336477aa6517e8761d2caefccf1b0df073556
|
4
|
+
data.tar.gz: f36c26256ff6531439a4c1e0f1e93694128157a17b4e9a4d92c98fc45e3e03d0
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 4d052875c6dc6d045c673b2e8d6505be0823d8a9ec45ed2c47c8ab8cfe9a9b17007cef94db6a05bece6a95900eae992888be12585e906bef8f40eede6bc96848
|
7
|
+
data.tar.gz: 3e3b6d5c73ae96dc28c44777207c6f7a70379e6c9efbbfa6ab241b2324a3f759d1c970e91cb868f06659945498dafbbb4e4f62f14b02cc3a623b0d3a23d7446f
|
data/lib/tapsoob/cli/root.rb
CHANGED
@@ -18,6 +18,7 @@ module Tapsoob
|
|
18
18
|
option :"disable-compression", desc: "Disable Compression", default: false, type: :boolean, aliases: "-g"
|
19
19
|
option :tables, desc: "Shortcut to filter on a list of tables", type: :array, aliases: "-t"
|
20
20
|
option :"exclude-tables", desc: "Shortcut to exclude a list of tables", type: :array, aliases: "-e"
|
21
|
+
option :progress, desc: "Show progress", default: true, type: :boolean
|
21
22
|
option :debug, desc: "Enable debug messages", default: false, type: :boolean, aliases: "-d"
|
22
23
|
def pull(dump_path, database_url)
|
23
24
|
opts = parse_opts(options)
|
@@ -41,6 +42,7 @@ module Tapsoob
|
|
41
42
|
option :purge, desc: "Purge data in tables prior to performing the import", default: false, type: :boolean, aliases: "-p"
|
42
43
|
option :"skip-duplicates", desc: "Remove duplicates when loading data", default: false, type: :boolean
|
43
44
|
option :"discard-identity", desc: "Remove identity when pushing data (may result in creating duplicates)", default: false, type: :boolean
|
45
|
+
option :progress, desc: "Show progress", default: true, type: :boolean
|
44
46
|
option :debug, desc: "Enable debug messages", default: false, type: :boolean, aliases: "-d"
|
45
47
|
def push(dump_path, database_url)
|
46
48
|
opts = parse_opts(options)
|
@@ -72,6 +74,7 @@ module Tapsoob
|
|
72
74
|
indexes_first: options[:"indexes_first"],
|
73
75
|
disable_compression: options[:"disable-compression"],
|
74
76
|
tables: options[:tables],
|
77
|
+
progress: options[:progress],
|
75
78
|
debug: options[:debug]
|
76
79
|
}
|
77
80
|
|
data/lib/tapsoob/data_stream.rb
CHANGED
@@ -74,6 +74,7 @@ module Tapsoob
|
|
74
74
|
def fetch_rows
|
75
75
|
state[:chunksize] = fetch_chunksize
|
76
76
|
ds = table.order(*order_by).limit(state[:chunksize], state[:offset])
|
77
|
+
state[:size] = table.count
|
77
78
|
log.debug "DataStream#fetch_rows SQL -> #{ds.sql}"
|
78
79
|
rows = Tapsoob::Utils.format_data(db, ds.all,
|
79
80
|
:string_columns => string_columns,
|
@@ -87,6 +88,7 @@ module Tapsoob
|
|
87
88
|
def fetch_file(dump_path)
|
88
89
|
state[:chunksize] = fetch_chunksize
|
89
90
|
ds = JSON.parse(File.read(File.join(dump_path, "data", "#{table_name}.json")))
|
91
|
+
state[:size] = ds["data"].size
|
90
92
|
log.debug "DataStream#fetch_file"
|
91
93
|
rows = {
|
92
94
|
:table_name => ds["table_name"],
|
@@ -132,71 +134,34 @@ module Tapsoob
|
|
132
134
|
t2 = Time.now
|
133
135
|
elapsed_time = t2 - t1
|
134
136
|
|
135
|
-
|
136
|
-
@complete = rows[:data] == [ ]
|
137
|
-
else
|
138
|
-
@complete = rows == { }
|
139
|
-
end
|
137
|
+
state[:offset] += (rows == {} ? 0 : rows[:data].size)
|
140
138
|
|
141
|
-
[encoded_data, (
|
139
|
+
[encoded_data, (rows == {} ? 0 : rows[:data].size), elapsed_time]
|
142
140
|
end
|
143
141
|
|
144
142
|
def complete?
|
145
|
-
|
143
|
+
state[:offset] >= state[:size]
|
146
144
|
end
|
147
145
|
|
148
|
-
def
|
149
|
-
params = fetch_from_database
|
146
|
+
def fetch_data_from_database(params)
|
150
147
|
encoded_data = params[:encoded_data]
|
151
|
-
json = params[:json]
|
152
148
|
|
153
|
-
rows = parse_encoded_data(encoded_data,
|
154
|
-
|
155
|
-
@complete = rows == { }
|
149
|
+
rows = parse_encoded_data(encoded_data, params[:checksum])
|
156
150
|
|
157
151
|
# update local state
|
158
|
-
state.merge!(
|
159
|
-
|
160
|
-
unless @complete
|
161
|
-
yield rows if block_given?
|
162
|
-
state[:offset] += rows[:data].size
|
163
|
-
rows[:data].size
|
164
|
-
else
|
165
|
-
0
|
166
|
-
end
|
167
|
-
end
|
152
|
+
state.merge!(params[:state].merge(:chunksize => state[:chunksize]))
|
168
153
|
|
169
|
-
|
170
|
-
|
171
|
-
log.debug "DataStream#fetch_from_database state -> #{state.inspect}"
|
172
|
-
state[:chunksize] = Tapsoob::Utils.calculate_chunksize(state[:chunksize]) do |c|
|
173
|
-
state[:chunksize] = c.to_i
|
174
|
-
encoded_data = fetch.first
|
175
|
-
|
176
|
-
checksum = Tapsoob::Utils.checksum(encoded_data).to_s
|
177
|
-
|
178
|
-
res = {
|
179
|
-
:json => { :checksum => checksum, :state => to_hash },
|
180
|
-
:encoded_data => encoded_data
|
181
|
-
}
|
182
|
-
end
|
183
|
-
|
184
|
-
res
|
154
|
+
yield rows if block_given?
|
155
|
+
(rows == {} ? 0 : rows[:data].size)
|
185
156
|
end
|
186
157
|
|
187
|
-
def
|
158
|
+
def fetch_data_to_database(params)
|
188
159
|
encoded_data = params[:encoded_data]
|
189
160
|
|
190
161
|
rows = parse_encoded_data(encoded_data, params[:checksum])
|
191
|
-
|
192
|
-
|
193
|
-
|
194
|
-
unless @complete
|
195
|
-
import_rows(rows)
|
196
|
-
rows[:data].size
|
197
|
-
else
|
198
|
-
0
|
199
|
-
end
|
162
|
+
|
163
|
+
import_rows(rows)
|
164
|
+
(rows == {} ? 0 : rows[:data].size)
|
200
165
|
end
|
201
166
|
|
202
167
|
def self.parse_json(json)
|
@@ -266,7 +231,6 @@ module Tapsoob
|
|
266
231
|
end
|
267
232
|
|
268
233
|
table.import(columns, data, :commit_every => 100)
|
269
|
-
state[:offset] += rows[:data].size
|
270
234
|
rescue Exception => ex
|
271
235
|
case ex.message
|
272
236
|
when /integer out of range/ then
|
data/lib/tapsoob/operation.rb
CHANGED
@@ -220,25 +220,56 @@ module Tapsoob
|
|
220
220
|
|
221
221
|
def pull_data_from_table(stream, progress)
|
222
222
|
loop do
|
223
|
-
|
224
|
-
|
223
|
+
if exiting?
|
224
|
+
store_session
|
225
|
+
exit 0
|
226
|
+
end
|
225
227
|
|
226
|
-
|
227
|
-
|
228
|
-
|
229
|
-
|
230
|
-
|
228
|
+
row_size = 0
|
229
|
+
chunksize = stream.state[:chunksize]
|
230
|
+
|
231
|
+
begin
|
232
|
+
chunksize = Tapsoob::Utils.calculate_chunksize(chunksize) do |c|
|
233
|
+
stream.state[:chunksize] = c.to_i
|
234
|
+
encoded_data, row_size, elapsed_time = nil
|
235
|
+
d1 = c.time_delta do
|
236
|
+
encoded_data, row_size, elapsed_time = stream.fetch
|
231
237
|
end
|
238
|
+
|
239
|
+
data = nil
|
240
|
+
d2 = c.time_delta do
|
241
|
+
data = {
|
242
|
+
:state => stream.to_hash,
|
243
|
+
:checksum => Tapsoob::Utils.checksum(encoded_data).to_s,
|
244
|
+
:encoded_data => encoded_data
|
245
|
+
}
|
246
|
+
end
|
247
|
+
|
248
|
+
stream.fetch_data_from_database(data) do |rows|
|
249
|
+
next if rows == {}
|
250
|
+
|
251
|
+
if dump_path.nil?
|
252
|
+
puts JSON.generate(rows)
|
253
|
+
else
|
254
|
+
Tapsoob::Utils.export_rows(dump_path, stream.table_name, rows)
|
255
|
+
end
|
256
|
+
end
|
257
|
+
log.debug "row size: #{row_size}"
|
258
|
+
stream.error = false
|
259
|
+
self.stream_state = stream.to_hash
|
260
|
+
|
261
|
+
c.idle_secs = (d1 + d2)
|
262
|
+
|
263
|
+
elapsed_time
|
232
264
|
end
|
233
|
-
stream.error = false
|
234
|
-
self.stream_state = stream.to_hash
|
235
265
|
rescue Tapsoob::CorruptedData => e
|
236
266
|
log.info "Corrupted Data Received #{e.message}, retrying..."
|
237
267
|
stream.error = true
|
238
268
|
next
|
239
269
|
end
|
240
270
|
|
241
|
-
progress.inc(
|
271
|
+
progress.inc(row_size) if progress
|
272
|
+
|
242
273
|
break if stream.complete?
|
243
274
|
end
|
244
275
|
|
@@ -407,7 +438,7 @@ module Tapsoob
|
|
407
438
|
:purge => opts[:purge] || false,
|
408
439
|
:debug => opts[:debug]
|
409
440
|
})
|
410
|
-
progress = ProgressBar.new(table_name.to_s, count)
|
441
|
+
progress = (opts[:progress] ? ProgressBar.new(table_name.to_s, count) : nil)
|
411
442
|
push_data_from_file(stream, progress)
|
412
443
|
end
|
413
444
|
end
|
@@ -429,17 +460,17 @@ module Tapsoob
|
|
429
460
|
d1 = c.time_delta do
|
430
461
|
encoded_data, row_size, elapsed_time = stream.fetch({ :type => "file", :source => dump_path })
|
431
462
|
end
|
432
|
-
break if stream.complete?
|
433
463
|
|
434
464
|
data = nil
|
435
465
|
d2 = c.time_delta do
|
436
466
|
data = {
|
437
|
-
:state
|
438
|
-
:checksum
|
467
|
+
:state => stream.to_hash,
|
468
|
+
:checksum => Tapsoob::Utils.checksum(encoded_data).to_s,
|
469
|
+
:encoded_data => encoded_data
|
439
470
|
}
|
440
471
|
end
|
441
472
|
|
442
|
-
|
473
|
+
stream.fetch_data_to_database(data)
|
443
474
|
log.debug "row size: #{row_size}"
|
444
475
|
self.stream_state = stream.to_hash
|
445
476
|
|
@@ -458,13 +489,12 @@ module Tapsoob
|
|
458
489
|
end
|
459
490
|
stream.state[:chunksize] = chunksize
|
460
491
|
|
461
|
-
progress.inc(row_size)
|
492
|
+
progress.inc(row_size) if progress
|
462
493
|
|
463
|
-
stream.increment(row_size)
|
464
494
|
break if stream.complete?
|
465
495
|
end
|
466
496
|
|
467
|
-
progress.finish
|
497
|
+
progress.finish if progress
|
468
498
|
completed_tables << stream.table_name.to_s
|
469
499
|
self.stream_state = {}
|
470
500
|
end
|
data/lib/tapsoob/version.rb
CHANGED
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: tapsoob
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.5.2
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Félix Bellanger
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2022-01
|
12
|
+
date: 2022-02-01 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: ripl
|