tapsoob 0.3.20 → 0.3.25
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/tapsoob/cli/root.rb +3 -1
- data/lib/tapsoob/data_stream.rb +19 -9
- data/lib/tapsoob/operation.rb +3 -3
- data/lib/tapsoob/version.rb +1 -1
- metadata +1 -1
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: cfc8db7ed412920a9e55e43d5ee66c57b0dd09a86e322c0b6f37bd90762ef261
|
4
|
+
data.tar.gz: 46dba518c294c033bf7c12c06f79278700be456d00554ad30d9149decaf8b62e
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: c2ef008de5be1f5e169055026c65b04518163e7987e34f2e1be5c554d7114be96d99f4fe7ae3284798f4e11225c1554f461db8c9cc487d1befb09ef654e8d0ad
|
7
|
+
data.tar.gz: b532c43909abc6afda5314e55da082ddbceb3a3ed285b5ba133f024f969120ec0d17b3d6f75c0423588b3fb61c015763038219f7b6c025d224c340416f7abf6c
|
data/lib/tapsoob/cli/root.rb
CHANGED
@@ -39,6 +39,7 @@ module Tapsoob
|
|
39
39
|
option :tables, desc: "Shortcut to filter on a list of tables", type: :array, aliases: "-t"
|
40
40
|
option :"exclude-tables", desc: "Shortcut to exclude a list of tables", type: :array, aliases: "-e"
|
41
41
|
option :purge, desc: "Purge data in tables prior to performing the import", default: false, type: :boolean, aliases: "-p"
|
42
|
+
option :"discard-identity", desc: "Remove identity when pushing data (may result in creating duplicates)", default: false, type: :boolean
|
42
43
|
option :debug, desc: "Enable debug messages", default: false, type: :boolean, aliases: "-d"
|
43
44
|
def push(dump_path, database_url)
|
44
45
|
opts = parse_opts(options)
|
@@ -71,8 +72,9 @@ module Tapsoob
|
|
71
72
|
debug: options[:debug]
|
72
73
|
}
|
73
74
|
|
74
|
-
#
|
75
|
+
# Push only options
|
75
76
|
opts[:purge] = options[:purge] if options.key?(:purge)
|
77
|
+
opts[:"discard-identity"] = options[:"discard-identity"] if options.key?(:"discard-identity")
|
76
78
|
|
77
79
|
# Resume
|
78
80
|
if options[:resume]
|
data/lib/tapsoob/data_stream.rb
CHANGED
@@ -6,9 +6,9 @@ module Tapsoob
|
|
6
6
|
class DataStream
|
7
7
|
DEFAULT_CHUNKSIZE = 1000
|
8
8
|
|
9
|
-
attr_reader :db, :state
|
9
|
+
attr_reader :db, :state, :options
|
10
10
|
|
11
|
-
def initialize(db, state)
|
11
|
+
def initialize(db, state, opts = {})
|
12
12
|
@db = db
|
13
13
|
@state = {
|
14
14
|
:offset => 0,
|
@@ -17,6 +17,7 @@ module Tapsoob
|
|
17
17
|
:total_chunksize => 0
|
18
18
|
}.merge(state)
|
19
19
|
@state[:chunksize] ||= DEFAULT_CHUNKSIZE
|
20
|
+
@options = opts
|
20
21
|
@complete = false
|
21
22
|
end
|
22
23
|
|
@@ -218,6 +219,9 @@ module Tapsoob
|
|
218
219
|
end
|
219
220
|
|
220
221
|
def import_rows(rows)
|
222
|
+
columns = rows[:header]
|
223
|
+
data = rows[:data]
|
224
|
+
|
221
225
|
# Decode blobs
|
222
226
|
if rows.has_key?(:types) && rows[:types].include?("blob")
|
223
227
|
blob_indices = rows[:types].each_index.select { |idx| rows[:types][idx] == "blob" }
|
@@ -227,8 +231,14 @@ module Tapsoob
|
|
227
231
|
end
|
228
232
|
end
|
229
233
|
end
|
234
|
+
|
235
|
+
# Remove id column
|
236
|
+
if @options[:"discard-identity"]
|
237
|
+
columns = rows[:header] - ["id"]
|
238
|
+
data = data.map { |d| d[1..-1] }
|
239
|
+
end
|
230
240
|
|
231
|
-
table.import(
|
241
|
+
table.import(columns, data, :commit_every => 100)
|
232
242
|
state[:offset] += rows[:data].size
|
233
243
|
rescue Exception => ex
|
234
244
|
case ex.message
|
@@ -246,19 +256,19 @@ module Tapsoob
|
|
246
256
|
state[:offset] = table.count
|
247
257
|
end
|
248
258
|
|
249
|
-
def self.factory(db, state)
|
259
|
+
def self.factory(db, state, opts)
|
250
260
|
if defined?(Sequel::MySQL) && Sequel::MySQL.respond_to?(:convert_invalid_date_time=)
|
251
261
|
Sequel::MySQL.convert_invalid_date_time = :nil
|
252
262
|
end
|
253
263
|
|
254
264
|
if state.has_key?(:klass)
|
255
|
-
return eval(state[:klass]).new(db, state)
|
265
|
+
return eval(state[:klass]).new(db, state, opts)
|
256
266
|
end
|
257
267
|
|
258
268
|
if Tapsoob::Utils.single_integer_primary_key(db, state[:table_name].to_sym)
|
259
|
-
DataStreamKeyed.new(db, state)
|
269
|
+
DataStreamKeyed.new(db, state, opts)
|
260
270
|
else
|
261
|
-
DataStream.new(db, state)
|
271
|
+
DataStream.new(db, state, opts)
|
262
272
|
end
|
263
273
|
end
|
264
274
|
end
|
@@ -266,8 +276,8 @@ module Tapsoob
|
|
266
276
|
class DataStreamKeyed < DataStream
|
267
277
|
attr_accessor :buffer
|
268
278
|
|
269
|
-
def initialize(db, state)
|
270
|
-
super(db, state)
|
279
|
+
def initialize(db, state, opts = {})
|
280
|
+
super(db, state, opts)
|
271
281
|
@state = { :primary_key => order_by(state[:table_name]).first, :filter => 0 }.merge(@state)
|
272
282
|
@state[:chunksize] ||= DEFAULT_CHUNKSIZE
|
273
283
|
@buffer = []
|
data/lib/tapsoob/operation.rb
CHANGED
@@ -394,9 +394,9 @@ module Tapsoob
|
|
394
394
|
tables.each do |table_name, count|
|
395
395
|
next unless File.exists?(File.join(dump_path, "data", "#{table_name}.json"))
|
396
396
|
db[table_name.to_sym].truncate if @opts[:purge]
|
397
|
-
stream = Tapsoob::DataStream.factory(db,
|
397
|
+
stream = Tapsoob::DataStream.factory(db, {
|
398
398
|
:table_name => table_name,
|
399
|
-
:chunksize => default_chunksize)
|
399
|
+
:chunksize => default_chunksize }, { :"discard-identity" => @opts[:"discard-identity"] || false })
|
400
400
|
progress = ProgressBar.new(table_name.to_s, count)
|
401
401
|
push_data_from_file(stream, progress)
|
402
402
|
end
|
@@ -477,7 +477,7 @@ module Tapsoob
|
|
477
477
|
|
478
478
|
def fetch_local_tables_info
|
479
479
|
tables_with_counts = {}
|
480
|
-
tbls = Dir.glob(File.join(dump_path, "
|
480
|
+
tbls = Dir.glob(File.join(dump_path, "data", "*")).map { |path| File.basename(path, ".json") }
|
481
481
|
tbls.each do |table|
|
482
482
|
if File.exists?(File.join(dump_path, "data", "#{table}.json"))
|
483
483
|
data = JSON.parse(File.read(File.join(dump_path, "data", "#{table}.json")))
|
data/lib/tapsoob/version.rb
CHANGED